From 4cc8484d5893a98fb6d25e9fd99228424bdd0df6 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 11 Jul 2022 12:25:39 -0400 Subject: [PATCH 01/40] Improve error when sorting on incompatible types (#88399) Currently when sorting on incompatible types, we get class_cast_exception error (code 500). This patch improves the error to explain that the problem is because of incompatible sort types for the field across different shards and returns user error (code 400). Closes #73146 --- docs/changelog/88399.yaml | 6 +++ .../search/sort/FieldSortIT.java | 39 ++++++++++++++ .../action/search/SearchPhaseController.java | 53 +++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 docs/changelog/88399.yaml diff --git a/docs/changelog/88399.yaml b/docs/changelog/88399.yaml new file mode 100644 index 0000000000000..f38fc092ae629 --- /dev/null +++ b/docs/changelog/88399.yaml @@ -0,0 +1,6 @@ +pr: 88399 +summary: Improve error when sorting on incompatible types +area: Search +type: enhancement +issues: + - 73146 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java index 1593aed61df44..4e4997134fdcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -2122,4 +2122,43 @@ public void testLongSortOptimizationCorrectResults() { } } + public void testSortMixedFieldTypes() { + assertAcked(prepareCreate("index_long").setMapping("foo", "type=long").get()); + assertAcked(prepareCreate("index_integer").setMapping("foo", "type=integer").get()); + assertAcked(prepareCreate("index_double").setMapping("foo", "type=double").get()); + assertAcked(prepareCreate("index_keyword").setMapping("foo", "type=keyword").get()); + + client().prepareIndex("index_long").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_integer").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_double").setId("1").setSource("foo", "123").get(); + client().prepareIndex("index_keyword").setId("1").setSource("foo", "123").get(); + refresh(); + + { // mixing long and integer types is ok, as we convert integer sort to long sort + SearchResponse searchResponse = client().prepareSearch("index_long", "index_integer") + .addSort(new FieldSortBuilder("foo")) + .setSize(10) + .get(); + assertSearchResponse(searchResponse); + } + + String errMsg = "Can't sort on field [foo]; the field has incompatible sort types"; + + { // mixing long and double types is not allowed + SearchPhaseExecutionException exc = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("index_long", "index_double").addSort(new FieldSortBuilder("foo")).setSize(10).get() + ); + assertThat(exc.getCause().toString(), containsString(errMsg)); + } + + { // mixing long and keyword types is not allowed + SearchPhaseExecutionException exc = expectThrows( + SearchPhaseExecutionException.class, + () -> client().prepareSearch("index_long", "index_keyword").addSort(new FieldSortBuilder("foo")).setSize(10).get() + ); + assertThat(exc.getCause().toString(), containsString(errMsg)); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index b119fcfb45bc3..aa44ab318b6f1 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -14,6 +14,8 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopFieldDocs; @@ -197,6 +199,7 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { final TopFieldGroups[] shardTopDocs = results.toArray(new TopFieldGroups[numShards]); mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false); } else if (topDocs instanceof TopFieldDocs firstTopDocs) { + checkSameSortTypes(results, firstTopDocs.fields); final Sort sort = new Sort(firstTopDocs.fields); final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs); @@ -207,6 +210,56 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { return mergedTopDocs; } + private static void checkSameSortTypes(Collection results, SortField[] firstSortFields) { + if (results.size() < 2) return; + + SortField.Type[] firstTypes = new SortField.Type[firstSortFields.length]; + boolean isFirstResult = true; + for (TopDocs topDocs : results) { + SortField[] curSortFields = ((TopFieldDocs) topDocs).fields; + if (isFirstResult) { + for (int i = 0; i < curSortFields.length; i++) { + firstTypes[i] = getType(firstSortFields[i]); + if (firstTypes[i] == SortField.Type.CUSTOM) { + // for custom types that we can't resolve, we can't do the check + return; + } + } + isFirstResult = false; + } else { + for (int i = 0; i < curSortFields.length; i++) { + SortField.Type curType = getType(curSortFields[i]); + if (curType != firstTypes[i]) { + if (curType == SortField.Type.CUSTOM) { + // for custom types that we can't resolve, we can't do the check + return; + } + throw new IllegalArgumentException( + "Can't sort on field [" + + curSortFields[i].getField() + + "]; the field has incompatible sort types: [" + + firstTypes[i] + + "] and [" + + curType + + "] across shards!" + ); + } + } + } + } + } + + private static SortField.Type getType(SortField sortField) { + if (sortField instanceof SortedNumericSortField) { + return ((SortedNumericSortField) sortField).getNumericType(); + } + if (sortField instanceof SortedSetSortField) { + return SortField.Type.STRING; + } else { + return sortField.getType(); + } + } + static void setShardIndex(TopDocs topDocs, int shardIndex) { assert topDocs.scoreDocs.length == 0 || topDocs.scoreDocs[0].shardIndex == -1 : "shardIndex is already set"; for (ScoreDoc doc : topDocs.scoreDocs) { From 86b35edb54d49b9f389546b5b83c27bb561f484f Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 17:06:26 -0500 Subject: [PATCH 02/40] Script: Configuration driven validation for CtxMap Adds FieldProperties record that configures how to validate fields. Fields have a type, are writeable or read-only, and nullable or not and may have an additional validation useful for Set/Enum validation. Splits IngestMetadata from Metadata in preparation for new Metdata subclasses. --- .../ingest/common/RenameProcessorTests.java | 6 +- .../elasticsearch/ingest/IngestCtxMap.java | 81 +++++- .../elasticsearch/ingest/IngestDocument.java | 7 +- .../java/org/elasticsearch/script/CtxMap.java | 5 + .../org/elasticsearch/script/Metadata.java | 260 ++++++------------ .../ingest/IngestCtxMapTests.java | 39 ++- .../elasticsearch/script/MetadataTests.java | 25 -- .../ingest/TestIngestCtxMetadata.java | 26 ++ .../ingest/TestIngestDocument.java | 17 +- .../elasticsearch/script/TestMetadata.java | 27 -- 10 files changed, 226 insertions(+), 267 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/script/MetadataTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 32566e82baf80..4cab0b999c248 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -140,11 +140,11 @@ public void testRenameAtomicOperationSetFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (o, k, v) -> { + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (k, v) -> { if (v != null) { throw new UnsupportedOperationException(); } - }, "list", (o, k, v) -> {})); + }, "list", (k, v) -> {})); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); @@ -160,7 +160,7 @@ public void testRenameAtomicOperationRemoveFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (o, k, v) -> { + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (k, v) -> { if (v == null) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index 51a6178087fab..dc5bf06697c51 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -8,7 +8,7 @@ package org.elasticsearch.ingest; -import org.elasticsearch.core.Tuple; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.CtxMap; import org.elasticsearch.script.Metadata; @@ -46,7 +46,7 @@ class IngestCtxMap extends CtxMap { ZonedDateTime timestamp, Map source ) { - super(new HashMap<>(source), new Metadata(index, id, version, routing, versionType, timestamp)); + super(new HashMap<>(source), new IngestMetadata(index, id, version, routing, versionType, timestamp)); } /** @@ -59,16 +59,6 @@ class IngestCtxMap extends CtxMap { super(source, metadata); } - /** - * Returns a new metadata map and the existing source map with metadata removed. - */ - public static Tuple, Map> splitSourceAndMetadata(Map sourceAndMetadata) { - return CtxMap.splitSourceAndMetadata( - sourceAndMetadata, - Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) - ); - } - /** * Fetch the timestamp from the ingestMetadata, if it exists * @return the timestamp for the document or null @@ -85,4 +75,71 @@ public static ZonedDateTime getTimestamp(Map ingestMetadata) { } return null; } + + static class IngestMetadata extends Metadata { + private static final FieldProperty UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null); + static final Map> PROPERTIES = Map.of( + INDEX, + UPDATABLE_STRING, + ID, + UPDATABLE_STRING, + ROUTING, + UPDATABLE_STRING, + VERSION_TYPE, + new FieldProperty<>(String.class, true, true, (k, v) -> { + try { + VersionType.fromString(v); + return; + } catch (IllegalArgumentException ignored) {} + throw new IllegalArgumentException( + k + + " must be a null or one of [" + + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) + + "] but was [" + + v + + "] with type [" + + v.getClass().getName() + + "]" + ); + }), + VERSION, + new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), + TYPE, + new FieldProperty<>(String.class, false, false, null), + IF_SEQ_NO, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + IF_PRIMARY_TERM, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + DYNAMIC_TEMPLATES, + new FieldProperty<>(Map.class, true, true, null) + ); + + protected final ZonedDateTime timestamp; + + IngestMetadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { + this(metadataMap(index, id, version, routing, versionType), timestamp); + } + + IngestMetadata(Map metadata, ZonedDateTime timestamp) { + super(metadata, PROPERTIES); + this.timestamp = timestamp; + } + + /** + * Create the backing metadata map with the standard contents assuming default validators. + */ + protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); + metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); + metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); + } + if (versionType != null) { + metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + } + return metadata; + } + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index dcb5b4e090567..81ecfa39bccd0 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -77,10 +77,13 @@ public IngestDocument(IngestDocument other) { * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ public IngestDocument(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata(sourceAndMetadata); + Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( + sourceAndMetadata, + Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) + ); this.sourceAndMetadata = new IngestCtxMap( sm.v1(), - new org.elasticsearch.script.Metadata(sm.v2(), IngestCtxMap.getTimestamp(ingestMetadata)) + new IngestCtxMap.IngestMetadata(sm.v2(), IngestCtxMap.getTimestamp(ingestMetadata)) ); this.ingestMetadata = new HashMap<>(ingestMetadata); this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> { diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java index 80fa299206571..2f82ed05b30a0 100644 --- a/server/src/main/java/org/elasticsearch/script/CtxMap.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -23,6 +23,11 @@ import java.util.Set; import java.util.stream.Collectors; +/** + * A scripting ctx map with metadata for write ingest contexts. Delegates all metadata updates to metadata and + * all other updates to source. Implements the {@link Map} interface for backwards compatibility while performing + * validation via {@link Metadata}. + */ public class CtxMap extends AbstractMap { protected final Map source; protected final Metadata metadata; diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 1cb9ce6e9a30f..27ef71906d7d4 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -8,13 +8,8 @@ package org.elasticsearch.script; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.VersionType; -import org.elasticsearch.ingest.IngestDocument; - import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -22,6 +17,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiConsumer; import java.util.stream.Collectors; /** @@ -52,80 +48,31 @@ public class Metadata { protected static final String IF_PRIMARY_TERM = "_if_primary_term"; protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates"; - protected static final Map VALIDATORS = Map.of( - INDEX, - Metadata::stringValidator, - ID, - Metadata::stringValidator, - ROUTING, - Metadata::stringValidator, - VERSION_TYPE, - Metadata::versionTypeValidator, - VERSION, - Metadata::notNullLongValidator, - TYPE, - Metadata::stringValidator, - IF_SEQ_NO, - Metadata::longValidator, - IF_PRIMARY_TERM, - Metadata::longValidator, - DYNAMIC_TEMPLATES, - Metadata::mapValidator - ); - protected final Map map; - protected final Map validators; - - // timestamp is new to ingest metadata, so it doesn't need to be backed by the map for back compat - protected final ZonedDateTime timestamp; - - public Metadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { - this(metadataMap(index, id, version, routing, versionType), timestamp, VALIDATORS); - } + protected final Map> properties; + protected static final FieldProperty BAD_KEY = new FieldProperty<>(null, false, false, null); - public Metadata(Map map, ZonedDateTime timestamp) { - this(map, timestamp, VALIDATORS); - } - - Metadata(Map map, ZonedDateTime timestamp, Map validators) { + public Metadata(Map map, Map> properties) { this.map = map; - this.timestamp = timestamp; - this.validators = validators; + this.properties = properties; validateMetadata(); } - /** - * Create the backing metadata map with the standard contents assuming default validators. - */ - protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); - metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); - metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); - if (routing != null) { - metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - if (versionType != null) { - metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); - } - return metadata; - } - /** * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata */ protected void validateMetadata() { int numMetadata = 0; - for (Map.Entry entry : validators.entrySet()) { + for (Map.Entry> entry : properties.entrySet()) { String key = entry.getKey(); if (map.containsKey(key)) { numMetadata++; } - entry.getValue().accept(MapOperation.INIT, key, map.get(key)); + entry.getValue().check(MapOperation.INIT, key, map.get(key)); } if (numMetadata < map.size()) { Set keys = new HashSet<>(map.keySet()); - keys.removeAll(validators.keySet()); + keys.removeAll(properties.keySet()); throw new IllegalArgumentException( "Unexpected metadata keys [" + keys.stream().sorted().map(k -> k + ":" + map.get(k)).collect(Collectors.joining(", ")) + "]" ); @@ -174,7 +121,7 @@ public void setVersion(long version) { } public ZonedDateTime getTimestamp() { - return timestamp; + throw new UnsupportedOperationException("unimplemented"); } // These are not available to scripts @@ -220,7 +167,7 @@ public Number getNumber(String key) { * this call. */ public boolean isMetadata(String key) { - return validators.containsKey(key); + return properties.containsKey(key); } /** @@ -228,8 +175,7 @@ public boolean isMetadata(String key) { * @throws IllegalArgumentException if {@link #isMetadata(String)} is false or the key cannot be updated to the value. */ public Object put(String key, Object value) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.UPDATE, key, value); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.UPDATE, key, value); return map.put(key, value); } @@ -259,8 +205,7 @@ public Object get(String key) { * @throws IllegalArgumentException if {@link #isMetadata(String)} is false or the key cannot be removed. */ public Object remove(String key) { - Validator v = validators.getOrDefault(key, this::badKey); - v.accept(MapOperation.REMOVE, key, null); + properties.getOrDefault(key, Metadata.BAD_KEY).check(MapOperation.REMOVE, key, null); return map.remove(key); } @@ -280,7 +225,7 @@ public int size() { @Override public Metadata clone() { - return new Metadata(new HashMap<>(map), timestamp, new HashMap<>(validators)); + return new Metadata(new HashMap<>(map), new HashMap<>(properties)); } /** @@ -295,7 +240,7 @@ public Map getMap() { */ public Set metadataKeys(Set other) { Set keys = null; - for (String key : validators.keySet()) { + for (String key : properties.keySet()) { if (other.contains(key)) { if (keys == null) { keys = new HashSet<>(); @@ -306,99 +251,6 @@ public Set metadataKeys(Set other) { return keys != null ? keys : Collections.emptySet(); } - /** - * Allow a String or null. - * @throws IllegalArgumentException if {@param value} is neither a {@link String} nor null - */ - protected static void stringValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof String) { - return; - } - throw new IllegalArgumentException( - key + " must be null or a String but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow Numbers that can be represented as longs without loss of precision or null - * @throws IllegalArgumentException if the value cannot be represented as a long - */ - public static void longValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof Number number) { - long version = number.longValue(); - // did we round? - if (number.doubleValue() == version) { - return; - } - } - throw new IllegalArgumentException( - key + " may only be set to an int or a long but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Same as {@link #longValidator(MapOperation, String, Object)} but {@param value} cannot be null. - * @throws IllegalArgumentException if value is null or cannot be represented as a long. - */ - protected static void notNullLongValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - throw new IllegalArgumentException(key + " cannot be removed or set to null"); - } - longValidator(op, key, value); - } - - /** - * Allow maps. - * @throws IllegalArgumentException if {@param value} is not a {@link Map} - */ - public static void mapValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null || value instanceof Map) { - return; - } - throw new IllegalArgumentException( - key + " must be a null or a Map but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow lower case Strings that map to VersionType values, or null. - * @throws IllegalArgumentException if {@param value} cannot be converted via {@link VersionType#fromString(String)} - */ - protected static void versionTypeValidator(MapOperation op, String key, Object value) { - if (op == MapOperation.REMOVE || value == null) { - return; - } - if (value instanceof String versionType) { - try { - VersionType.fromString(versionType); - return; - } catch (IllegalArgumentException ignored) {} - } - throw new IllegalArgumentException( - key - + " must be a null or one of [" - + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) - + "] but was [" - + value - + "] with type [" - + value.getClass().getName() - + "]" - ); - } - - private void badKey(MapOperation op, String key, Object value) { - throw new IllegalArgumentException( - "unexpected metadata key [" - + key - + "], expected one of [" - + validators.keySet().stream().sorted().collect(Collectors.joining(", ")) - + "]" - ); - } - /** * The operation being performed on the value in the map. * INIT: Initial value - the metadata value as passed into this class @@ -411,26 +263,72 @@ public enum MapOperation { REMOVE } - /** - * A "TriConsumer" that tests if the {@link MapOperation}, the metadata key and value are valid. - * - * throws IllegalArgumentException if the given triple is invalid - */ - @FunctionalInterface - public interface Validator { - void accept(MapOperation op, String key, Object value); - } + public record FieldProperty (Class type, boolean nullable, boolean writable, BiConsumer extendedValidation) { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Metadata metadata = (Metadata) o; - return Objects.equals(map, metadata.map) && Objects.equals(timestamp, metadata.timestamp); - } + public static BiConsumer LONGABLE_NUMBER = (k, v) -> { + long version = v.longValue(); + // did we round? + if (v.doubleValue() == version) { + return; + } + throw new IllegalArgumentException( + k + " may only be set to an int or a long but was [" + v + "] with type [" + v.getClass().getName() + "]" + ); + }; + + public static FieldProperty ALLOW_ALL = new FieldProperty<>(null, true, true, null); + + @SuppressWarnings("fallthrough") + public void check(MapOperation op, String key, Object value) { + switch (op) { + case UPDATE: + if (writable == false) { + throw new IllegalArgumentException(key + " cannot be updated"); + } + // fall through + + case INIT: + if (value == null) { + if (nullable == false) { + throw new IllegalArgumentException(key + " cannot be null"); + } + } else { + checkType(key, value); + } + break; + + case REMOVE: + if (writable == false || nullable == false) { + throw new IllegalArgumentException(key + " cannot be removed"); + } + break; + + default: + throw new IllegalArgumentException("unexpected op [" + op + "] for key [" + key + "] and value [" + value + "]"); - @Override - public int hashCode() { - return Objects.hash(map, timestamp); + } + } + + @SuppressWarnings("unchecked") + private void checkType(String key, Object value) { + if (type == null) { + return; + } + if (type.isAssignableFrom(value.getClass()) == false) { + throw new IllegalArgumentException( + key + + " [" + + value + + "] is wrong type, expected assignable to [" + + type.getName() + + "], not [" + + value.getClass().getName() + + "]" + ); + } + if (extendedValidation != null) { + extendedValidation.accept(key, (T) value); + } + } } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java index 098794dde902b..e5575910ad881 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -21,6 +20,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.notNullValue; public class IngestCtxMapTests extends ESTestCase { @@ -76,7 +77,7 @@ public String toString() { }); metadata.put("c", null); metadata.put("d", 1234); - map = new IngestCtxMap(new HashMap<>(), new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); + map = new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); md = map.getMetadata(); assertNull(md.getString("c")); assertNull(md.getString("no key")); @@ -91,7 +92,7 @@ public void testGetNumber() { metadata.put("b", Double.MAX_VALUE); metadata.put("c", "NaN"); metadata.put("d", null); - map = new IngestCtxMap(new HashMap<>(), new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); + map = new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, allowAllValidators("a", "b", "c", "d"))); md = map.getMetadata(); assertEquals(Long.MAX_VALUE, md.getNumber("a")); assertEquals(Double.MAX_VALUE, md.getNumber("b")); @@ -146,11 +147,18 @@ public void testRemove() { String canRemove = "canRemove"; Map metadata = new HashMap<>(); metadata.put(cannotRemove, "value"); - map = new IngestCtxMap(new HashMap<>(), new TestMetadata(metadata, Map.of(cannotRemove, (o, k, v) -> { - if (v == null) { - throw new IllegalArgumentException(k + " cannot be null or removed"); - } - }, canRemove, (o, k, v) -> {}))); + map = new IngestCtxMap( + new HashMap<>(), + new TestIngestCtxMetadata( + metadata, + Map.of( + cannotRemove, + new Metadata.FieldProperty<>(String.class, false, true, null), + canRemove, + new Metadata.FieldProperty<>(String.class, true, true, null) + ) + ) + ); String msg = "cannotRemove cannot be null or removed"; IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.remove(cannotRemove)); assertEquals(msg, err.getMessage()); @@ -211,7 +219,7 @@ public void testEntryAndIterator() { source.put("foo", "bar"); source.put("baz", "qux"); source.put("noz", "zon"); - map = new IngestCtxMap(source, TestMetadata.withNullableVersion(metadata)); + map = new IngestCtxMap(source, TestIngestCtxMetadata.withNullableVersion(metadata)); md = map.getMetadata(); for (Map.Entry entry : map.entrySet()) { @@ -364,11 +372,18 @@ public Object setValue(Object value) { } } - private static Map allowAllValidators(String... keys) { - Map validators = new HashMap<>(); + private static Map> allowAllValidators(String... keys) { + Map> validators = new HashMap<>(); for (String key : keys) { - validators.put(key, (o, k, v) -> {}); + validators.put(key, Metadata.FieldProperty.ALLOW_ALL); } return validators; } + + public void testDefaultFieldPropertiesForAllMetadata() { + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + assertThat(IngestCtxMap.IngestMetadata.PROPERTIES, hasEntry(equalTo(m.getFieldName()), notNullValue())); + } + assertEquals(IngestDocument.Metadata.values().length, IngestCtxMap.IngestMetadata.PROPERTIES.size()); + } } diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java deleted file mode 100644 index c241d14d93b1b..0000000000000 --- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.test.ESTestCase; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; - -public class MetadataTests extends ESTestCase { - public void testDefaultValidatorForAllMetadata() { - for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { - assertThat(Metadata.VALIDATORS, hasEntry(equalTo(m.getFieldName()), notNullValue())); - } - assertEquals(IngestDocument.Metadata.values().length, Metadata.VALIDATORS.size()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java new file mode 100644 index 0000000000000..2040fa5ee9322 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.script.Metadata; + +import java.util.HashMap; +import java.util.Map; + +public class TestIngestCtxMetadata extends Metadata { + public TestIngestCtxMetadata(Map map, Map> properties) { + super(map, properties); + } + + public static TestIngestCtxMetadata withNullableVersion(Map map) { + Map> updatedProperties = new HashMap<>(IngestCtxMap.IngestMetadata.PROPERTIES); + updatedProperties.replace(VERSION, new Metadata.FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER)); + return new TestIngestCtxMetadata(map, updatedProperties); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index 58afd55d38480..a9135b80ae790 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -12,11 +12,13 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; -import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; /** * Construct ingest documents for testing purposes @@ -37,8 +39,11 @@ public static IngestDocument withNullableVersion(Map sourceAndMe * _versions. Normally null _version is not allowed, but many tests don't care about that invariant. */ public static IngestDocument ofIngestWithNullableVersion(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata(sourceAndMetadata); - return new IngestDocument(new IngestCtxMap(sm.v1(), TestMetadata.withNullableVersion(sm.v2())), ingestMetadata); + Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( + sourceAndMetadata, + Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) + ); + return new IngestDocument(new IngestCtxMap(sm.v1(), TestIngestCtxMetadata.withNullableVersion(sm.v2())), ingestMetadata); } /** @@ -56,8 +61,10 @@ public static IngestDocument withDefaultVersion(Map sourceAndMet * Create an IngestDocument with a metadata map and validators. The metadata map is passed by reference, not copied, so callers * can observe changes to the map directly. */ - public static IngestDocument ofMetadataWithValidator(Map metadata, Map validators) { - return new IngestDocument(new IngestCtxMap(new HashMap<>(), new TestMetadata(metadata, validators)), new HashMap<>()); + public static IngestDocument ofMetadataWithValidator(Map metadata, Map> validators) { + Map> properties = new HashMap<>(); + validators.forEach((k, v) -> properties.put(k, new Metadata.FieldProperty<>(Object.class, true, true, v))); + return new IngestDocument(new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, properties)), new HashMap<>()); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java deleted file mode 100644 index 87a45ea857549..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import java.util.HashMap; -import java.util.Map; - -/** - * An implementation of {@link Metadata} with customizable {@link org.elasticsearch.script.Metadata.Validator}s for use in testing. - */ -public class TestMetadata extends Metadata { - public TestMetadata(Map map, Map validators) { - super(map, null, validators); - } - - public static TestMetadata withNullableVersion(Map map) { - Map updatedValidators = new HashMap<>(VALIDATORS); - updatedValidators.replace(VERSION, Metadata::longValidator); - return new TestMetadata(map, updatedValidators); - } -} From 472b824fe945b1c328b98afa3e8e8d040c6c11e8 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 17:27:22 -0500 Subject: [PATCH 03/40] Fix hashCode in CtxMap --- server/src/main/java/org/elasticsearch/script/CtxMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java index 2f82ed05b30a0..d7916e91fc450 100644 --- a/server/src/main/java/org/elasticsearch/script/CtxMap.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -294,6 +294,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(super.hashCode(), source, metadata); + return Objects.hash(source, metadata); } } From b55a0bc0aadadb655015f171b853d6d62a584d7e Mon Sep 17 00:00:00 2001 From: Nikola Grcevski <6207777+grcevski@users.noreply.github.com> Date: Mon, 11 Jul 2022 21:58:53 -0400 Subject: [PATCH 04/40] Fix test memory leak (#88362) --- .../elasticsearch/plugins/PluginsService.java | 13 ++++++++++--- .../plugins/PluginsServiceTests.java | 4 ++++ .../ilm/ILMImmutableStateHandlerProvider.java | 19 ++++++++++--------- .../xpack/ilm/IndexLifecycle.java | 10 +++++++--- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 058885de79f81..7e2e13d5343f5 100644 --- a/server/src/main/java/org/elasticsearch/plugins/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -309,7 +309,7 @@ public List loadServiceProviders(Class service) { List result = new ArrayList<>(); for (LoadedPlugin pluginTuple : plugins()) { - ServiceLoader.load(service, pluginTuple.loader()).iterator().forEachRemaining(c -> result.add(c)); + result.addAll(createExtensions(service, pluginTuple.instance)); } return Collections.unmodifiableList(result); @@ -348,11 +348,18 @@ static T createExtension(Class extensionClass, Class extensi throw new IllegalStateException("no public " + extensionConstructorMessage(extensionClass, extensionPointType)); } - if (constructors.length > 1) { + Constructor constructor = constructors[0]; + // Using modules and SPI requires that we declare the default no-arg constructor apart from our custom + // one arg constructor with a plugin. + if (constructors.length == 2) { + // we prefer the one arg constructor in this case + if (constructors[1].getParameterCount() > 0) { + constructor = constructors[1]; + } + } else if (constructors.length > 1) { throw new IllegalStateException("no unique public " + extensionConstructorMessage(extensionClass, extensionPointType)); } - final Constructor constructor = constructors[0]; if (constructor.getParameterCount() > 1) { throw new IllegalStateException(extensionSignatureMessage(extensionClass, extensionPointType, plugin)); } diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index ad334a2d42f8a..ac0e967face4f 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -531,6 +531,10 @@ public TestExtension() {} public TestExtension(TestPlugin plugin) { } + + public TestExtension(TestPlugin plugin, String anotherArg) { + + } } IllegalStateException e = expectThrows( IllegalStateException.class, diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java index afd2397b8fb6f..fb69802fbf09f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/ILMImmutableStateHandlerProvider.java @@ -10,23 +10,24 @@ import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; import org.elasticsearch.immutablestate.ImmutableClusterStateHandlerProvider; -import java.util.Arrays; import java.util.Collection; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; /** * ILM Provider implementation for the {@link ImmutableClusterStateHandlerProvider} service interface */ public class ILMImmutableStateHandlerProvider implements ImmutableClusterStateHandlerProvider { - private static final Set> handlers = ConcurrentHashMap.newKeySet(); + private final IndexLifecycle plugin; - @Override - public Collection> handlers() { - return handlers; + public ILMImmutableStateHandlerProvider() { + throw new IllegalStateException("Provider must be constructed using PluginsService"); + } + + public ILMImmutableStateHandlerProvider(IndexLifecycle plugin) { + this.plugin = plugin; } - public static void registerHandlers(ImmutableClusterStateHandler... stateHandlers) { - handlers.addAll(Arrays.asList(stateHandlers)); + @Override + public Collection> handlers() { + return plugin.immutableClusterStateHandlers(); } } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java index b7f824b4c2845..e477442bcc2e4 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java @@ -27,6 +27,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.health.HealthIndicatorService; +import org.elasticsearch.immutablestate.ImmutableClusterStateHandler; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.XPackLicenseState; @@ -159,6 +160,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin, HealthPlugin private final SetOnce snapshotHistoryStore = new SetOnce<>(); private final SetOnce ilmHealthIndicatorService = new SetOnce<>(); private final SetOnce slmHealthIndicatorService = new SetOnce<>(); + private final SetOnce immutableLifecycleAction = new SetOnce<>(); private final Settings settings; public IndexLifecycle(Settings settings) { @@ -268,10 +270,8 @@ public Collection createComponents( components.addAll(Arrays.asList(snapshotLifecycleService.get(), snapshotHistoryStore.get(), snapshotRetentionService.get())); ilmHealthIndicatorService.set(new IlmHealthIndicatorService(clusterService)); slmHealthIndicatorService.set(new SlmHealthIndicatorService(clusterService)); + immutableLifecycleAction.set(new ImmutableLifecycleAction(xContentRegistry, client, XPackPlugin.getSharedLicenseState())); - ILMImmutableStateHandlerProvider.registerHandlers( - new ImmutableLifecycleAction(xContentRegistry, client, XPackPlugin.getSharedLicenseState()) - ); return components; } @@ -422,6 +422,10 @@ public List getRestHandlers( return actions; } + List> immutableClusterStateHandlers() { + return List.of(immutableLifecycleAction.get()); + } + @Override public Collection getHealthIndicatorServices() { return List.of(ilmHealthIndicatorService.get(), slmHealthIndicatorService.get()); From f9ac7ae508b1645e0383015fdf67d3ac9cc6d86c Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 21:15:09 -0500 Subject: [PATCH 05/40] Fix unit tests --- .../elasticsearch/ingest/IngestCtxMap.java | 2 +- .../java/org/elasticsearch/script/CtxMap.java | 4 +- .../org/elasticsearch/script/Metadata.java | 2 - .../ingest/IngestCtxMapTests.java | 44 ++++++++++++------- .../elasticsearch/script/MetadataTests.java | 18 ++------ 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index dc5bf06697c51..9e98d6386d2be 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -105,7 +105,7 @@ static class IngestMetadata extends Metadata { VERSION, new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), TYPE, - new FieldProperty<>(String.class, false, false, null), + new FieldProperty<>(String.class, true, false, null), IF_SEQ_NO, new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), IF_PRIMARY_TERM, diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java index 683a2b816069a..7301052a27a7e 100644 --- a/server/src/main/java/org/elasticsearch/script/CtxMap.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -17,6 +17,7 @@ import java.util.AbstractSet; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Objects; @@ -92,7 +93,8 @@ public Metadata getMetadata() { */ @Override public Set> entrySet() { - return new EntrySet(source.entrySet(), metadata.keySet()); + // Make a copy of the Metadata.keySet() to avoid a ConcurrentModificationException when removing a value from the iterator + return new EntrySet(source.entrySet(), new HashSet<>(metadata.keySet())); } /** diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 471acc901bc4d..09b655e64b26f 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -9,8 +9,6 @@ package org.elasticsearch.script; import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java index eba6fb1a20148..f7a3761ebf0f2 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java @@ -20,6 +20,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.notNullValue; public class IngestCtxMapTests extends ESTestCase { @@ -36,7 +38,7 @@ public void testSettersAndGetters() { metadata.put("_if_primary_term", 10000); metadata.put("_version_type", "internal"); metadata.put("_dynamic_templates", Map.of("foo", "bar")); - map = new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)); md = map.getMetadata(); assertEquals("myIndex", md.getIndex()); md.setIndex("myIndex2"); @@ -69,7 +71,7 @@ public void testInvalidMetadata() { metadata.put("_version", Double.MAX_VALUE); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)) ); assertThat(err.getMessage(), containsString("_version may only be set to an int or a long but was [")); assertThat(err.getMessage(), containsString("] with type [java.lang.Double]")); @@ -80,7 +82,7 @@ public void testSourceInMetadata() { source.put("_version", 25); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(source, new Metadata(source, null)) + () -> new IngestCtxMap(source, new IngestCtxMap.IngestMetadata(source, null)) ); assertEquals("unexpected metadata [_version:25] in source", err.getMessage()); } @@ -92,7 +94,7 @@ public void testExtraMetadata() { metadata.put("routing", "myRouting"); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(new HashMap<>(), new Metadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)) ); assertEquals("Unexpected metadata keys [routing:myRouting, version:567]", err.getMessage()); } @@ -101,7 +103,7 @@ public void testPutSource() { Map metadata = new HashMap<>(); metadata.put("_version", 123); Map source = new HashMap<>(); - map = new IngestCtxMap(source, new Metadata(metadata, null)); + map = new IngestCtxMap(source, new IngestCtxMap.IngestMetadata(metadata, null)); } public void testRemove() { @@ -121,15 +123,15 @@ public void testRemove() { ) ) ); - String msg = "cannotRemove cannot be null or removed"; + String msg = "cannotRemove cannot be removed"; IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.remove(cannotRemove)); assertEquals(msg, err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> map.put(cannotRemove, null)); - assertEquals(msg, err.getMessage()); + assertEquals("cannotRemove cannot be null", err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> map.entrySet().iterator().next().setValue(null)); - assertEquals(msg, err.getMessage()); + assertEquals("cannotRemove cannot be null", err.getMessage()); err = expectThrows(IllegalArgumentException.class, () -> { Iterator> it = map.entrySet().iterator(); @@ -221,7 +223,7 @@ public void testEntryAndIterator() { } public void testContainsValue() { - map = new IngestCtxMap(Map.of("myField", "fieldValue"), new Metadata(Map.of("_version", 5678), null)); + map = new IngestCtxMap(Map.of("myField", "fieldValue"), new IngestCtxMap.IngestMetadata(Map.of("_version", 5678), null)); assertTrue(map.containsValue(5678)); assertFalse(map.containsValue(5679)); assertTrue(map.containsValue("fieldValue")); @@ -232,24 +234,24 @@ public void testValidators() { map = new IngestCtxMap("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); md = map.getMetadata(); IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.put("_index", 555)); - assertEquals("_index must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_index [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myIndex", md.getIndex()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_id", 555)); - assertEquals("_id must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_id [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myId", md.getId()); map.put("_id", "myId2"); assertEquals("myId2", md.getId()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_routing", 555)); - assertEquals("_routing must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); + assertEquals("_routing [555] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); assertEquals("myRouting", md.getRouting()); map.put("_routing", "myRouting2"); assertEquals("myRouting2", md.getRouting()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version", "five-five-five")); assertEquals( - "_version may only be set to an int or a long but was [five-five-five] with type [java.lang.String]", + "_version [five-five-five] is wrong type, expected assignable to [java.lang.Number], not [java.lang.String]", err.getMessage() ); assertEquals(1234, md.getVersion()); @@ -271,7 +273,7 @@ public void testValidators() { ); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version_type", VersionType.EXTERNAL)); assertEquals( - "_version_type must be a null or one of [internal, external, external_gte] but was [EXTERNAL] with type" + "_version_type [EXTERNAL] is wrong type, expected assignable to [java.lang.String], not" + " [org.elasticsearch.index.VersionType$2]", err.getMessage() ); @@ -283,7 +285,10 @@ public void testValidators() { ); err = expectThrows(IllegalArgumentException.class, () -> map.put("_dynamic_templates", "5")); - assertEquals("_dynamic_templates must be a null or a Map but was [5] with type [java.lang.String]", err.getMessage()); + assertEquals( + "_dynamic_templates [5] is wrong type, expected assignable to [java.util.Map], not [java.lang.String]", + err.getMessage() + ); Map dt = Map.of("a", "b"); map.put("_dynamic_templates", dt); assertThat(dt, equalTo(md.getDynamicTemplates())); @@ -292,7 +297,7 @@ public void testValidators() { public void testHandlesAllVersionTypes() { Map mdRawMap = new HashMap<>(); mdRawMap.put("_version", 1234); - map = new IngestCtxMap(new HashMap<>(), new Metadata(mdRawMap, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(mdRawMap, null)); md = map.getMetadata(); assertNull(md.getVersionType()); for (VersionType vt : VersionType.values()) { @@ -309,6 +314,13 @@ public void testHandlesAllVersionTypes() { assertNull(md.getVersionType()); } + public void testDefaultFieldPropertiesForAllMetadata() { + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + assertThat(IngestCtxMap.IngestMetadata.PROPERTIES, hasEntry(equalTo(m.getFieldName()), notNullValue())); + } + assertEquals(IngestDocument.Metadata.values().length, IngestCtxMap.IngestMetadata.PROPERTIES.size()); + } + private static class TestEntry implements Map.Entry { String key; Object value; diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java index c3e9c599076b2..1b3d5574b496b 100644 --- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/MetadataTests.java @@ -8,26 +8,14 @@ package org.elasticsearch.script; -import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; - public class MetadataTests extends ESTestCase { Metadata md; - public void testDefaultFieldPropertiesForAllMetadata() { - for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { - assertThat(IngestCtxMap.IngestMetadata.PROPERTIES, hasEntry(equalTo(m.getFieldName()), notNullValue())); - } - assertEquals(IngestDocument.Metadata.values().length, IngestCtxMap.IngestMetadata.PROPERTIES.size()); - } - public void testGetString() { Map metadata = new HashMap<>(); metadata.put("a", "A"); @@ -39,7 +27,7 @@ public String toString() { }); metadata.put("c", null); metadata.put("d", 1234); - md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + md = new Metadata(metadata, allowAllValidators("a", "b", "c", "d")); assertNull(md.getString("c")); assertNull(md.getString("no key")); assertEquals("myToString()", md.getString("b")); @@ -53,10 +41,10 @@ public void testGetNumber() { metadata.put("b", Double.MAX_VALUE); metadata.put("c", "NaN"); metadata.put("d", null); - md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + md = new Metadata(metadata, allowAllValidators("a", "b", "c", "d")); assertEquals(Long.MAX_VALUE, md.getNumber("a")); assertEquals(Double.MAX_VALUE, md.getNumber("b")); - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> md.getNumber("c")); + IllegalStateException err = expectThrows(IllegalStateException.class, () -> md.getNumber("c")); assertEquals("unexpected type for [c] with value [NaN], expected Number, got [java.lang.String]", err.getMessage()); assertNull(md.getNumber("d")); assertNull(md.getNumber("no key")); From 15328b18780faa248b472109ade03d9b391c2e66 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 21:20:07 -0500 Subject: [PATCH 06/40] FieldProperty javadoc --- .../src/main/java/org/elasticsearch/script/Metadata.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 09b655e64b26f..e2e7551e4cb9f 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -245,6 +245,13 @@ public enum MapOperation { REMOVE } + /** + * The properties of a metadata field. + * @param type - the class of the field. Updates must be assignable to this type. If null, no type checking is performed. + * @param nullable - can the field value be null and can it be removed + * @param writable - can the field be updated after the initial set + * @param extendedValidation - value validation after type checking, may be used for values that may be one of a set + */ public record FieldProperty (Class type, boolean nullable, boolean writable, BiConsumer extendedValidation) { public static BiConsumer LONGABLE_NUMBER = (k, v) -> { From 824cdc85cedfe2e66d42551550e051ab5c2212b9 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 21:38:38 -0500 Subject: [PATCH 07/40] Implement getTimestamp() in IngestMetadata --- .../src/main/java/org/elasticsearch/ingest/IngestCtxMap.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index 9e98d6386d2be..d1fddd11c3a9c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -141,5 +141,10 @@ protected static Map metadataMap(String index, String id, long v } return metadata; } + + @Override + public ZonedDateTime getTimestamp() { + return timestamp; + } } } From c39bb9d1d0cf50684c115abdf0ed7d76b1c613ce Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 22:15:57 -0500 Subject: [PATCH 08/40] Use extended validators, add hashcode and equals for Metadata --- .../ingest/common/RenameProcessorTests.java | 25 +++++++++++-------- .../org/elasticsearch/script/Metadata.java | 13 ++++++++++ .../ingest/TestIngestDocument.java | 5 +--- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 4cab0b999c248..5908fc8784d8f 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.TestIngestDocument; import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.script.Metadata; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -140,11 +141,14 @@ public void testRenameAtomicOperationSetFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (k, v) -> { - if (v != null) { - throw new UnsupportedOperationException(); - } - }, "list", (k, v) -> {})); + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator( + metadata, + Map.of("new_field", new Metadata.FieldProperty<>(Object.class, true, true, (k, v) -> { + if (v != null) { + throw new UnsupportedOperationException(); + } + }), "list", new Metadata.FieldProperty<>(Object.class, true, true, null)) + ); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); @@ -160,16 +164,15 @@ public void testRenameAtomicOperationRemoveFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (k, v) -> { - if (v == null) { - throw new UnsupportedOperationException(); - } - })); + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator( + metadata, + Map.of("list", new Metadata.FieldProperty<>(Object.class, false, true, null)) + ); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); - } catch (UnsupportedOperationException e) { + } catch (IllegalArgumentException e) { // the set failed, the old field has not been removed assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("new_field"), equalTo(false)); diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index e2e7551e4cb9f..c435dff5ca7fa 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -233,6 +233,19 @@ public Map getMap() { return map; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ((o instanceof Metadata) == false) return false; + Metadata metadata = (Metadata) o; + return map.equals(metadata.map); + } + + @Override + public int hashCode() { + return Objects.hash(map); + } + /** * The operation being performed on the value in the map. * INIT: Initial value - the metadata value as passed into this class diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index a9135b80ae790..9dbb74ea80608 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -17,7 +17,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.function.BiConsumer; import java.util.stream.Collectors; /** @@ -61,9 +60,7 @@ public static IngestDocument withDefaultVersion(Map sourceAndMet * Create an IngestDocument with a metadata map and validators. The metadata map is passed by reference, not copied, so callers * can observe changes to the map directly. */ - public static IngestDocument ofMetadataWithValidator(Map metadata, Map> validators) { - Map> properties = new HashMap<>(); - validators.forEach((k, v) -> properties.put(k, new Metadata.FieldProperty<>(Object.class, true, true, v))); + public static IngestDocument ofMetadataWithValidator(Map metadata, Map> properties) { return new IngestDocument(new IngestCtxMap(new HashMap<>(), new TestIngestCtxMetadata(metadata, properties)), new HashMap<>()); } From e81f5fa0b32428681a2613bf35a9807c28a6027e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 23:16:12 -0500 Subject: [PATCH 09/40] _type can be updated --- server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index d1fddd11c3a9c..6bb6003a96c5a 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -105,7 +105,7 @@ static class IngestMetadata extends Metadata { VERSION, new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), TYPE, - new FieldProperty<>(String.class, true, false, null), + new FieldProperty<>(String.class, true, true, null), IF_SEQ_NO, new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), IF_PRIMARY_TERM, From 7276bb0c3efada0cbd6e1c889aee58877b24d99a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Mon, 11 Jul 2022 23:20:17 -0500 Subject: [PATCH 10/40] Fix testExecutePropagateAllMetadataUpdates rather than allow _type to be updateable --- server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java | 2 +- .../test/java/org/elasticsearch/ingest/IngestServiceTests.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index 6bb6003a96c5a..d1fddd11c3a9c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -105,7 +105,7 @@ static class IngestMetadata extends Metadata { VERSION, new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), TYPE, - new FieldProperty<>(String.class, true, true, null), + new FieldProperty<>(String.class, true, false, null), IF_SEQ_NO, new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), IF_PRIMARY_TERM, diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 6182e4709a6b8..7c64cf31f52aa 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -1153,6 +1153,8 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception { ingestDocument.setFieldValue(metadata.getFieldName(), ifPrimaryTerm); } else if (metadata == IngestDocument.Metadata.DYNAMIC_TEMPLATES) { ingestDocument.setFieldValue(metadata.getFieldName(), Map.of("foo", "bar")); + } else if (metadata == IngestDocument.Metadata.TYPE) { + // can't update _type } else { ingestDocument.setFieldValue(metadata.getFieldName(), "update" + metadata.getFieldName()); } From 78244b74d4a5eeffc16edb92309578560280ec4d Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 12 Jul 2022 06:45:50 +0200 Subject: [PATCH 11/40] Remove usages of TestGeoShapeFieldMapperPlugin from enrich module (#88440) --- x-pack/plugin/enrich/build.gradle | 1 + .../java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java | 4 ++-- .../elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/enrich/build.gradle b/x-pack/plugin/enrich/build.gradle index 0fb22fe48ced8..ea59ebe2a2a92 100644 --- a/x-pack/plugin/enrich/build.gradle +++ b/x-pack/plugin/enrich/build.gradle @@ -14,6 +14,7 @@ dependencies { testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: ':modules:ingest-common') testImplementation project(path: ':modules:lang-mustache') + testImplementation project(path: ':modules:legacy-geo') testImplementation project(xpackModule('spatial')) testImplementation(testArtifact(project(xpackModule('monitoring')))) } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java index 48d97fe6d7b35..43e982cec08f1 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -33,6 +32,7 @@ import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyAction; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; import org.elasticsearch.xpack.core.enrich.action.PutEnrichPolicyAction; +import org.elasticsearch.xpack.spatial.SpatialPlugin; import java.util.Arrays; import java.util.Collection; @@ -61,7 +61,7 @@ protected Collection> getPlugins() { ReindexPlugin.class, IngestCommonPlugin.class, MustachePlugin.class, - TestGeoShapeFieldMapperPlugin.class + SpatialPlugin.class ); } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java index 051ce60e87639..000864758d984 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunnerTests.java @@ -52,7 +52,6 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; @@ -63,6 +62,7 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.core.enrich.action.ExecuteEnrichPolicyStatus; import org.elasticsearch.xpack.enrich.action.EnrichReindexAction; +import org.elasticsearch.xpack.spatial.SpatialPlugin; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -87,7 +87,7 @@ public class EnrichPolicyRunnerTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(ReindexPlugin.class, IngestCommonPlugin.class, TestGeoShapeFieldMapperPlugin.class, LocalStateEnrich.class); + return Arrays.asList(ReindexPlugin.class, IngestCommonPlugin.class, SpatialPlugin.class, LocalStateEnrich.class); } private static ThreadPool testThreadPool; From dd1bd8323495814cae713ae67f28b96c63495bd7 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 12 Jul 2022 06:52:14 +0200 Subject: [PATCH 12/40] Don't index geo_shape field in AbstractBuilderTestCase (#88437) This commit stops adding the geo_shape field mapper by default and adds the mapper only when it is needed. --- .../extras/RankFeatureQueryBuilderTests.java | 3 +- .../join/aggregations/ChildrenTests.java | 3 +- .../join/aggregations/ParentTests.java | 3 +- .../join/query/HasChildQueryBuilderTests.java | 3 +- .../query/HasParentQueryBuilderTests.java | 3 +- .../join/query/ParentIdQueryBuilderTests.java | 3 +- .../PercolateQueryBuilderTests.java | 3 +- .../GeoBoundingBoxQueryBuilderTests.java | 29 +++++++++++++++++- .../query/GeoDistanceQueryBuilderTests.java | 30 ++++++++++++++++++- .../GeoShapeQueryBuilderGeoShapeTests.java | 24 +++++++++++++++ .../index/query/TermsQueryBuilderTests.java | 1 - .../query/TermsSetQueryBuilderTests.java | 12 ++------ .../FunctionScoreQueryBuilderTests.java | 3 +- .../vectors/KnnVectorQueryBuilderTests.java | 9 ------ .../DelayedShardAggregationBuilderTests.java | 3 +- .../errorquery/ErrorQueryBuilderTests.java | 3 +- .../test/AbstractBuilderTestCase.java | 9 ++---- .../PinnedQueryBuilderTests.java | 2 -- .../index/query/GeoGridQueryBuilderTests.java | 23 ++++++++++++-- 19 files changed, 117 insertions(+), 52 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java index 4bd2d0714e93d..0e496990efc40 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.io.IOException; import java.util.ArrayList; @@ -54,7 +53,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws @Override protected Collection> getPlugins() { - return Arrays.asList(MapperExtrasPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(MapperExtrasPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java index 96660aea34d23..9d12d1bc3d72f 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -20,7 +19,7 @@ public class ChildrenTests extends BaseAggregationTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java index c424dab398803..75fb02eab2dbb 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -20,7 +19,7 @@ public class ParentTests extends BaseAggregationTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index a32164322ce63..6f857674b6fe5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -77,7 +76,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java index 88176cd0f39fc..5801919aa4b24 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasParentQueryBuilderTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -60,7 +59,7 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java index 0340ac991db21..19a268bc73391 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentIdQueryBuilderTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.join.ParentJoinPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; @@ -48,7 +47,7 @@ public class ParentIdQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ParentJoinPlugin.class); } @Override diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java index 9003e749f0d9c..b0df61bad4c53 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -77,7 +76,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(PercolatorPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(PercolatorPlugin.class); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java index 233cd44765214..c0875b88207da 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java @@ -15,6 +15,9 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.geo.GeometryTestUtils; @@ -23,9 +26,15 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; @@ -35,10 +44,28 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase { /** Randomly generate either NaN or one of the two infinity values. */ private static final Double[] brokenDoubles = { Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY }; + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } @Override protected GeoBoundingBoxQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoBoundingBoxQueryBuilder builder = new GeoBoundingBoxQueryBuilder(fieldName); // make sure that minX != maxX and minY != maxY after geohash encoding Rectangle box = randomValueOtherThanMany( diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java index f3c9090012d44..a564b04122537 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java @@ -14,7 +14,10 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.GeoDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; @@ -22,9 +25,15 @@ import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; @@ -33,9 +42,28 @@ @SuppressWarnings("checkstyle:MissingJavadocMethod") public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected GeoDistanceQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoDistanceQueryBuilder qb = new GeoDistanceQueryBuilder(fieldName); String distance = "" + randomDouble(); if (randomBoolean()) { diff --git a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java index 430906c6602e1..f359f4600faaf 100644 --- a/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderGeoShapeTests.java @@ -8,13 +8,37 @@ package org.elasticsearch.index.query; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoShapeQueryBuilderGeoShapeTests extends GeoShapeQueryBuilderTests { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + XContentBuilder builder = PutMappingRequest.simpleMapping(GEO_SHAPE_FIELD_NAME, "type=geo_shape"); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + + @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled + protected Collection> getPlugins() { + return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + } + protected String fieldName() { return GEO_SHAPE_FIELD_NAME; } diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java index 5b553a46aed6c..9cc03612ccbe4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java @@ -68,7 +68,6 @@ protected TermsQueryBuilder doCreateTestQueryBuilder() { String fieldName = randomValueOtherThanMany( choice -> choice.equals(GEO_POINT_FIELD_NAME) || choice.equals(GEO_POINT_ALIAS_FIELD_NAME) - || choice.equals(GEO_SHAPE_FIELD_NAME) || choice.equals(INT_RANGE_FIELD_NAME) || choice.equals(DATE_RANGE_FIELD_NAME) || choice.equals(DATE_NANOS_FIELD_NAME), // TODO: needs testing for date_nanos type diff --git a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java index 52aafbff72d9a..a9c9ae7f422cf 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermsSetQueryBuilderTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.rest.ObjectPath; import java.io.IOException; @@ -61,7 +60,7 @@ public class TermsSetQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(CustomScriptPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(CustomScriptPlugin.class); } @Override @@ -76,10 +75,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws @Override protected TermsSetQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomValueOtherThanMany( - value -> value.equals(GEO_POINT_FIELD_NAME) || value.equals(GEO_SHAPE_FIELD_NAME), - () -> randomFrom(MAPPED_FIELD_NAMES) - ); + String fieldName = randomValueOtherThanMany(value -> value.equals(GEO_POINT_FIELD_NAME), () -> randomFrom(MAPPED_FIELD_NAMES)); List randomTerms = randomValues(fieldName); TermsSetQueryBuilder queryBuilder = new TermsSetQueryBuilder(TEXT_FIELD_NAME, randomTerms); if (randomBoolean()) { @@ -151,9 +147,7 @@ public TermsSetQueryBuilder mutateInstance(final TermsSetQueryBuilder instance) switch (randomIntBetween(0, 3)) { case 0 -> { - Predicate predicate = s -> s.equals(instance.getFieldName()) == false - && s.equals(GEO_POINT_FIELD_NAME) == false - && s.equals(GEO_SHAPE_FIELD_NAME) == false; + Predicate predicate = s -> s.equals(instance.getFieldName()) == false && s.equals(GEO_POINT_FIELD_NAME) == false; fieldName = randomValueOtherThanMany(predicate, () -> randomFrom(MAPPED_FIELD_NAMES)); values = randomValues(fieldName); } diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java index b779d7c747b19..75f28e8cdbf3c 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -90,7 +89,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(TestPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(TestPlugin.class); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java index 9e87d2009d626..d52cafc8e6857 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilderTests.java @@ -24,18 +24,14 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractBuilderTestCase; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -46,11 +42,6 @@ public class KnnVectorQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(TestGeoShapeFieldMapperPlugin.class); - } - @Override protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java b/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java index 7a442920c0f00..e289bec89e8c7 100644 --- a/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java +++ b/test/external-modules/delayed-aggs/src/test/java/org/elasticsearch/test/delayedshard/DelayedShardAggregationBuilderTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.BaseAggregationTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.util.Arrays; import java.util.Collection; @@ -18,7 +17,7 @@ public class DelayedShardAggregationBuilderTests extends BaseAggregationTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(DelayedShardAggregationPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(DelayedShardAggregationPlugin.class); } @Override diff --git a/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java b/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java index c92c2a5f89362..058e66caad4b9 100644 --- a/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java +++ b/test/external-modules/error-query/src/test/java/org/elasticsearch/test/errorquery/ErrorQueryBuilderTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import java.io.IOException; import java.util.ArrayList; @@ -24,7 +23,7 @@ public class ErrorQueryBuilderTests extends AbstractQueryTestCase { @Override protected Collection> getPlugins() { - return Arrays.asList(ErrorQueryPlugin.class, TestGeoShapeFieldMapperPlugin.class); + return Arrays.asList(ErrorQueryPlugin.class); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 660554efc6d2c..1a49a898a69f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -110,7 +110,6 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { protected static final String OBJECT_FIELD_NAME = "mapped_object"; protected static final String GEO_POINT_FIELD_NAME = "mapped_geo_point"; protected static final String GEO_POINT_ALIAS_FIELD_NAME = "mapped_geo_point_alias"; - protected static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; // we don't include the binary field in the arrays below as it is not searchable protected static final String BINARY_FIELD_NAME = "mapped_binary"; protected static final String[] MAPPED_FIELD_NAMES = new String[] { @@ -125,8 +124,7 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { DATE_RANGE_FIELD_NAME, OBJECT_FIELD_NAME, GEO_POINT_FIELD_NAME, - GEO_POINT_ALIAS_FIELD_NAME, - GEO_SHAPE_FIELD_NAME }; + GEO_POINT_ALIAS_FIELD_NAME }; protected static final String[] MAPPED_LEAF_FIELD_NAMES = new String[] { TEXT_FIELD_NAME, TEXT_ALIAS_FIELD_NAME, @@ -159,9 +157,8 @@ protected static Index getIndex() { return index; } - @SuppressWarnings("deprecation") // dependencies in server for geo_shape field should be decoupled protected Collection> getPlugins() { - return Collections.singletonList(TestGeoShapeFieldMapperPlugin.class); + return Collections.emptyList(); } /** @@ -455,8 +452,6 @@ public void onRemoval(ShardId shardId, Accountable accountable) { "type=geo_point", GEO_POINT_ALIAS_FIELD_NAME, "type=alias,path=" + GEO_POINT_FIELD_NAME, - GEO_SHAPE_FIELD_NAME, - "type=geo_shape", BINARY_FIELD_NAME, "type=binary" ) diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java index b8880330d3d30..3931af7d770a5 100644 --- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -110,7 +109,6 @@ protected void doAssertLuceneQuery(PinnedQueryBuilder queryBuilder, Query query, protected Collection> getPlugins() { List> classpathPlugins = new ArrayList<>(); classpathPlugins.add(SearchBusinessRules.class); - classpathPlugins.add(TestGeoShapeFieldMapperPlugin.class); return classpathPlugins; } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java index 3016fd92dc33f..cd174d4c6fb0a 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java @@ -12,17 +12,22 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.h3.H3; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.AbstractQueryTestCase; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import java.io.IOException; @@ -38,6 +43,20 @@ public class GeoGridQueryBuilderTests extends AbstractQueryTestCase { + private static final String GEO_SHAPE_FIELD_NAME = "mapped_geo_shape"; + protected static final String GEO_SHAPE_ALIAS_FIELD_NAME = "mapped_geo_shape_alias"; + + @Override + protected void initializeAdditionalMappings(MapperService mapperService) throws IOException { + final XContentBuilder builder = PutMappingRequest.simpleMapping( + GEO_SHAPE_FIELD_NAME, + "type=geo_shape", + GEO_SHAPE_ALIAS_FIELD_NAME, + "type=alias,path=" + GEO_SHAPE_FIELD_NAME + ); + mapperService.merge("_doc", new CompressedXContent(Strings.toString(builder)), MapperService.MergeReason.MAPPING_UPDATE); + } + @Override protected Collection> getPlugins() { return Arrays.asList(LocalStateSpatialPlugin.class); @@ -45,11 +64,11 @@ protected Collection> getPlugins() { @Override protected GeoGridQueryBuilder doCreateTestQueryBuilder() { - String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME); + String fieldName = randomFrom(GEO_POINT_FIELD_NAME, GEO_POINT_ALIAS_FIELD_NAME, GEO_SHAPE_FIELD_NAME, GEO_SHAPE_ALIAS_FIELD_NAME); GeoGridQueryBuilder builder = new GeoGridQueryBuilder(fieldName); // Only use geohex for points - int path = randomIntBetween(0, GEO_SHAPE_FIELD_NAME.equals(fieldName) ? 1 : 2); + int path = randomIntBetween(0, GEO_SHAPE_FIELD_NAME.equals(fieldName) || GEO_SHAPE_ALIAS_FIELD_NAME.equals(fieldName) ? 1 : 3); switch (path) { case 0 -> builder.setGridId(GeoGridQueryBuilder.Grid.GEOHASH, randomGeohash()); case 1 -> builder.setGridId(GeoGridQueryBuilder.Grid.GEOTILE, randomGeotile()); From 47510adab5064c09bc608de40c8737fa47abf14a Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Tue, 12 Jul 2022 08:27:00 +0200 Subject: [PATCH 13/40] Reduce map lookups (#88418) This change replcase 2 hash map operations with a single one and a null check. --- .../allocation/decider/NodeReplacementAllocationDecider.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java index dbaecd67cf429..485e0656c8163 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java @@ -15,7 +15,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; -import java.util.Map; import java.util.Optional; public class NodeReplacementAllocationDecider extends AllocationDecider { @@ -185,8 +184,8 @@ private static boolean isReplacementSource(RoutingAllocation allocation, String if (nodeId == null || replacementOngoing(allocation) == false) { return false; } - final Map nodeShutdowns = allocation.nodeShutdowns(); - return nodeShutdowns.containsKey(nodeId) && nodeShutdowns.get(nodeId).getType().equals(SingleNodeShutdownMetadata.Type.REPLACE); + final SingleNodeShutdownMetadata shutdown = allocation.nodeShutdowns().get(nodeId); + return shutdown != null && shutdown.getType().equals(SingleNodeShutdownMetadata.Type.REPLACE); } /** From 9ebbe1c62bf0157d332f20f392544a1921a710df Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Jul 2022 09:03:25 +0200 Subject: [PATCH 14/40] Make ClusterInfo use immutable maps in all cases (#88447) This class's maps are used very hot in the disk threshold allocation decider. Moving them from hppc maps to unmodifiable map wrapping `HashMap` has led to a measurable slowdown in the many-shards benchmark bootstrapping. Lets use immutable map copies here exclusively to make performance outright better and more predictable via a single implementation. --- .../java/org/elasticsearch/cluster/ClusterInfo.java | 12 ++++++------ .../cluster/InternalClusterInfoService.java | 13 ++++++------- .../allocation/DiskThresholdMonitorTests.java | 2 +- .../decider/DiskThresholdDeciderTests.java | 2 +- .../decider/DiskThresholdDeciderUnitTests.java | 10 +++++----- .../storage/ReactiveStorageDeciderService.java | 9 ++++++++- .../AutoscalingCalculateCapacityServiceTests.java | 4 ++-- .../storage/FrozenStorageDeciderServiceTests.java | 2 +- .../storage/ReactiveStorageDeciderServiceTests.java | 4 ++-- 9 files changed, 32 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index af931e6d34996..7cc4f1bdca65b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -68,12 +68,12 @@ public ClusterInfo( Map routingToDataPath, Map reservedSpace ) { - this.leastAvailableSpaceUsage = leastAvailableSpaceUsage; - this.shardSizes = shardSizes; - this.shardDataSetSizes = shardDataSetSizes; - this.mostAvailableSpaceUsage = mostAvailableSpaceUsage; - this.routingToDataPath = routingToDataPath; - this.reservedSpace = reservedSpace; + this.leastAvailableSpaceUsage = Map.copyOf(leastAvailableSpaceUsage); + this.shardSizes = Map.copyOf(shardSizes); + this.shardDataSetSizes = Map.copyOf(shardDataSetSizes); + this.mostAvailableSpaceUsage = Map.copyOf(mostAvailableSpaceUsage); + this.routingToDataPath = Map.copyOf(routingToDataPath); + this.reservedSpace = Map.copyOf(reservedSpace); } public ClusterInfo(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java index 0bed0fd0f7b6f..4db01fc3f796c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java +++ b/server/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java @@ -39,7 +39,6 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -188,8 +187,8 @@ public void onResponse(NodesStatsResponse nodesStatsResponse) { leastAvailableUsagesBuilder, mostAvailableUsagesBuilder ); - leastAvailableSpaceUsages = Collections.unmodifiableMap(leastAvailableUsagesBuilder); - mostAvailableSpaceUsages = Collections.unmodifiableMap(mostAvailableUsagesBuilder); + leastAvailableSpaceUsages = Map.copyOf(leastAvailableUsagesBuilder); + mostAvailableSpaceUsages = Map.copyOf(mostAvailableUsagesBuilder); } @Override @@ -262,10 +261,10 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) { reservedSpaceBuilders.forEach((nodeAndPath, builder) -> rsrvdSpace.put(nodeAndPath, builder.build())); indicesStatsSummary = new IndicesStatsSummary( - Collections.unmodifiableMap(shardSizeByIdentifierBuilder), - Collections.unmodifiableMap(shardDataSetSizeBuilder), - Collections.unmodifiableMap(dataPathByShardRoutingBuilder), - Collections.unmodifiableMap(rsrvdSpace) + Map.copyOf(shardSizeByIdentifierBuilder), + Map.copyOf(shardDataSetSizeBuilder), + Map.copyOf(dataPathByShardRoutingBuilder), + Map.copyOf(rsrvdSpace) ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java index 08e68db23e8e2..1e3be3f54c205 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java @@ -964,7 +964,7 @@ private static ClusterInfo clusterInfo( Map diskUsages, Map reservedSpace ) { - return new ClusterInfo(diskUsages, null, null, null, null, reservedSpace); + return new ClusterInfo(diskUsages, Map.of(), Map.of(), Map.of(), Map.of(), reservedSpace); } private static DiscoveryNode newFrozenOnlyNode(String nodeId) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index b1244ff4f17e5..373d915e37460 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -1308,7 +1308,7 @@ static class DevNullClusterInfo extends ClusterInfo { Map shardSizes, Map reservedSpace ) { - super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, null, null, reservedSpace); + super(leastAvailableSpaceUsage, mostAvailableSpaceUsage, shardSizes, Map.of(), Map.of(), reservedSpace); } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 37d94aba96d1a..cdc68d885c573 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -107,7 +107,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { leastAvailableUsages, mostAvailableUsage, Map.of("[test][0][p]", 10L), // 10 bytes, - null, + Map.of(), Map.of(), Map.of() ); @@ -185,7 +185,7 @@ public void testCannotAllocateDueToLackOfDiskResources() { leastAvailableUsages, mostAvailableUsage, Map.of("[test][0][p]", shardSize), - null, + Map.of(), Map.of(), Map.of() ); @@ -307,7 +307,7 @@ public void testCanRemainUsesLeastAvailableSpace() { leastAvailableUsages, mostAvailableUsage, shardSizes, - null, + Map.of(), shardRoutingMap, Map.of() ); @@ -745,7 +745,7 @@ public void testDecidesYesIfWatermarksIgnored() { allFullUsages, allFullUsages, Map.of("[test][0][p]", 10L), - null, + Map.of(), Map.of(), Map.of() ); @@ -815,7 +815,7 @@ public void testCannotForceAllocateOver100PercentUsage() { // bigger than available space final long shardSize = randomIntBetween(1, 10); shardSizes.put("[test][0][p]", shardSize); - ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, null, Map.of(), Map.of()); + ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages, mostAvailableUsage, shardSizes, Map.of(), Map.of(), Map.of()); RoutingAllocation allocation = new RoutingAllocation( new AllocationDeciders(Collections.singleton(decider)), clusterState, diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 5d0c1b426a3f7..c168e7b3b08f0 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -793,7 +793,14 @@ private static class ExtendedClusterInfo extends ClusterInfo { private final ClusterInfo delegate; private ExtendedClusterInfo(Map extraShardSizes, ClusterInfo info) { - super(info.getNodeLeastAvailableDiskUsages(), info.getNodeMostAvailableDiskUsages(), extraShardSizes, Map.of(), null, null); + super( + info.getNodeLeastAvailableDiskUsages(), + info.getNodeMostAvailableDiskUsages(), + extraShardSizes, + Map.of(), + Map.of(), + Map.of() + ); this.delegate = info; } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java index 0ea41e147cec7..d7d715334a2dc 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityServiceTests.java @@ -260,7 +260,7 @@ public void testContext() { } } state = ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build(); - info = new ClusterInfo(leastUsages, mostUsages, null, null, null, null); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, @@ -306,7 +306,7 @@ public void testContext() { ) ); - info = new ClusterInfo(leastUsages, mostUsages, null, null, null, null); + info = new ClusterInfo(leastUsages, mostUsages, Map.of(), Map.of(), Map.of(), Map.of()); context = new AutoscalingCalculateCapacityService.DefaultAutoscalingDeciderContext( roleNames, state, diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java index f1a52024dcca2..d88e067777185 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/FrozenStorageDeciderServiceTests.java @@ -109,7 +109,7 @@ public Tuple sizeAndClusterInfo(IndexMetadata indexMetadata) // add irrelevant shards noise for completeness (should not happen IRL). sizes.put(new ShardId(index, i), randomLongBetween(0, Integer.MAX_VALUE)); } - ClusterInfo info = new ClusterInfo(null, null, null, sizes, null, null); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), Map.of(), sizes, Map.of(), Map.of()); return Tuple.tuple(totalSize, info); } } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index 4483a1bbe9261..0cbb3b27725a3 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -403,7 +403,7 @@ public void validateSizeOf(ClusterState clusterState, ShardRouting subjectShard, } private ReactiveStorageDeciderService.AllocationState createAllocationState(Map shardSize, ClusterState clusterState) { - ClusterInfo info = new ClusterInfo(null, null, shardSize, null, null, null); + ClusterInfo info = new ClusterInfo(Map.of(), Map.of(), shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, null, @@ -567,7 +567,7 @@ public void testUnmovableSize() { if (shardsWithSizes.isEmpty() == false) { shardSize.put(shardIdentifier(randomFrom(shardsWithSizes)), ByteSizeUnit.KB.toBytes(minShardSize)); } - ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, null, null, null); + ClusterInfo info = new ClusterInfo(diskUsages, diskUsages, shardSize, Map.of(), Map.of(), Map.of()); ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState( clusterState, From a4ec9c69cafceabf4694bae370897c217b6809ed Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Tue, 12 Jul 2022 09:57:05 +0200 Subject: [PATCH 15/40] Correct some typos/mistakes in comments/docs (#88446) --- .../elasticsearch/action/support/ListenableActionFuture.java | 2 +- .../main/java/org/elasticsearch/snapshots/RestoreService.java | 2 +- .../main/java/org/elasticsearch/snapshots/package-info.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java index 7da2324bb873b..bbac7cba72b43 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java @@ -14,7 +14,7 @@ import java.util.List; /** - * A {@code Future} and {@link ActionListener} against which which other {@link ActionListener}s can be registered later, to support + * A {@code Future} and {@link ActionListener} against which other {@link ActionListener}s can be registered later, to support * fanning-out a result to a dynamic collection of listeners. */ public class ListenableActionFuture extends AdapterActionFuture { diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index c1307e8706833..6a2f49f99a2d7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1307,7 +1307,7 @@ public ClusterState execute(ClusterState currentState) { // that will be opened by the restore if (currentIndexMetadata == null) { // Index doesn't exist - create it and start recovery - // Make sure that the index we are about to create has a validate name + // Make sure that the index we are about to create has a valid name ensureValidIndexName(currentState, snapshotIndexMetadata, renamedIndexName); shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState); diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 2df9a96656d01..3972a9ac02d5f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -43,9 +43,9 @@ * {@code UpdateIndexShardSnapshotStatusRequest}. * *
  • If as a result of the received status update requests, all shards in the cluster state are in a completed state, i.e are marked as - * either {@code SUCCESS}, {@code FAILED} or {@code MISSING}, the {@code SnapshotShardsService} will update the state of the {@code Entry} + * either {@code SUCCESS}, {@code FAILED} or {@code MISSING}, the {@code SnapshotsService} will update the state of the {@code Entry} * itself and mark it as {@code SUCCESS}. At the same time {@link org.elasticsearch.snapshots.SnapshotsService#endSnapshot} is executed, - * writing the metadata necessary to finalize the snapshot in the repository to the repository.
    • + * writing to the repository the metadata necessary to finalize the snapshot in the repository. * *
  • After writing the final metadata to the repository, a cluster state update to remove the snapshot from the cluster state is * submitted and the removal of the snapshot's {@code SnapshotsInProgress.Entry} from the cluster state completes the snapshot process. From ecc9605ff6ff38a8bb2e45e723ded36552bb91bd Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 12 Jul 2022 16:23:06 +0800 Subject: [PATCH 16/40] [TSDB] Cache rollup bucket timestamp to reduce rounding cost (#88420) --- .../xpack/rollup/v2/RollupShardIndexer.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 68e09e470cdce..056e96d51af09 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -209,6 +209,7 @@ private class TimeSeriesBucketCollector extends BucketCollector { private long bucketsCreated; private final RollupBucketBuilder rollupBucketBuilder = new RollupBucketBuilder(); long lastTimestamp = Long.MAX_VALUE; + long lastHistoTimestamp = Long.MAX_VALUE; BytesRef lastTsid = null; TimeSeriesBucketCollector(BulkProcessor bulkProcessor) { @@ -232,14 +233,18 @@ public void collect(int docId, long owningBucketOrd) throws IOException { final BytesRef tsid = aggCtx.getTsid(); assert tsid != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; final long timestamp = aggCtx.getTimestamp(); - final long histoTimestamp = rounding.round(timestamp); + + boolean tsidChanged = tsid.equals(rollupBucketBuilder.tsid()) == false; + if (tsidChanged || timestamp < lastHistoTimestamp) { + lastHistoTimestamp = rounding.round(timestamp); + } logger.trace( "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", docId, DocValueFormat.TIME_SERIES_ID.format(tsid), timestampFormat.format(timestamp), - timestampFormat.format(histoTimestamp) + timestampFormat.format(lastHistoTimestamp) ); /* @@ -262,7 +267,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { lastTsid = BytesRef.deepCopyOf(tsid); lastTimestamp = timestamp; - if (tsid.equals(rollupBucketBuilder.tsid()) == false || rollupBucketBuilder.timestamp() != histoTimestamp) { + if (tsidChanged || rollupBucketBuilder.timestamp() != lastHistoTimestamp) { // Flush rollup doc if not empty if (rollupBucketBuilder.isEmpty() == false) { Map doc = rollupBucketBuilder.buildRollupDocument(); @@ -270,7 +275,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { } // Create new rollup bucket - rollupBucketBuilder.init(tsid, histoTimestamp); + rollupBucketBuilder.init(tsid, lastHistoTimestamp); bucketsCreated++; } From 48896509d8982bdca59a104e556d350e13deedd9 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Jul 2022 10:53:24 +0200 Subject: [PATCH 17/40] Pass IndexMetadata to AllocationDecider.can_remain (#88453) We need the metadata in a number of allocation deciders and pass it to other allocation methods. Passing it here avoids redundant lookups across deciders. --- .../allocation/decider/AllocationDecider.java | 2 +- .../decider/AllocationDeciders.java | 5 +- .../decider/AwarenessAllocationDecider.java | 15 ++++-- .../decider/DiskThresholdDecider.java | 4 +- .../decider/FilterAllocationDecider.java | 18 ++----- .../NodeReplacementAllocationDecider.java | 2 +- .../NodeShutdownAllocationDecider.java | 2 +- .../decider/ShardsLimitAllocationDecider.java | 14 ++++-- .../RandomAllocationDeciderTests.java | 2 +- .../decider/AllocationDecidersTests.java | 47 +++++++++++++++---- .../decider/DiskThresholdDeciderTests.java | 23 +++++++-- .../DiskThresholdDeciderUnitTests.java | 19 +++++--- ...NodeReplacementAllocationDeciderTests.java | 8 ++-- .../NodeShutdownAllocationDeciderTests.java | 4 +- .../ReactiveStorageDeciderDecisionTests.java | 6 +-- .../ReactiveStorageDeciderServiceTests.java | 7 ++- .../allocation/DataTierAllocationDecider.java | 10 ++-- .../DataTierAllocationDeciderTests.java | 7 ++- .../DedicatedFrozenNodeAllocationDecider.java | 4 +- .../HasFrozenCacheAllocationDecider.java | 4 +- ...TransportGetShutdownStatusActionTests.java | 7 ++- 21 files changed, 139 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java index af36ca86ed44c..1b5cf0805a821 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java @@ -42,7 +42,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing * Returns a {@link Decision} whether the given shard routing can be remain * on the given node. The default is {@link Decision#ALWAYS}. */ - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return Decision.ALWAYS; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 02546ccbd9b90..ff0634e5ee2f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -86,10 +86,11 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl } return Decision.NO; } + final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (allocation.debugDecision()) { Decision.Multi ret = new Decision.Multi(); for (AllocationDecider allocationDecider : allocations) { - Decision decision = allocationDecider.canRemain(shardRouting, node, allocation); + Decision decision = allocationDecider.canRemain(indexMetadata, shardRouting, node, allocation); // short track if a NO is returned. if (decision.type() == Decision.Type.NO) { maybeTraceLogNoDecision(shardRouting, node, allocationDecider); @@ -103,7 +104,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl // tighter loop if debug information is not collected: don't collect yes decisions + break out right away on NO Decision ret = Decision.YES; for (AllocationDecider allocationDecider : allocations) { - switch (allocationDecider.canRemain(shardRouting, node, allocation).type()) { + switch (allocationDecider.canRemain(indexMetadata, shardRouting, node, allocation).type()) { case NO -> { maybeTraceLogNoDecision(shardRouting, node, allocationDecider); return Decision.NO; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 5f981123e2c53..73873e4a3d693 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -123,7 +123,7 @@ private void setAwarenessAttributes(List awarenessAttributes) { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return underCapacity(shardRouting, node, allocation, true); + return underCapacity(allocation.metadata().getIndexSafe(shardRouting.index()), shardRouting, node, allocation, true); } @Override @@ -135,8 +135,8 @@ public Decision canForceAllocateDuringReplace(ShardRouting shardRouting, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return underCapacity(shardRouting, node, allocation, false); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return underCapacity(indexMetadata, shardRouting, node, allocation, false); } private static final Decision YES_NOT_ENABLED = Decision.single( @@ -155,13 +155,18 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl private static final Decision YES_ALL_MET = Decision.single(Decision.Type.YES, NAME, "node meets all awareness attribute requirements"); - private Decision underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) { + private Decision underCapacity( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation, + boolean moveToNode + ) { if (awarenessAttributes.isEmpty()) { return YES_NOT_ENABLED; } final boolean debug = allocation.debugDecision(); - final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (indexMetadata.getAutoExpandReplicas().expandToAllNodes()) { return YES_AUTO_EXPAND_ALL; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 6917b9d9b7b18..69aae52a20ca4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -461,7 +461,7 @@ public Decision canForceAllocateDuringReplace(ShardRouting shardRouting, Routing ); @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (shardRouting.currentNodeId().equals(node.nodeId()) == false) { throw new IllegalArgumentException("Shard [" + shardRouting + "] is not allocated on node: [" + node.nodeId() + "]"); } @@ -472,7 +472,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl return decision; } - if (allocation.metadata().index(shardRouting.index()).ignoreDiskWatermarks()) { + if (indexMetadata.ignoreDiskWatermarks()) { return YES_DISK_WATERMARKS_IGNORED; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java index fda55b419ff01..e32fc14bc617b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java @@ -90,11 +90,11 @@ public FilterAllocationDecider(Settings settings, ClusterSettings clusterSetting @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); if (shardRouting.unassigned() && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { // only for unassigned - we filter allocation right after the index creation (for shard shrinking) to ensure // that once it has been allocated post API the replicas can be allocated elsewhere without user interaction // this is a setting that can only be set within the system! - IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index()); DiscoveryNodeFilters initialRecoveryFilters = DiscoveryNodeFilters.trimTier(indexMetadata.getInitialRecoveryFilters()); if (initialRecoveryFilters != null && initialRecoveryFilters.match(node.node()) == false) { String explanation = @@ -102,7 +102,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters); } } - return shouldFilter(shardRouting, node.node(), allocation); + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -111,8 +111,8 @@ public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, Routi } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -126,16 +126,6 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters"); } - private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) { - Decision decision = shouldClusterFilter(node, allocation); - if (decision != null) return decision; - - decision = shouldIndexFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node, allocation); - if (decision != null) return decision; - - return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters"); - } - private Decision shouldFilter(IndexMetadata indexMd, DiscoveryNode node, RoutingAllocation allocation) { Decision decision = shouldClusterFilter(node, allocation); if (decision != null) return decision; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java index 485e0656c8163..f97e8ce9b5cea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDecider.java @@ -73,7 +73,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (replacementOngoing(allocation) == false) { return NO_REPLACEMENTS; } else if (isReplacementSource(allocation, node.nodeId())) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java index ff6cd0796e98e..0c6a481ce03eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java @@ -63,7 +63,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing * determine if shards can remain on their current node. */ @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return this.canAllocate(shardRouting, node, allocation); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java index f9868bf312224..e53688654e64b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java @@ -81,22 +81,28 @@ private void setClusterShardLimit(int clusterShardLimit) { @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return doDecide(shardRouting, node, allocation, (count, limit) -> count >= limit); + return doDecide( + allocation.metadata().getIndexSafe(shardRouting.index()), + shardRouting, + node, + allocation, + (count, limit) -> count >= limit + ); } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return doDecide(shardRouting, node, allocation, (count, limit) -> count > limit); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return doDecide(indexMetadata, shardRouting, node, allocation, (count, limit) -> count > limit); } private Decision doDecide( + IndexMetadata indexMd, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, BiPredicate decider ) { - IndexMetadata indexMd = allocation.metadata().getIndexSafe(shardRouting.index()); final int indexShardLimit = indexMd.getShardsPerNodeLimit(); // Capture the limit here in case it changes during this method's // execution diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java index 59419dd4aa3d0..76b1aba8a39cf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java @@ -224,7 +224,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return getRandomDecision(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java index 7d62cd4da927b..c8a06783ebba6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; @@ -54,7 +55,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return Decision.YES; } @@ -79,18 +85,25 @@ public Decision canRebalance(RoutingAllocation allocation) { } })); - ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build(); + IndexMetadata idx = IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build(); + IndexMetadata testIdx = IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(idx, false).put(testIdx, false).build()) + .build(); final RoutingAllocation allocation = new RoutingAllocation(deciders, clusterState, null, null, 0L); allocation.setDebugMode(mode); final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"); final ShardRouting shardRouting = ShardRouting.newUnassigned( - new ShardId("test", "testUUID", 0), + new ShardId(testIdx.getIndex(), 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, unassignedInfo ); - IndexMetadata idx = IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build(); RoutingNode routingNode = RoutingNodesHelper.routingNode("testNode", null); verify(deciders.canAllocate(shardRouting, routingNode, allocation), matcher); @@ -130,7 +143,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return decisionOne; } @@ -171,7 +189,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return decision(allocation); } @@ -208,20 +231,28 @@ private Decision decision(RoutingAllocation allocation) { } })); + IndexMetadata testIdx = IndexMetadata.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + // no debug should just short-circuit to no, no matter what kind of no type return the first decider returns final ShardRouting shardRouting = ShardRouting.newUnassigned( - new ShardId("test", "testUUID", 0), + new ShardId(testIdx.getIndex(), 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message") ); final RoutingNode routingNode = RoutingNodesHelper.routingNode("testNode", null); - final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build(); final IndexMetadata indexMetadata = IndexMetadata.builder("idx") .settings(settings(Version.CURRENT)) .numberOfShards(1) .numberOfReplicas(0) .build(); + final ClusterState clusterState = ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().put(testIdx, false).put(indexMetadata, false).build()) + .build(); final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, clusterState, null, null, 0L); assertSame(Decision.NO, allocationDeciders.canAllocate(shardRouting, routingNode, allocation)); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 373d915e37460..582521729fdea 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -919,10 +919,15 @@ public void testCanRemainWithShardRelocatingAway() { System.nanoTime() ); routingAllocation.debugDecision(true); - Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); + Decision decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(firstRouting.index()), + firstRouting, + firstRoutingNode, + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( - ((Decision.Single) decision).getExplanation(), + decision.getExplanation(), containsString( "the shard cannot remain on this node because it is above the high watermark cluster setting " + "[cluster.routing.allocation.disk.watermark.high=70%] and there is less than the required [30.0%] free disk on node, " @@ -951,7 +956,12 @@ public void testCanRemainWithShardRelocatingAway() { System.nanoTime() ); routingAllocation.debugDecision(true); - decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); + decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(firstRouting.index()), + firstRouting, + firstRoutingNode, + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertEquals( "there is enough disk on this node for the shard to remain, free: [60b]", @@ -1109,7 +1119,12 @@ public void testWatermarksEnabledForSingleDataNode() { System.nanoTime() ); routingAllocation.debugDecision(true); - Decision decision = diskThresholdDecider.canRemain(startedShard, clusterState.getRoutingNodes().node("data"), routingAllocation); + Decision decision = diskThresholdDecider.canRemain( + routingAllocation.metadata().getIndexSafe(startedShard.index()), + startedShard, + clusterState.getRoutingNodes().node("data"), + routingAllocation + ); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( decision.getExplanation(), diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index cdc68d885c573..dd380fb98e725 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -319,13 +319,13 @@ public void testCanRemainUsesLeastAvailableSpace() { System.nanoTime() ); allocation.debugDecision(true); - Decision decision = decider.canRemain(test_0, RoutingNodesHelper.routingNode("node_0", node_0), allocation); + Decision decision = decider.canRemain(indexMetadata, test_0, RoutingNodesHelper.routingNode("node_0", node_0), allocation); assertEquals(Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), containsString("there is enough disk on this node for the shard to remain, free: [10b]") ); - decision = decider.canRemain(test_1, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_1, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals(Decision.Type.NO, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), @@ -336,26 +336,26 @@ public void testCanRemainUsesLeastAvailableSpace() { ) ); try { - decider.canRemain(test_0, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decider.canRemain(indexMetadata, test_0, RoutingNodesHelper.routingNode("node_1", node_1), allocation); fail("not allocated on this node"); } catch (IllegalArgumentException ex) { // not allocated on that node } try { - decider.canRemain(test_1, RoutingNodesHelper.routingNode("node_0", node_0), allocation); + decider.canRemain(indexMetadata, test_1, RoutingNodesHelper.routingNode("node_0", node_0), allocation); fail("not allocated on this node"); } catch (IllegalArgumentException ex) { // not allocated on that node } - decision = decider.canRemain(test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals("can stay since allocated on a different path with enough space", Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), containsString("this shard is not allocated on the most utilized disk and can remain") ); - decision = decider.canRemain(test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); + decision = decider.canRemain(indexMetadata, test_2, RoutingNodesHelper.routingNode("node_1", node_1), allocation); assertEquals("can stay since we don't have information about this shard", Decision.Type.YES, decision.type()); assertThat( ((Decision.Single) decision).getExplanation(), @@ -762,7 +762,12 @@ public void testDecidesYesIfWatermarksIgnored() { assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), containsString("disk watermarks are ignored on this index")); - decision = decider.canRemain(test_0.initialize(node_0.getId(), null, 0L).moveToStarted(), routingNode, allocation); + decision = decider.canRemain( + metadata.getIndexSafe(test_0.index()), + test_0.initialize(node_0.getId(), null, 0L).moveToStarted(), + routingNode, + allocation + ); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), containsString("disk watermarks are ignored on this index")); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java index 173fb7619e5ec..824af0be3e0cc 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java @@ -95,7 +95,7 @@ public void testNoReplacements() { assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo(NodeReplacementAllocationDecider.NO_REPLACEMENTS.getExplanation())); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo(NodeReplacementAllocationDecider.NO_REPLACEMENTS.getExplanation())); } @@ -149,7 +149,7 @@ public void testCannotRemainOnReplacedNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(NODE_A.getId(), NODE_A, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat( decision.getExplanation(), @@ -158,13 +158,13 @@ public void testCannotRemainOnReplacedNode() { routingNode = RoutingNodesHelper.routingNode(NODE_B.getId(), NODE_B, shard); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo("node [" + NODE_B.getId() + "] is not being replaced")); routingNode = RoutingNodesHelper.routingNode(NODE_C.getId(), NODE_C, shard); - decision = decider.canRemain(shard, routingNode, allocation); + decision = decider.canRemain(indexMetadata, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat(decision.getExplanation(), equalTo("node [" + NODE_C.getId() + "] is not being replaced")); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java index f018ef21ac225..ca92761dba516 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java @@ -117,7 +117,7 @@ public void testShardsCanRemainOnRestartingNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); assertThat( decision.getExplanation(), @@ -134,7 +134,7 @@ public void testShardsCannotRemainOnRemovingNode() { RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard); allocation.debugDecision(true); - Decision decision = decider.canRemain(shard, routingNode, allocation); + Decision decision = decider.canRemain(null, shard, routingNode, allocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster")); } diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java index c7d81944cdee1..4d13727cd268b 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java @@ -88,7 +88,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing }; private static final AllocationDecider CAN_REMAIN_NO_DECIDER = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return Decision.NO; } }; @@ -116,13 +116,13 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing // say NO with disk label for subject shards private final AllocationDecider mockCanRemainDiskDecider = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (subjectShards.contains(shardRouting.shardId()) && node.node().getName().startsWith("hot")) return allocation.decision( Decision.NO, DiskThresholdDecider.NAME, "test" ); - return super.canRemain(shardRouting, node, allocation); + return super.canRemain(indexMetadata, shardRouting, node, allocation); } }; diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java index 0cbb3b27725a3..353fa8d789c84 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java @@ -615,7 +615,12 @@ public void testCanRemainOnlyHighestTierPreference() { AllocationDecider no = new AllocationDecider() { @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return Decision.NO; } }; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java index 0dc1f22a85c46..cf2f17903960e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java @@ -41,7 +41,7 @@ private DataTierAllocationDecider() {} @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + return shouldFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node.node(), allocation); } @Override @@ -50,8 +50,8 @@ public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, Routi } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return shouldFilter(shardRouting, node.node(), allocation); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.node(), allocation); } @Override @@ -59,8 +59,8 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod return shouldFilter(indexMetadata, node.getRoles(), allocation); } - private Decision shouldFilter(ShardRouting shardRouting, DiscoveryNode node, RoutingAllocation allocation) { - return shouldFilter(allocation.metadata().getIndexSafe(shardRouting.index()), node.getRoles(), allocation); + private Decision shouldFilter(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) { + return shouldFilter(indexMetadata, node.getRoles(), allocation); } private static Decision shouldFilter(IndexMetadata indexMd, Set roles, RoutingAllocation allocation) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java index 712053287c683..bdbdfc4f77f56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderTests.java @@ -646,7 +646,12 @@ private void assertAllocationDecision(ClusterState state, DiscoveryNode node, De } { - final var decision = DataTierAllocationDecider.INSTANCE.canRemain(shard, routingNode, allocation); + final var decision = DataTierAllocationDecider.INSTANCE.canRemain( + allocation.metadata().getIndexSafe(shard.index()), + shard, + routingNode, + allocation + ); assertThat(routingNode.toString(), decision.type(), equalTo(decisionType)); assertThat(routingNode.toString(), decision.getExplanation(), containsString(explanationMessage)); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java index 58f2af3a0c944..cf9159d3f5af5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java @@ -48,8 +48,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node.node()); } @Override diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java index 666b33512f351..d16f5483aab09 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/HasFrozenCacheAllocationDecider.java @@ -60,8 +60,8 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node.node()); } @Override diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java index a12fe6bd57a15..f1ccfdd77f19b 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java @@ -93,7 +93,12 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca } @Override - public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + public Decision canRemain( + IndexMetadata indexMetadata, + ShardRouting shardRouting, + RoutingNode node, + RoutingAllocation allocation + ) { return canRemain.get().test(shardRouting, node, allocation); } From a2ee4c5393dbb184255b98f77665265e70c63c57 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 12 Jul 2022 12:48:56 +0200 Subject: [PATCH 18/40] Polish reworked LoggedExec task (#88424) Some polishing of reworked LoggedExec task --- .../gradle/LoggedExecFuncTest.groovy | 6 +++--- .../org/elasticsearch/gradle/LoggedExec.java | 16 +++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy index 302fb2bcc2257..5a92f61c70d8c 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy @@ -35,7 +35,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.LoggedExec tasks.register('loggedExec', LoggedExec) { commandLine 'ls', '-lh' - spoolOutput = $spooling + getSpoolOutput().set($spooling) } """ when: @@ -54,7 +54,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { import org.elasticsearch.gradle.LoggedExec tasks.register('loggedExec', LoggedExec) { commandLine 'ls', 'wtf' - spoolOutput = $spooling + getSpoolOutput().set($spooling) } """ when: @@ -97,7 +97,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest { tasks.register('loggedExec', LoggedExec) { commandLine 'echo', 'HELLO' getCaptureOutput().set(true) - spoolOutput = true + getSpoolOutput().set(true) } """ when: diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java index 9740a0c2f5425..acb526cf9a3bb 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java @@ -56,7 +56,6 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat protected FileSystemOperations fileSystemOperations; private ProjectLayout projectLayout; private ExecOperations execOperations; - private boolean spoolOutput; @Input @Optional @@ -84,6 +83,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat @Input abstract public Property getWorkingDir(); + @Internal + abstract public Property getSpoolOutput(); + private String output; @Inject @@ -95,14 +97,16 @@ public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, Fi // For now mimic default behaviour of Gradle Exec task here getEnvironment().putAll(System.getenv()); getCaptureOutput().convention(false); + getSpoolOutput().convention(false); } @TaskAction public void run() { + boolean spoolOutput = getSpoolOutput().get(); if (spoolOutput && getCaptureOutput().get()) { throw new GradleException("Capturing output is not supported when spoolOutput is true."); } - if (getCaptureOutput().getOrElse(false) && getIndentingConsoleOutput().isPresent()) { + if (getCaptureOutput().get() && getIndentingConsoleOutput().isPresent()) { throw new GradleException("Capturing output is not supported when indentingConsoleOutput is configured."); } Consumer outputLogger; @@ -156,7 +160,9 @@ public void run() { if (getLogger().isInfoEnabled() == false) { if (exitValue != 0) { try { - getLogger().error("Output for " + getExecutable().get() + ":"); + if (getIndentingConsoleOutput().isPresent() == false) { + getLogger().error("Output for " + getExecutable().get() + ":"); + } outputLogger.accept(getLogger()); } catch (Exception e) { throw new GradleException("Failed to read exec output", e); @@ -173,10 +179,6 @@ private String byteStreamToString(OutputStream out) { return ((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8); } - public void setSpoolOutput(boolean spoolOutput) { - this.spoolOutput = spoolOutput; - } - public static ExecResult exec(ExecOperations execOperations, Action action) { return genericExec(execOperations::exec, action); } From c56715f479f56ef738b4a5292f476d115c17a208 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 12 Jul 2022 14:16:11 +0200 Subject: [PATCH 19/40] Updatable API keys - logging audit trail event (#88276) This PR adds a new audit trail event for when API keys are updated. --- docs/changelog/88276.yaml | 5 +++ .../en/security/auditing/event-types.asciidoc | 35 ++++++++++++++++--- .../audit/logfile/LoggingAuditTrail.java | 30 +++++++++++++++- .../audit/logfile/LoggingAuditTrailTests.java | 32 ++++++++++++++++- 4 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/88276.yaml diff --git a/docs/changelog/88276.yaml b/docs/changelog/88276.yaml new file mode 100644 index 0000000000000..942d83375b361 --- /dev/null +++ b/docs/changelog/88276.yaml @@ -0,0 +1,5 @@ +pr: 88276 +summary: Updatable API keys - logging audit trail event +area: Audit +type: enhancement +issues: [] diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 7cc041ff4ccf8..f3ca8a303725b 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -231,6 +231,29 @@ event action. ["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} ==== +[[event-change-apikey]] +`change_apikey`:: +Logged when the <> API is +invoked to update the attributes of an existing API key. ++ +You must include the `security_config_change` event type to audit the related +event action. ++ +.Example +[%collapsible%open] +==== +[source,js] +{"type":"audit", "timestamp":"2020-12-31T00:33:52,521+0200", "node.id": +"9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action": +"change_apikey", "request.id":"9FteCmovTzWHVI-9Gpa_vQ", "change":{"apikey": +{"id":"zcwN3YEBBmnjw-K-hW5_","role_descriptors":[{"cluster": +["monitor","manage_ilm"],"indices":[{"names":["index-a*"],"privileges": +["read","maintenance"]},{"names":["in*","alias*"],"privileges":["read"], +"field_security":{"grant":["field1*","@timestamp"],"except":["field11"]}}], +"applications":[],"run_as":[]},{"cluster":["all"],"indices":[{"names": +["index-b*"],"privileges":["all"]}],"applications":[],"run_as":[]}]}}} +==== + [[event-delete-privileges]] `delete_privileges`:: Logged when the @@ -535,8 +558,8 @@ In addition, if `event.type` equals <>. The `role_descriptors` objects have the same schema as the `role_descriptor` object that is part of the above `role` config object. +The object for an API key update will differ in that it will not include +a `name` or `expiration`. + `grant` :: An object like: + [source,js] diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 33f48b65fe9d1..6ea823564db02 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -49,6 +49,8 @@ import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -287,7 +289,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { DeleteServiceAccountTokenAction.NAME, ActivateProfileAction.NAME, UpdateProfileDataAction.NAME, - SetProfileEnabledAction.NAME + SetProfileEnabledAction.NAME, + UpdateApiKeyAction.NAME ); private static final String FILTER_POLICY_PREFIX = setting("audit.logfile.events.ignore_filters."); // because of the default wildcard value (*) for the field filter, a policy with @@ -747,6 +750,9 @@ public void accessGranted( } else if (msg instanceof final SetProfileEnabledRequest setProfileEnabledRequest) { assert SetProfileEnabledAction.NAME.equals(action); securityChangeLogEntryBuilder(requestId).withRequestBody(setProfileEnabledRequest).build(); + } else if (msg instanceof final UpdateApiKeyRequest updateApiKeyRequest) { + assert UpdateApiKeyAction.NAME.equals(action); + securityChangeLogEntryBuilder(requestId).withRequestBody(updateApiKeyRequest).build(); } else { throw new IllegalStateException( "Unknown message class type [" @@ -1215,6 +1221,16 @@ LogEntryBuilder withRequestBody(GrantApiKeyRequest grantApiKeyRequest) throws IO return this; } + LogEntryBuilder withRequestBody(final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + logEntry.with(EVENT_ACTION_FIELD_NAME, "change_apikey"); + XContentBuilder builder = JsonXContent.contentBuilder().humanReadable(true); + builder.startObject(); + withRequestBody(builder, updateApiKeyRequest); + builder.endObject(); + logEntry.with(CHANGE_CONFIG_FIELD_NAME, Strings.toString(builder)); + return this; + } + private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest createApiKeyRequest) throws IOException { TimeValue expiration = createApiKeyRequest.getExpiration(); builder.startObject("apikey") @@ -1228,6 +1244,18 @@ private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest create .endObject(); // apikey } + private void withRequestBody(final XContentBuilder builder, final UpdateApiKeyRequest updateApiKeyRequest) throws IOException { + builder.startObject("apikey").field("id", updateApiKeyRequest.getId()); + if (updateApiKeyRequest.getRoleDescriptors() != null) { + builder.startArray("role_descriptors"); + for (RoleDescriptor roleDescriptor : updateApiKeyRequest.getRoleDescriptors()) { + withRoleDescriptor(builder, roleDescriptor); + } + builder.endArray(); + } + builder.endObject(); + } + private void withRoleDescriptor(XContentBuilder builder, RoleDescriptor roleDescriptor) throws IOException { builder.startObject().array(RoleDescriptor.Fields.CLUSTER.getPreferredName(), roleDescriptor.getClusterPrivileges()); if (roleDescriptor.getConditionalClusterPrivileges() != null && roleDescriptor.getConditionalClusterPrivileges().length > 0) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 4ea4ce42a19eb..540b5fa5c1507 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -46,12 +46,15 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequest; import org.elasticsearch.xpack.core.security.action.privilege.PutPrivilegesAction; @@ -605,6 +608,32 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException // clear log CapturingLogger.output(logger.getName(), Level.INFO).clear(); + final String keyId = randomAlphaOfLength(10); + final var updateApiKeyRequest = new UpdateApiKeyRequest( + keyId, + randomBoolean() ? null : keyRoleDescriptors, + ApiKeyTests.randomMetadata() + ); + auditTrail.accessGranted(requestId, authentication, UpdateApiKeyAction.NAME, updateApiKeyRequest, authorizationInfo); + final var expectedUpdateKeyAuditEventString = """ + "change":{"apikey":{"id":"%s"%s}}\ + """.formatted(keyId, updateApiKeyRequest.getRoleDescriptors() == null ? "" : "," + roleDescriptorsStringBuilder); + output = CapturingLogger.output(logger.getName(), Level.INFO); + assertThat(output.size(), is(2)); + String generatedUpdateKeyAuditEventString = output.get(1); + assertThat(generatedUpdateKeyAuditEventString, containsString(expectedUpdateKeyAuditEventString)); + generatedUpdateKeyAuditEventString = generatedUpdateKeyAuditEventString.replace(", " + expectedUpdateKeyAuditEventString, ""); + checkedFields = new MapBuilder<>(commonFields); + checkedFields.remove(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME); + checkedFields.remove(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME); + checkedFields.put("type", "audit") + .put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, "security_config_change") + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "change_apikey") + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + assertMsg(generatedUpdateKeyAuditEventString, checkedFields.map()); + // clear log + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(); grantApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); grantApiKeyRequest.getGrant().setType(randomFrom(randomAlphaOfLength(8), null)); @@ -1800,7 +1829,8 @@ public void testSecurityConfigChangedEventSelection() { new Tuple<>( SetProfileEnabledAction.NAME, new SetProfileEnabledRequest(randomAlphaOfLength(20), randomBoolean(), WriteRequest.RefreshPolicy.WAIT_UNTIL) - ) + ), + new Tuple<>(UpdateApiKeyAction.NAME, UpdateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(10))) ); auditTrail.accessGranted(requestId, authentication, actionAndRequest.v1(), actionAndRequest.v2(), authorizationInfo); List output = CapturingLogger.output(logger.getName(), Level.INFO); From 5628b8792e5b7331bb0df899115396b8751488e9 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 12 Jul 2022 21:24:49 +0900 Subject: [PATCH 20/40] Bound random negative size test in SearchSourceBuilderTests#testNegativeSizeErrors (#88457) -1 is handled differently by the xcontent code path so this test will fail when `randomIntBetween` lands on -1. To fix, we add another integer for the xcontent test which starts at -2. --- .../search/builder/SearchSourceBuilderTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index ffe2d3c1bafbf..9c0c11a179115 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -500,10 +500,12 @@ public void testNegativeSizeErrors() throws IOException { expected = expectThrows(IllegalArgumentException.class, () -> new SearchSourceBuilder().size(-1)); assertEquals("[size] parameter cannot be negative, found [-1]", expected.getMessage()); - String restContent = "{\"size\" : " + randomSize + "}"; + // SearchSourceBuilder.fromXContent treats -1 as not-set + int boundedRandomSize = randomIntBetween(-100000, -2); + String restContent = "{\"size\" : " + boundedRandomSize + "}"; try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> SearchSourceBuilder.fromXContent(parser)); - assertThat(ex.getMessage(), containsString(Integer.toString(randomSize))); + assertThat(ex.getMessage(), containsString(Integer.toString(boundedRandomSize))); } restContent = "{\"size\" : -1}"; From 24d2520ae48652fa76ae74e4562ac8a7b2147ace Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 12 Jul 2022 22:29:04 +1000 Subject: [PATCH 21/40] Audit API key ID when create or grant API keys (#88456) The API key ID generation is handled by the Request class since #63221. This makes it possible to audit it when creating or granting API keys. This PR makes the necessary changes for it to happen. Relates: #63221 --- docs/changelog/88456.yaml | 5 +++++ .../docs/en/security/auditing/event-types.asciidoc | 2 +- .../security/audit/logfile/LoggingAuditTrail.java | 1 + .../audit/logfile/LoggingAuditTrailTests.java | 13 ++++++++++--- 4 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/88456.yaml diff --git a/docs/changelog/88456.yaml b/docs/changelog/88456.yaml new file mode 100644 index 0000000000000..bb3a5d1182365 --- /dev/null +++ b/docs/changelog/88456.yaml @@ -0,0 +1,5 @@ +pr: 88456 +summary: Audit API key ID when create or grant API keys +area: Audit +type: enhancement +issues: [] diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index f3ca8a303725b..db4209ec60e9d 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -757,7 +757,7 @@ the <>. + [source,js] ---- -`{"name": , "expiration": , "role_descriptors" []}` +`{"id": , "name": , "expiration": , "role_descriptors" []}` ---- // NOTCONSOLE + diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 6ea823564db02..867dd495e82a0 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1234,6 +1234,7 @@ LogEntryBuilder withRequestBody(final UpdateApiKeyRequest updateApiKeyRequest) t private void withRequestBody(XContentBuilder builder, CreateApiKeyRequest createApiKeyRequest) throws IOException { TimeValue expiration = createApiKeyRequest.getExpiration(); builder.startObject("apikey") + .field("id", createApiKeyRequest.getId()) .field("name", createApiKeyRequest.getName()) .field("expiration", expiration != null ? expiration.toString() : null) .startArray("role_descriptors"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 540b5fa5c1507..f85b28bcbba98 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -590,8 +590,13 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException createApiKeyRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); auditTrail.accessGranted(requestId, authentication, CreateApiKeyAction.NAME, createApiKeyRequest, authorizationInfo); String expectedCreateKeyAuditEventString = """ - "create":{"apikey":{"name":"%s","expiration":%s,%s}}\ - """.formatted(keyName, expiration != null ? "\"" + expiration + "\"" : "null", roleDescriptorsStringBuilder); + "create":{"apikey":{"id":"%s","name":"%s","expiration":%s,%s}}\ + """.formatted( + createApiKeyRequest.getId(), + keyName, + expiration != null ? "\"" + expiration + "\"" : "null", + roleDescriptorsStringBuilder + ); List output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); String generatedCreateKeyAuditEventString = output.get(1); @@ -646,7 +651,9 @@ public void testSecurityConfigChangeEventFormattingForRoles() throws IOException output = CapturingLogger.output(logger.getName(), Level.INFO); assertThat(output.size(), is(2)); String generatedGrantKeyAuditEventString = output.get(1); - StringBuilder grantKeyAuditEventStringBuilder = new StringBuilder().append("\"create\":{\"apikey\":{\"name\":\"") + StringBuilder grantKeyAuditEventStringBuilder = new StringBuilder().append("\"create\":{\"apikey\":{\"id\":\"") + .append(grantApiKeyRequest.getApiKeyRequest().getId()) + .append("\",\"name\":\"") .append(keyName) .append("\",\"expiration\":") .append(expiration != null ? "\"" + expiration + "\"" : "null") From 3ed549b36cae886532a598adf5385054935ca616 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 12 Jul 2022 20:31:58 +0800 Subject: [PATCH 22/40] TSDB: RollupShardIndexer logging improvements (#88416) 1. Add trace log guards to avoid high cost method 2. Log the time it took to rollup a shard --- .../xpack/rollup/v2/RollupShardIndexer.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java index 056e96d51af09..a9498c785971f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/RollupShardIndexer.java @@ -127,6 +127,7 @@ class RollupShardIndexer { } public RollupIndexerAction.ShardRollupResponse execute() throws IOException { + long startTime = System.currentTimeMillis(); BulkProcessor bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { // TODO: add cancellations @@ -138,11 +139,12 @@ public RollupIndexerAction.ShardRollupResponse execute() throws IOException { } logger.info( - "Shard {} successfully sent [{}], indexed [{}], failed [{}]", + "Shard [{}] successfully sent [{}], indexed [{}], failed [{}], took [{}]", indexShard.shardId(), numSent.get(), numIndexed.get(), - numFailed.get() + numFailed.get(), + TimeValue.timeValueMillis(System.currentTimeMillis() - startTime) ); if (numIndexed.get() != numSent.get()) { @@ -239,13 +241,15 @@ public void collect(int docId, long owningBucketOrd) throws IOException { lastHistoTimestamp = rounding.round(timestamp); } - logger.trace( - "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", - docId, - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp), - timestampFormat.format(lastHistoTimestamp) - ); + if (logger.isTraceEnabled()) { + logger.trace( + "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> rollup bucket ts: [{}]", + docId, + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp), + timestampFormat.format(lastHistoTimestamp) + ); + } /* * Sanity checks to ensure that we receive documents in the correct order @@ -349,11 +353,14 @@ public RollupBucketBuilder init(BytesRef tsid, long timestamp) { this.timestamp = timestamp; this.docCount = 0; this.metricFieldProducers.values().stream().forEach(p -> p.reset()); - logger.trace( - "New bucket for _tsid: [{}], @timestamp: [{}]", - DocValueFormat.TIME_SERIES_ID.format(tsid), - timestampFormat.format(timestamp) - ); + if (logger.isTraceEnabled()) { + logger.trace( + "New bucket for _tsid: [{}], @timestamp: [{}]", + DocValueFormat.TIME_SERIES_ID.format(tsid), + timestampFormat.format(timestamp) + ); + } + return this; } From 4af02b8c80d8b8f9464e85c3c08ac0157b6c679e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 12 Jul 2022 14:42:45 +0200 Subject: [PATCH 23/40] Stop registering TestGeoShapeFieldMapperPlugin in ESIntegTestCase (#88460) Instead of registering the plugin by default, implementations that need it are responsible on registering the plugin. --- .../search/GeoBoundingBoxQueryLegacyGeoShapeIT.java | 5 ----- .../legacygeo/search/LegacyGeoShapeIT.java | 5 ----- .../percolator/PercolatorQuerySearchIT.java | 5 ----- .../indices/diskusage/IndexDiskUsageAnalyzerIT.java | 2 ++ .../search/geo/GeoBoundingBoxQueryGeoPointIT.java | 10 ++++++++++ .../search/geo/GeoBoundingBoxQueryGeoShapeIT.java | 10 ++++++++++ .../java/org/elasticsearch/search/geo/GeoShapeIT.java | 10 ++++++++++ .../org/elasticsearch/search/query/QueryStringIT.java | 9 +++++++++ .../search/query/SimpleQueryStringIT.java | 4 ++-- .../search/geo/GeoPointShapeQueryTests.java | 10 ++++++++++ .../elasticsearch/search/geo/GeoShapeQueryTests.java | 10 ++++++++++ .../search/geo/GeoPointShapeQueryTestCase.java | 9 --------- .../java/org/elasticsearch/test/ESIntegTestCase.java | 8 -------- .../integration/DocumentLevelSecurityTests.java | 6 ------ .../integration/FieldLevelSecurityTests.java | 6 ------ .../GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java | 5 ----- ...oBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java | 5 ----- .../search/GeoGridAggAndQueryConsistencyIT.java | 5 ----- .../xpack/spatial/search/GeoShapeWithDocValuesIT.java | 5 ----- .../spatial/search/LegacyGeoShapeWithDocValuesIT.java | 5 ----- 20 files changed, 63 insertions(+), 71 deletions(-) diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java index ee12e28c35616..b3c888af3327a 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java @@ -22,11 +22,6 @@ public class GeoBoundingBoxQueryLegacyGeoShapeIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class); diff --git a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java index 263d3840f7333..a92aba3cbee66 100644 --- a/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java +++ b/modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java @@ -28,11 +28,6 @@ public class LegacyGeoShapeIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class); diff --git a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 6f64e697e907e..315ff6492a23c 100644 --- a/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/internalClusterTest/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -62,11 +62,6 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Arrays.asList(PercolatorPlugin.class, TestGeoShapeFieldMapperPlugin.class); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java index 21677e420dd23..e7ba86ed5bfcd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentBuilder; @@ -57,6 +58,7 @@ protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); plugins.add(EngineTestPlugin.class); plugins.add(MockTransportService.TestPlugin.class); + plugins.add(TestGeoShapeFieldMapperPlugin.class); return plugins; } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java index 5d4b24d12b63f..f9909f9de5b02 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoPointIT.java @@ -9,14 +9,24 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoBoundingBoxQueryGeoPointIT extends GeoBoundingBoxQueryIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override public XContentBuilder getMapping() throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java index a81801c53c34b..403b890ea3dc6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoBoundingBoxQueryGeoShapeIT.java @@ -9,14 +9,24 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoBoundingBoxQueryGeoShapeIT extends GeoBoundingBoxQueryIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override public XContentBuilder getMapping() throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java index 5394232ace8fa..a12de847a62bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/geo/GeoShapeIT.java @@ -9,13 +9,23 @@ package org.elasticsearch.search.geo; import org.elasticsearch.Version; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; public class GeoShapeIT extends GeoShapeIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void getGeoShapeMapping(XContentBuilder b) throws IOException { b.field("type", "geo_shape"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java index 2090c4909870a..743cf268caf78 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/QueryStringIT.java @@ -13,14 +13,17 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.Operator; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -35,6 +38,12 @@ public class QueryStringIT extends ESIntegTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> nodePlugins() { + return List.of(TestGeoShapeFieldMapperPlugin.class); + } + @Before public void setup() throws Exception { String indexBody = copyToStringFromClasspath("/org/elasticsearch/search/query/all-query-index.json"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 387123189217c..7a4941d8454df 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -28,13 +28,13 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ public class SimpleQueryStringIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singletonList(MockAnalysisPlugin.class); + return List.of(MockAnalysisPlugin.class, TestGeoShapeFieldMapperPlugin.class); } public void testSimpleQueryString() throws ExecutionException, InterruptedException { diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java index 63c651107735b..b468e963a1f83 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPointShapeQueryTests.java @@ -15,10 +15,14 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -27,6 +31,12 @@ public class GeoPointShapeQueryTests extends GeoPointShapeQueryTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> getPlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void createMapping(String indexName, String fieldName, Settings settings) throws Exception { XContentBuilder xcb = XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java index f2e9bfcc28f8a..50f209b67790c 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java @@ -14,10 +14,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery; @@ -25,6 +29,12 @@ public class GeoShapeQueryTests extends GeoShapeQueryTestCase { + @SuppressWarnings("deprecation") + @Override + protected Collection> getPlugins() { + return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); + } + @Override protected void createMapping(String indexName, String fieldName, Settings settings) throws Exception { XContentBuilder xcb = XContentFactory.jsonBuilder() diff --git a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java index fe1b4c6e80602..d21a9f710c3d7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/geo/GeoPointShapeQueryTestCase.java @@ -35,15 +35,11 @@ import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -57,11 +53,6 @@ public abstract class GeoPointShapeQueryTestCase extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return Collections.singleton(TestGeoShapeFieldMapperPlugin.class); - } - protected abstract void createMapping(String indexName, String fieldName, Settings settings) throws Exception; protected void createMapping(String indexName, String fieldName) throws Exception { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 3e45394e70e24..7d52f28d32d6c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2049,11 +2049,6 @@ protected boolean addMockInternalEngine() { return true; } - /** Returns {@code true} iff this test cluster should use a dummy geo_shape field mapper */ - protected boolean addMockGeoShapeFieldMapper() { - return true; - } - /** * Returns a function that allows to wrap / filter all clients that are exposed by the test cluster. This is useful * for debugging or request / response pre and post processing. It also allows to intercept all calls done by the test @@ -2095,9 +2090,6 @@ protected Collection> getMockPlugins() { mocks.add(TestSeedPlugin.class); mocks.add(AssertActionNamePlugin.class); mocks.add(MockScriptService.TestPlugin.class); - if (addMockGeoShapeFieldMapper()) { - mocks.add(TestGeoShapeFieldMapperPlugin.class); - } return Collections.unmodifiableList(mocks); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index 7ddded23b73b1..5beffd910382f 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -119,12 +119,6 @@ protected Collection> nodePlugins() { ); } - @Override - protected boolean addMockGeoShapeFieldMapper() { - // a test requires the real SpatialPlugin because it utilizes the shape query - return false; - } - @Override protected String configUsers() { final String usersPasswdHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java index 8311ce48df312..0feb8f4b03add 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/FieldLevelSecurityTests.java @@ -104,12 +104,6 @@ protected Collection> nodePlugins() { ); } - @Override - protected boolean addMockGeoShapeFieldMapper() { - // a test requires the real SpatialPlugin because it utilizes the shape query - return false; - } - @Override protected String configUsers() { final String usersPasswHashed = new String(getFastStoredHashAlgoForTests().hash(USERS_PASSWD)); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java index 8206454e926b7..2b4400bb5a3fd 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryGeoShapeWithDocValuesIT.java @@ -21,11 +21,6 @@ public class GeoBoundingBoxQueryGeoShapeWithDocValuesIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java index 2073eac56218f..9b8814053c066 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT.java @@ -21,11 +21,6 @@ public class GeoBoundingBoxQueryLegacyGeoShapeWithDocValuesIT extends GeoBoundingBoxQueryIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java index 67b009ed9bf86..52497a0cc9b7d 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoGridAggAndQueryConsistencyIT.java @@ -51,11 +51,6 @@ public class GeoGridAggAndQueryConsistencyIT extends ESIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java index 16b4735cb55c1..faca74d83c6cc 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/GeoShapeWithDocValuesIT.java @@ -25,11 +25,6 @@ public class GeoShapeWithDocValuesIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java index 660fc7fd95bc1..f6fec0f64f75b 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/search/LegacyGeoShapeWithDocValuesIT.java @@ -29,11 +29,6 @@ public class LegacyGeoShapeWithDocValuesIT extends GeoShapeIntegTestCase { - @Override - protected boolean addMockGeoShapeFieldMapper() { - return false; - } - @Override protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); From 47ecd204029fafbc2ac57d0bf454595f8e5cef89 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Jul 2022 15:06:54 +0200 Subject: [PATCH 24/40] Use consistent shard map type in IndexService (#88465) It's faster and easier to reason about if we always have an immutable collections map here and not have the type depend on what the last operation on the index service was. --- .../java/org/elasticsearch/index/IndexService.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index e651b6b7a4da9..02b806406575c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -96,7 +96,6 @@ import java.util.function.Supplier; import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.core.Strings.format; public class IndexService extends AbstractIndexComponent implements IndicesClusterStateService.AllocatedIndex { @@ -118,7 +117,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private final SimilarityService similarityService; private final EngineFactory engineFactory; private final IndexWarmer warmer; - private volatile Map shards = emptyMap(); + private volatile Map shards = Map.of(); private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean deleted = new AtomicBoolean(false); private final IndexSettings indexSettings; @@ -523,16 +522,13 @@ public synchronized IndexShard createShard( @Override public synchronized void removeShard(int shardId, String reason) { - final ShardId sId = new ShardId(index(), shardId); - final IndexShard indexShard; - if (shards.containsKey(shardId) == false) { + final IndexShard indexShard = shards.get(shardId); + if (indexShard == null) { return; } logger.debug("[{}] closing... (reason: [{}])", shardId, reason); - HashMap newShards = new HashMap<>(shards); - indexShard = newShards.remove(shardId); - shards = unmodifiableMap(newShards); - closeShard(reason, sId, indexShard, indexShard.store(), indexShard.getIndexEventListener()); + shards = Maps.copyMapWithRemovedEntry(shards, shardId); + closeShard(reason, indexShard.shardId(), indexShard, indexShard.store(), indexShard.getIndexEventListener()); logger.debug("[{}] closed (reason: [{}])", shardId, reason); } From 67cacde18b0aa08846b158054342cbd126eea626 Mon Sep 17 00:00:00 2001 From: Sean Letendre <32531321+local-ghost-127@users.noreply.github.com> Date: Tue, 12 Jul 2022 09:19:58 -0400 Subject: [PATCH 25/40] Corrected an incomplete sentence. (#86542) * Corrected an incomplete sentence. * Update docs/reference/aggregations/metrics/avg-aggregation.asciidoc Co-authored-by: Christos Soulios <1561376+csoulios@users.noreply.github.com> Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Co-authored-by: Christos Soulios <1561376+csoulios@users.noreply.github.com> --- docs/reference/aggregations/metrics/avg-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc index 05d112a13ac03..4883eec3e617a 100644 --- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc @@ -4,7 +4,7 @@ Avg ++++ -A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents. +A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric or <> fields in the documents. Assuming the data consists of documents representing exams grades (between 0 and 100) of students we can average their scores with: From 28048a5dbe696b458fb225257f7d338a0ba3d770 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 12 Jul 2022 15:34:45 +0200 Subject: [PATCH 26/40] Updatable API keys - noop check (#88346) This PR adds a noop check for API key updates. If we detect a noop update, i.e., an update that does not result in any changes to the existing doc, we skip the index step and return updated = false in the response. This PR also extends test coverage around various corner cases. --- docs/changelog/88346.yaml | 5 + .../core/security/authc/RealmDomain.java | 5 + .../authc/AuthenticationTestHelper.java | 30 ++ .../xpack/security/apikey/ApiKeyRestIT.java | 24 +- .../security/authc/ApiKeyIntegTests.java | 307 +++++++++++++++--- .../xpack/security/authc/ApiKeyService.java | 256 ++++++++++----- .../security/authc/ApiKeyServiceTests.java | 195 ++++++----- .../rest-api-spec/test/api_key/30_update.yml | 2 +- 8 files changed, 623 insertions(+), 201 deletions(-) create mode 100644 docs/changelog/88346.yaml diff --git a/docs/changelog/88346.yaml b/docs/changelog/88346.yaml new file mode 100644 index 0000000000000..ca2537f28a5a9 --- /dev/null +++ b/docs/changelog/88346.yaml @@ -0,0 +1,5 @@ +pr: 88346 +summary: Updatable API keys - noop check +area: Security +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java index 8863953dc844d..53de14b5b68bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/RealmDomain.java @@ -14,6 +14,7 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.List; @@ -48,6 +49,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static RealmDomain fromXContent(final XContentParser parser) { + return REALM_DOMAIN_PARSER.apply(parser, null); + } + @Override public String toString() { return "RealmDomain{" + "name='" + name + '\'' + ", realms=" + realms + '}'; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java index 8cb8b684a64ab..1bbcec5d92ce8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java @@ -90,6 +90,36 @@ public static User randomUser() { ); } + public static User userWithRandomMetadataAndDetails(final String username, final String... roles) { + return new User( + username, + roles, + ESTestCase.randomFrom(ESTestCase.randomAlphaOfLengthBetween(1, 10), null), + // Not a very realistic email address, but we don't validate this nor rely on correct format, so keeping it simple + ESTestCase.randomFrom(ESTestCase.randomAlphaOfLengthBetween(1, 10), null), + randomUserMetadata(), + true + ); + } + + public static Map randomUserMetadata() { + return ESTestCase.randomFrom( + Map.of( + "employee_id", + ESTestCase.randomAlphaOfLength(5), + "number", + 1, + "numbers", + List.of(1, 3, 5), + "extra", + Map.of("favorite pizza", "hawaii", "age", 42) + ), + Map.of(ESTestCase.randomAlphaOfLengthBetween(3, 8), ESTestCase.randomAlphaOfLengthBetween(3, 8)), + Map.of(), + null + ); + } + public static RealmDomain randomDomain(boolean includeInternal) { final Supplier randomRealmTypeSupplier = randomRealmTypeSupplier(includeInternal); final Set domainRealms = new HashSet<>( diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java index c7f89106338cd..516d3b3d7a3a2 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java @@ -198,7 +198,7 @@ public void testGrantApiKeyWithOnlyManageOwnApiKeyPrivilegeFails() throws IOExce public void testUpdateApiKey() throws IOException { final var apiKeyName = "my-api-key-name"; - final Map apiKeyMetadata = Map.of("not", "returned"); + final Map apiKeyMetadata = Map.of("not", "returned"); final Map createApiKeyRequestBody = Map.of("name", apiKeyName, "metadata", apiKeyMetadata); final Request createApiKeyRequest = new Request("POST", "_security/api_key"); @@ -215,7 +215,7 @@ public void testUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded, apiKeyMetadata); } public void testGrantTargetCanUpdateApiKey() throws IOException { @@ -240,7 +240,7 @@ public void testGrantTargetCanUpdateApiKey() throws IOException { assertThat(apiKeyId, not(emptyString())); assertThat(apiKeyEncoded, not(emptyString())); - doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded); + doTestUpdateApiKey(apiKeyName, apiKeyId, apiKeyEncoded, null); } public void testGrantorCannotUpdateApiKeyOfGrantTarget() throws IOException { @@ -283,18 +283,26 @@ private void doTestAuthenticationWithApiKey(final String apiKeyName, final Strin assertThat(authenticate, hasEntry("api_key", Map.of("id", apiKeyId, "name", apiKeyName))); } - private void doTestUpdateApiKey(String apiKeyName, String apiKeyId, String apiKeyEncoded) throws IOException { + private void doTestUpdateApiKey( + final String apiKeyName, + final String apiKeyId, + final String apiKeyEncoded, + final Map oldMetadata + ) throws IOException { final var updateApiKeyRequest = new Request("PUT", "_security/api_key/" + apiKeyId); - final Map expectedApiKeyMetadata = Map.of("not", "returned (changed)", "foo", "bar"); - final Map updateApiKeyRequestBody = Map.of("metadata", expectedApiKeyMetadata); + final boolean updated = randomBoolean(); + final Map expectedApiKeyMetadata = updated ? Map.of("not", "returned (changed)", "foo", "bar") : oldMetadata; + final Map updateApiKeyRequestBody = expectedApiKeyMetadata == null + ? Map.of() + : Map.of("metadata", expectedApiKeyMetadata); updateApiKeyRequest.setJsonEntity(XContentTestUtils.convertToXContent(updateApiKeyRequestBody, XContentType.JSON).utf8ToString()); final Response updateApiKeyResponse = doUpdateUsingRandomAuthMethod(updateApiKeyRequest); assertOK(updateApiKeyResponse); final Map updateApiKeyResponseMap = responseAsMap(updateApiKeyResponse); - assertTrue((Boolean) updateApiKeyResponseMap.get("updated")); - expectMetadata(apiKeyId, expectedApiKeyMetadata); + assertEquals(updated, updateApiKeyResponseMap.get("updated")); + expectMetadata(apiKeyId, expectedApiKeyMetadata == null ? Map.of() : expectedApiKeyMetadata); // validate authentication still works after update doTestAuthenticationWithApiKey(apiKeyName, apiKeyId, apiKeyEncoded); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index 36d60d6e25fd3..a2624cc1fdca1 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -28,11 +28,13 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; @@ -43,6 +45,8 @@ import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.action.ClearSecurityCacheAction; @@ -137,6 +141,13 @@ public class ApiKeyIntegTests extends SecurityIntegTestCase { private static final long DELETE_INTERVAL_MILLIS = 100L; private static final int CRYPTO_THREAD_POOL_QUEUE_SIZE = 10; + private static final RoleDescriptor DEFAULT_API_KEY_ROLE_DESCRIPTOR = new RoleDescriptor( + "role", + new String[] { "monitor" }, + null, + null + ); + @Override public Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() @@ -1435,6 +1446,7 @@ public void testSecurityIndexStateChangeWillInvalidateApiKeyCaches() throws Exce public void testUpdateApiKey() throws ExecutionException, InterruptedException, IOException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); + final Map oldMetadata = createdApiKey.v2(); final var newRoleDescriptors = randomRoleDescriptors(); final boolean nullRoleDescriptors = newRoleDescriptors == null; // Role descriptor corresponding to SecuritySettingsSource.TEST_ROLE_YML @@ -1449,11 +1461,14 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, ); final var request = new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, ApiKeyTests.randomMetadata()); - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request); assertNotNull(response); - assertTrue(response.isUpdated()); + // In this test, non-null roleDescriptors always result in an update since they either update the role name, or associated + // privileges. As such null descriptors (plus matching or null metadata) is the only way we can get a noop here + final boolean metadataChanged = request.getMetadata() != null && false == request.getMetadata().equals(oldMetadata); + final boolean isUpdated = nullRoleDescriptors == false || metadataChanged; + assertEquals(isUpdated, response.isUpdated()); final PlainActionFuture getListener = new PlainActionFuture<>(); client().filterWithHeader( @@ -1475,31 +1490,37 @@ public void testUpdateApiKey() throws ExecutionException, InterruptedException, final var updatedApiKeyDoc = getApiKeyDocument(apiKeyId); expectMetadataForApiKey(expectedMetadata, updatedApiKeyDoc); expectRoleDescriptorsForApiKey("limited_by_role_descriptors", expectedLimitedByRoleDescriptors, updatedApiKeyDoc); - if (nullRoleDescriptors) { - // Default role descriptor assigned to api key in `createApiKey` - final var expectedRoleDescriptor = new RoleDescriptor("role", new String[] { "monitor" }, null, null); - expectRoleDescriptorsForApiKey("role_descriptors", List.of(expectedRoleDescriptor), updatedApiKeyDoc); - - // Create user action unauthorized because we did not update key role; it only has `monitor` cluster priv - final Map authorizationHeaders = Collections.singletonMap( - "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) - ); + final var expectedRoleDescriptors = nullRoleDescriptors ? List.of(DEFAULT_API_KEY_ROLE_DESCRIPTOR) : newRoleDescriptors; + expectRoleDescriptorsForApiKey("role_descriptors", expectedRoleDescriptors, updatedApiKeyDoc); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", TEST_USER_NAME); + expectedCreator.put("full_name", null); + expectedCreator.put("email", null); + expectedCreator.put("metadata", Map.of()); + expectedCreator.put("realm_type", "file"); + expectedCreator.put("realm", "file"); + expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc); + + // Check if update resulted in API key role going from `monitor` to `all` cluster privilege and assert that action that requires + // `all` is authorized or denied accordingly + final boolean hasAllClusterPrivilege = expectedRoleDescriptors.stream() + .filter(rd -> Arrays.asList(rd.getClusterPrivileges()).contains("all")) + .toList() + .isEmpty() == false; + final var authorizationHeaders = Collections.singletonMap( + "Authorization", + "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) + ); + if (hasAllClusterPrivilege) { + createUserWithRunAsRole(authorizationHeaders); + } else { ExecutionException e = expectThrows(ExecutionException.class, () -> createUserWithRunAsRole(authorizationHeaders)); assertThat(e.getMessage(), containsString("unauthorized")); assertThat(e.getCause(), instanceOf(ElasticsearchSecurityException.class)); - } else { - expectRoleDescriptorsForApiKey("role_descriptors", newRoleDescriptors, updatedApiKeyDoc); - // Create user action authorized because we updated key role to `all` cluster priv - final var authorizationHeaders = Collections.singletonMap( - "Authorization", - "ApiKey " + getBase64EncodedApiKeyValue(createdApiKey.v1().getId(), createdApiKey.v1().getKey()) - ); - createUserWithRunAsRole(authorizationHeaders); } } - public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, ExecutionException, InterruptedException { + public void testUpdateApiKeyAutoUpdatesUserFields() throws IOException, ExecutionException, InterruptedException { // Create separate native realm user and role for user role change test final var nativeRealmUser = randomAlphaOfLengthBetween(5, 10); final var nativeRealmRole = randomAlphaOfLengthBetween(5, 10); @@ -1536,13 +1557,39 @@ public void testUpdateApiKeyAutoUpdatesUserRoles() throws IOException, Execution newClusterPrivileges.toArray(new String[0]) ); - // Update API key - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), listener); + UpdateApiKeyResponse response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); assertNotNull(response); assertTrue(response.isUpdated()); expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorAfterUpdate), getApiKeyDocument(apiKeyId)); + + // Update user role name only + final RoleDescriptor roleDescriptorWithNewName = putRoleWithClusterPrivileges( + randomValueOtherThan(nativeRealmRole, () -> randomAlphaOfLength(10)), + // Keep old privileges + newClusterPrivileges.toArray(new String[0]) + ); + final User updatedUser = AuthenticationTestHelper.userWithRandomMetadataAndDetails( + nativeRealmUser, + roleDescriptorWithNewName.getName() + ); + updateUser(updatedUser); + + // Update API key + response = executeUpdateApiKey(nativeRealmUser, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); + + assertNotNull(response); + assertTrue(response.isUpdated()); + final Map updatedApiKeyDoc = getApiKeyDocument(apiKeyId); + expectRoleDescriptorsForApiKey("limited_by_role_descriptors", Set.of(roleDescriptorWithNewName), updatedApiKeyDoc); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", updatedUser.principal()); + expectedCreator.put("full_name", updatedUser.fullName()); + expectedCreator.put("email", updatedUser.email()); + expectedCreator.put("metadata", updatedUser.metadata()); + expectedCreator.put("realm_type", "native"); + expectedCreator.put("realm", "index"); + expectCreatorForApiKey(expectedCreator, updatedApiKeyDoc); } public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, InterruptedException { @@ -1552,8 +1599,7 @@ public void testUpdateApiKeyNotFoundScenarios() throws ExecutionException, Inter final var request = new UpdateApiKeyRequest(apiKeyId, List.of(expectedRoleDescriptor), ApiKeyTests.randomMetadata()); // Validate can update own API key - final PlainActionFuture listener = new PlainActionFuture<>(); - final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request, listener); + final UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, request); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1646,7 +1692,7 @@ public void testInvalidUpdateApiKeyScenarios() throws ExecutionException, Interr } } - public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionException, InterruptedException { + public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionException, InterruptedException, IOException { final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); final var apiKeyId = createdApiKey.v1().getId(); @@ -1679,6 +1725,147 @@ public void testUpdateApiKeyAccountsForSecurityDomains() throws ExecutionExcepti assertNotNull(response); assertTrue(response.isUpdated()); + final Map expectedCreator = new HashMap<>(); + expectedCreator.put("principal", TEST_USER_NAME); + expectedCreator.put("full_name", null); + expectedCreator.put("email", null); + expectedCreator.put("metadata", Map.of()); + expectedCreator.put("realm_type", authenticatingRealm.getType()); + expectedCreator.put("realm", authenticatingRealm.getName()); + final XContentBuilder builder = realmDomain.toXContent(XContentFactory.jsonBuilder(), null); + expectedCreator.put("realm_domain", XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2()); + expectCreatorForApiKey(expectedCreator, getApiKeyDocument(apiKeyId)); + } + + public void testNoopUpdateApiKey() throws ExecutionException, InterruptedException, IOException { + final Tuple> createdApiKey = createApiKey(TEST_USER_NAME, null); + final var apiKeyId = createdApiKey.v1().getId(); + + final var initialRequest = new UpdateApiKeyRequest( + apiKeyId, + List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)), + ApiKeyTests.randomMetadata() + ); + UpdateApiKeyResponse response = executeUpdateApiKey(TEST_USER_NAME, initialRequest); + assertNotNull(response); + // First update is not noop, because role descriptors changed and possibly metadata + assertTrue(response.isUpdated()); + + // Update with same request is a noop and does not clear cache + authenticateWithApiKey(apiKeyId, createdApiKey.v1().getKey()); + final var serviceWithNameForDoc1 = Arrays.stream(internalCluster().getNodeNames()) + .map(n -> internalCluster().getInstance(ApiKeyService.class, n)) + .filter(s -> s.getDocCache().get(apiKeyId) != null) + .findFirst() + .orElseThrow(); + final int count = serviceWithNameForDoc1.getDocCache().count(); + response = executeUpdateApiKey(TEST_USER_NAME, initialRequest); + assertNotNull(response); + assertFalse(response.isUpdated()); + assertEquals(count, serviceWithNameForDoc1.getDocCache().count()); + + // Update with empty request is a noop + response = executeUpdateApiKey(TEST_USER_NAME, UpdateApiKeyRequest.usingApiKeyId(apiKeyId)); + assertNotNull(response); + assertFalse(response.isUpdated()); + + // Update with different role descriptors is not a noop + final List newRoleDescriptors = List.of( + randomValueOtherThanMany( + rd -> (RoleDescriptorRequestValidator.validate(rd) != null) && initialRequest.getRoleDescriptors().contains(rd) == false, + () -> RoleDescriptorTests.randomRoleDescriptor(false) + ), + randomValueOtherThanMany( + rd -> (RoleDescriptorRequestValidator.validate(rd) != null) && initialRequest.getRoleDescriptors().contains(rd) == false, + () -> RoleDescriptorTests.randomRoleDescriptor(false) + ) + ); + response = executeUpdateApiKey(TEST_USER_NAME, new UpdateApiKeyRequest(apiKeyId, newRoleDescriptors, null)); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // Update with re-ordered role descriptors is a noop + response = executeUpdateApiKey( + TEST_USER_NAME, + new UpdateApiKeyRequest(apiKeyId, List.of(newRoleDescriptors.get(1), newRoleDescriptors.get(0)), null) + ); + assertNotNull(response); + assertFalse(response.isUpdated()); + + // Update with different metadata is not a noop + response = executeUpdateApiKey( + TEST_USER_NAME, + new UpdateApiKeyRequest( + apiKeyId, + null, + randomValueOtherThanMany(md -> md == null || md.equals(initialRequest.getMetadata()), ApiKeyTests::randomMetadata) + ) + ); + assertNotNull(response); + assertTrue(response.isUpdated()); + + // Update with different creator info is not a noop + // First, ensure that the user role descriptors alone do *not* cause an update, so we can test that we correctly perform the noop + // check when we update creator info + final ServiceWithNodeName serviceWithNodeName = getServiceWithNodeName(); + PlainActionFuture listener = new PlainActionFuture<>(); + // Role descriptor corresponding to SecuritySettingsSource.TEST_ROLE_YML, i.e., should not result in update + final Set oldUserRoleDescriptors = Set.of( + new RoleDescriptor( + TEST_ROLE, + new String[] { "ALL" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").allowRestrictedIndices(true).privileges("ALL").build() }, + null + ) + ); + serviceWithNodeName.service() + .updateApiKey( + Authentication.newRealmAuthentication( + new User(TEST_USER_NAME, TEST_ROLE), + new Authentication.RealmRef("file", "file", serviceWithNodeName.nodeName()) + ), + UpdateApiKeyRequest.usingApiKeyId(apiKeyId), + oldUserRoleDescriptors, + listener + ); + response = listener.get(); + assertNotNull(response); + assertFalse(response.isUpdated()); + final User updatedUser = AuthenticationTestHelper.userWithRandomMetadataAndDetails(TEST_USER_NAME, TEST_ROLE); + final RealmConfig.RealmIdentifier creatorRealmOnCreatedApiKey = new RealmConfig.RealmIdentifier(FileRealmSettings.TYPE, "file"); + final boolean noUserChanges = updatedUser.equals(new User(TEST_USER_NAME, TEST_ROLE)); + final Authentication.RealmRef realmRef; + if (randomBoolean() || noUserChanges) { + final RealmConfig.RealmIdentifier otherRealmInDomain = AuthenticationTestHelper.randomRealmIdentifier(true); + final var realmDomain = new RealmDomain( + ESTestCase.randomAlphaOfLengthBetween(3, 8), + Set.of(creatorRealmOnCreatedApiKey, otherRealmInDomain) + ); + // Using other realm from domain should result in update + realmRef = new Authentication.RealmRef( + otherRealmInDomain.getName(), + otherRealmInDomain.getType(), + serviceWithNodeName.nodeName(), + realmDomain + ); + } else { + realmRef = new Authentication.RealmRef( + creatorRealmOnCreatedApiKey.getName(), + creatorRealmOnCreatedApiKey.getType(), + serviceWithNodeName.nodeName() + ); + } + final var authentication = randomValueOtherThanMany( + Authentication::isApiKey, + () -> AuthenticationTestHelper.builder().user(updatedUser).realmRef(realmRef).build() + ); + listener = new PlainActionFuture<>(); + serviceWithNodeName.service() + .updateApiKey(authentication, UpdateApiKeyRequest.usingApiKeyId(apiKeyId), oldUserRoleDescriptors, listener); + response = listener.get(); + assertNotNull(response); + assertTrue(response.isUpdated()); } public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, ExecutionException, InterruptedException { @@ -1720,7 +1907,12 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution final Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) ); - client.execute(UpdateApiKeyAction.INSTANCE, new UpdateApiKeyRequest(apiKey1.v1(), List.of(), null), listener); + client.execute( + UpdateApiKeyAction.INSTANCE, + // Set metadata to ensure update + new UpdateApiKeyRequest(apiKey1.v1(), List.of(), Map.of(randomAlphaOfLength(5), randomAlphaOfLength(10))), + listener + ); final var response = listener.get(); assertNotNull(response); assertTrue(response.isUpdated()); @@ -1741,7 +1933,7 @@ public void testUpdateApiKeyClearsApiKeyDocCache() throws IOException, Execution } private List randomRoleDescriptors() { - int caseNo = randomIntBetween(0, 2); + int caseNo = randomIntBetween(0, 3); return switch (caseNo) { case 0 -> List.of(new RoleDescriptor(randomAlphaOfLength(10), new String[] { "all" }, null, null)); case 1 -> List.of( @@ -1752,6 +1944,15 @@ private List randomRoleDescriptors() { ) ); case 2 -> null; + // vary default role descriptor assigned to created API keys by name only + case 3 -> List.of( + new RoleDescriptor( + randomValueOtherThan(DEFAULT_API_KEY_ROLE_DESCRIPTOR.getName(), () -> randomAlphaOfLength(10)), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges(), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getIndicesPrivileges(), + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getRunAs() + ) + ); default -> throw new IllegalStateException("unexpected case no"); }; } @@ -1774,6 +1975,13 @@ private void expectMetadataForApiKey(final Map expectedMetadata, assertThat("for api key doc " + actualRawApiKeyDoc, actualMetadata, equalTo(expectedMetadata)); } + private void expectCreatorForApiKey(final Map expectedCreator, final Map actualRawApiKeyDoc) { + assertNotNull(actualRawApiKeyDoc); + @SuppressWarnings("unchecked") + final var actualCreator = (Map) actualRawApiKeyDoc.get("creator"); + assertThat("for api key doc " + actualRawApiKeyDoc, actualCreator, equalTo(expectedCreator)); + } + @SuppressWarnings("unchecked") private void expectRoleDescriptorsForApiKey( final String roleDescriptorType, @@ -1940,12 +2148,17 @@ private void verifyGetResponse( } private Tuple> createApiKey(String user, TimeValue expiration) { - final Tuple, List>> res = createApiKeys(user, 1, expiration, "monitor"); + final Tuple, List>> res = createApiKeys( + user, + 1, + expiration, + DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges() + ); return new Tuple<>(res.v1().get(0), res.v2().get(0)); } private Tuple, List>> createApiKeys(int noOfApiKeys, TimeValue expiration) { - return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, "monitor"); + return createApiKeys(ES_TEST_ROOT_USER, noOfApiKeys, expiration, DEFAULT_API_KEY_ROLE_DESCRIPTOR.getClusterPrivileges()); } private Tuple, List>> createApiKeys( @@ -1996,7 +2209,7 @@ private Tuple, List>> createApiKe List> metadatas = new ArrayList<>(noOfApiKeys); List responses = new ArrayList<>(); for (int i = 0; i < noOfApiKeys; i++) { - final RoleDescriptor descriptor = new RoleDescriptor("role", clusterPrivileges, null, null); + final RoleDescriptor descriptor = new RoleDescriptor(DEFAULT_API_KEY_ROLE_DESCRIPTOR.getName(), clusterPrivileges, null, null); Client client = client().filterWithHeader(headers); final Map metadata = ApiKeyTests.randomMetadata(); metadatas.add(metadata); @@ -2041,13 +2254,27 @@ private void createNativeRealmUser( assertTrue(putUserResponse.created()); } + private void updateUser(User user) throws ExecutionException, InterruptedException { + final PutUserRequest putUserRequest = new PutUserRequest(); + putUserRequest.username(user.principal()); + putUserRequest.roles(user.roles()); + putUserRequest.metadata(user.metadata()); + putUserRequest.fullName(user.fullName()); + putUserRequest.email(user.email()); + final PlainActionFuture listener = new PlainActionFuture<>(); + final Client client = client().filterWithHeader( + Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING)) + ); + client.execute(PutUserAction.INSTANCE, putUserRequest, listener); + final PutUserResponse putUserResponse = listener.get(); + assertFalse(putUserResponse.created()); + } + private RoleDescriptor putRoleWithClusterPrivileges(final String nativeRealmRoleName, String... clusterPrivileges) throws InterruptedException, ExecutionException { final PutRoleRequest putRoleRequest = new PutRoleRequest(); putRoleRequest.name(nativeRealmRoleName); - for (final String clusterPrivilege : clusterPrivileges) { - putRoleRequest.cluster(clusterPrivilege); - } + putRoleRequest.cluster(clusterPrivileges); final PlainActionFuture roleListener = new PlainActionFuture<>(); client().filterWithHeader(Map.of("Authorization", basicAuthHeaderValue(ES_TEST_ROOT_USER, TEST_PASSWORD_SECURE_STRING))) .execute(PutRoleAction.INSTANCE, putRoleRequest, roleListener); @@ -2066,11 +2293,9 @@ private Client getClientForRunAsUser() { ); } - private UpdateApiKeyResponse executeUpdateApiKey( - final String username, - final UpdateApiKeyRequest request, - final PlainActionFuture listener - ) throws InterruptedException, ExecutionException { + private UpdateApiKeyResponse executeUpdateApiKey(final String username, final UpdateApiKeyRequest request) throws InterruptedException, + ExecutionException { + final var listener = new PlainActionFuture(); final Client client = client().filterWithHeader( Collections.singletonMap("Authorization", basicAuthHeaderValue(username, TEST_PASSWORD_SECURE_STRING)) ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index fe30462e9b8d6..883ad3ca98c19 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -119,6 +119,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -279,27 +280,27 @@ public void invalidateAll() { * Asynchronously creates a new API key based off of the request and authentication * @param authentication the authentication that this api key should be based off of * @param request the request to create the api key included any permission restrictions - * @param userRoles the user's actual roles that we always enforce + * @param userRoleDescriptors the user's actual roles that we always enforce * @param listener the listener that will be used to notify of completion */ public void createApiKey( Authentication authentication, CreateApiKeyRequest request, - Set userRoles, + Set userRoleDescriptors, ActionListener listener ) { ensureEnabled(); if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); } else { - createApiKeyAndIndexIt(authentication, request, userRoles, listener); + createApiKeyAndIndexIt(authentication, request, userRoleDescriptors, listener); } } private void createApiKeyAndIndexIt( Authentication authentication, CreateApiKeyRequest request, - Set roleDescriptorSet, + Set userRoleDescriptors, ActionListener listener ) { final Instant created = clock.instant(); @@ -313,7 +314,7 @@ private void createApiKeyAndIndexIt( apiKeyHashChars, request.getName(), authentication, - roleDescriptorSet, + userRoleDescriptors, created, expiration, request.getRoleDescriptors(), @@ -358,7 +359,7 @@ private void createApiKeyAndIndexIt( public void updateApiKey( final Authentication authentication, final UpdateApiKeyRequest request, - final Set userRoles, + final Set userRoleDescriptors, final ActionListener listener ) { ensureEnabled(); @@ -386,10 +387,7 @@ public void updateApiKey( validateCurrentApiKeyDocForUpdate(apiKeyId, authentication, versionedDoc.doc()); - executeBulkRequest( - buildBulkRequestForUpdate(versionedDoc, authentication, request, userRoles), - ActionListener.wrap(bulkResponse -> translateResponseAndClearCache(apiKeyId, bulkResponse, listener), listener::onFailure) - ); + doUpdateApiKey(authentication, request, userRoleDescriptors, versionedDoc, listener); }, listener::onFailure)); } @@ -418,10 +416,10 @@ static XContentBuilder newDocument( char[] apiKeyHashChars, String name, Authentication authentication, - Set userRoles, + Set userRoleDescriptors, Instant created, Instant expiration, - List keyRoles, + List keyRoleDescriptors, Version version, @Nullable Map metadata ) throws IOException { @@ -433,8 +431,8 @@ static XContentBuilder newDocument( .field("api_key_invalidated", false); addApiKeyHash(builder, apiKeyHashChars); - addRoleDescriptors(builder, keyRoles); - addLimitedByRoleDescriptors(builder, userRoles); + addRoleDescriptors(builder, keyRoleDescriptors); + addLimitedByRoleDescriptors(builder, userRoleDescriptors); builder.field("name", name).field("version", version.id).field("metadata_flattened", metadata); addCreator(builder, authentication); @@ -442,14 +440,22 @@ static XContentBuilder newDocument( return builder.endObject(); } - static XContentBuilder buildUpdatedDocument( + // package private for testing + + /** + * @return `null` if the update is a noop, i.e., if no changes to `currentApiKeyDoc` are required + */ + XContentBuilder maybeBuildUpdatedDocument( final ApiKeyDoc currentApiKeyDoc, + final Version targetDocVersion, final Authentication authentication, - final Set userRoles, - final List keyRoles, - final Version version, - final Map metadata + final UpdateApiKeyRequest request, + final Set userRoleDescriptors ) throws IOException { + if (isNoop(currentApiKeyDoc, targetDocVersion, authentication, request, userRoleDescriptors)) { + return null; + } + final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject() .field("doc_type", "api_key") @@ -459,6 +465,7 @@ static XContentBuilder buildUpdatedDocument( addApiKeyHash(builder, currentApiKeyDoc.hash.toCharArray()); + final List keyRoles = request.getRoleDescriptors(); if (keyRoles != null) { logger.trace(() -> format("Building API key doc with updated role descriptors [{}]", keyRoles)); addRoleDescriptors(builder, keyRoles); @@ -467,14 +474,15 @@ static XContentBuilder buildUpdatedDocument( builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } - addLimitedByRoleDescriptors(builder, userRoles); + addLimitedByRoleDescriptors(builder, userRoleDescriptors); - builder.field("name", currentApiKeyDoc.name).field("version", version.id); + builder.field("name", currentApiKeyDoc.name).field("version", targetDocVersion.id); assert currentApiKeyDoc.metadataFlattened == null || MetadataUtils.containsReservedMetadata( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() ) == false : "API key doc to be updated contains reserved metadata"; + final Map metadata = request.getMetadata(); if (metadata != null) { logger.trace(() -> format("Building API key doc with updated metadata [{}]", metadata)); builder.field("metadata_flattened", metadata); @@ -493,6 +501,90 @@ static XContentBuilder buildUpdatedDocument( return builder.endObject(); } + private boolean isNoop( + final ApiKeyDoc apiKeyDoc, + final Version targetDocVersion, + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoleDescriptors + ) { + if (apiKeyDoc.version != targetDocVersion.id) { + return false; + } + + final Map currentCreator = apiKeyDoc.creator; + final var user = authentication.getEffectiveSubject().getUser(); + final var sourceRealm = authentication.getEffectiveSubject().getRealm(); + if (false == (Objects.equals(user.principal(), currentCreator.get("principal")) + && Objects.equals(user.fullName(), currentCreator.get("full_name")) + && Objects.equals(user.email(), currentCreator.get("email")) + && Objects.equals(user.metadata(), currentCreator.get("metadata")) + && Objects.equals(sourceRealm.getName(), currentCreator.get("realm")) + && Objects.equals(sourceRealm.getType(), currentCreator.get("realm_type")))) { + return false; + } + if (sourceRealm.getDomain() != null) { + if (currentCreator.get("realm_domain") == null) { + return false; + } + @SuppressWarnings("unchecked") + final var currentRealmDomain = RealmDomain.fromXContent( + XContentHelper.mapToXContentParser( + XContentParserConfiguration.EMPTY, + (Map) currentCreator.get("realm_domain") + ) + ); + if (sourceRealm.getDomain().equals(currentRealmDomain) == false) { + return false; + } + } else { + if (currentCreator.get("realm_domain") != null) { + return false; + } + } + + final Map newMetadata = request.getMetadata(); + if (newMetadata != null) { + if (apiKeyDoc.metadataFlattened == null) { + return false; + } + final Map currentMetadata = XContentHelper.convertToMap(apiKeyDoc.metadataFlattened, false, XContentType.JSON) + .v2(); + if (newMetadata.equals(currentMetadata) == false) { + return false; + } + } + + final List newRoleDescriptors = request.getRoleDescriptors(); + if (newRoleDescriptors != null) { + final List currentRoleDescriptors = parseRoleDescriptorsBytes( + request.getId(), + apiKeyDoc.roleDescriptorsBytes, + RoleReference.ApiKeyRoleType.ASSIGNED + ); + if (false == (newRoleDescriptors.size() == currentRoleDescriptors.size() + && Set.copyOf(newRoleDescriptors).containsAll(new HashSet<>(currentRoleDescriptors)))) { + return false; + } + } + + assert userRoleDescriptors != null; + // There is an edge case here when we update an 7.x API key that has a `LEGACY_SUPERUSER_ROLE_DESCRIPTOR` role descriptor: + // `parseRoleDescriptorsBytes` automatically transforms it to `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR`. As such, when we + // perform the noop check on `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR` we will treat it as a noop even though the actual + // role descriptor bytes on the API key are different, and correspond to `LEGACY_SUPERUSER_ROLE_DESCRIPTOR`. + // + // This does *not* present a functional issue, since whenever a `LEGACY_SUPERUSER_ROLE_DESCRIPTOR` is loaded at authentication time, + // it is likewise automatically transformed to `ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR`. + final List currentLimitedByRoleDescriptors = parseRoleDescriptorsBytes( + request.getId(), + apiKeyDoc.limitedByRoleDescriptorsBytes, + RoleReference.ApiKeyRoleType.LIMITED_BY + ); + return (userRoleDescriptors.size() == currentLimitedByRoleDescriptors.size() + && userRoleDescriptors.containsAll(currentLimitedByRoleDescriptors)); + } + void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener> listener) { if (false == isEnabled()) { listener.onResponse(AuthenticationResult.notHandled()); @@ -971,6 +1063,68 @@ public void logRemovedField(String parserName, Supplier locati } } + private void doUpdateApiKey( + final Authentication authentication, + final UpdateApiKeyRequest request, + final Set userRoleDescriptors, + final VersionedApiKeyDoc currentVersionedDoc, + final ActionListener listener + ) throws IOException { + logger.trace( + "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", + request.getId(), + currentVersionedDoc.seqNo(), + currentVersionedDoc.primaryTerm() + ); + final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); + final var currentDocVersion = Version.fromId(currentVersionedDoc.doc().version); + assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; + if (currentDocVersion.before(targetDocVersion)) { + logger.debug( + "API key update for [{}] will update version from [{}] to [{}]", + request.getId(), + currentDocVersion, + targetDocVersion + ); + } + + final XContentBuilder builder = maybeBuildUpdatedDocument( + currentVersionedDoc.doc(), + targetDocVersion, + authentication, + request, + userRoleDescriptors + ); + final boolean isNoop = builder == null; + if (isNoop) { + logger.debug("Detected noop update request for API key [{}]. Skipping index request.", request.getId()); + listener.onResponse(new UpdateApiKeyResponse(false)); + return; + } + + final IndexRequest indexRequest = client.prepareIndex(SECURITY_MAIN_ALIAS) + .setId(request.getId()) + .setSource(builder) + .setIfSeqNo(currentVersionedDoc.seqNo()) + .setIfPrimaryTerm(currentVersionedDoc.primaryTerm()) + .setOpType(DocWriteRequest.OpType.INDEX) + .request(); + logger.trace("Executing index request to update API key [{}]", request.getId()); + securityIndex.prepareIndexIfNeededThenExecute( + listener::onFailure, + () -> executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + SECURITY_ORIGIN, + client.prepareBulk().add(indexRequest).setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).request(), + ActionListener.wrap( + bulkResponse -> translateResponseAndClearCache(request.getId(), bulkResponse, listener), + listener::onFailure + ), + client::bulk + ) + ); + } + /** * Invalidate API keys for given realm, user name, API key name and id. * @param realmNames realm names @@ -1235,63 +1389,11 @@ private static VersionedApiKeyDoc singleDoc(final String apiKeyId, final Collect return elements.iterator().next(); } - private BulkRequest buildBulkRequestForUpdate( - final VersionedApiKeyDoc versionedDoc, - final Authentication authentication, - final UpdateApiKeyRequest request, - final Set userRoles - ) throws IOException { - logger.trace( - "Building update request for API key doc [{}] with seqNo [{}] and primaryTerm [{}]", - request.getId(), - versionedDoc.seqNo(), - versionedDoc.primaryTerm() - ); - final var currentDocVersion = Version.fromId(versionedDoc.doc().version); - final var targetDocVersion = clusterService.state().nodes().getMinNodeVersion(); - assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; - if (currentDocVersion.before(targetDocVersion)) { - logger.debug( - "API key update for [{}] will update version from [{}] to [{}]", - request.getId(), - currentDocVersion, - targetDocVersion - ); - } - final var bulkRequestBuilder = client.prepareBulk(); - bulkRequestBuilder.add( - client.prepareIndex(SECURITY_MAIN_ALIAS) - .setId(request.getId()) - .setSource( - buildUpdatedDocument( - versionedDoc.doc(), - authentication, - userRoles, - request.getRoleDescriptors(), - targetDocVersion, - request.getMetadata() - ) - ) - .setIfSeqNo(versionedDoc.seqNo()) - .setIfPrimaryTerm(versionedDoc.primaryTerm()) - .setOpType(DocWriteRequest.OpType.INDEX) - .request() - ); - bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); - return bulkRequestBuilder.request(); - } - - private void executeBulkRequest(final BulkRequest bulkRequest, final ActionListener listener) { - securityIndex.prepareIndexIfNeededThenExecute( - listener::onFailure, - () -> executeAsyncWithOrigin(client.threadPool().getThreadContext(), SECURITY_ORIGIN, bulkRequest, listener, client::bulk) - ); - } - - private static void addLimitedByRoleDescriptors(final XContentBuilder builder, final Set userRoles) throws IOException { - assert userRoles != null; + private static void addLimitedByRoleDescriptors(final XContentBuilder builder, final Set limitedByRoleDescriptors) + throws IOException { + assert limitedByRoleDescriptors != null; builder.startObject("limited_by_role_descriptors"); - for (RoleDescriptor descriptor : userRoles) { + for (RoleDescriptor descriptor : limitedByRoleDescriptors) { builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true)); } builder.endObject(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 181dcf8211283..ae4fa08bc0806 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -68,12 +69,14 @@ import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.UpdateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationTests; +import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; @@ -1671,91 +1674,135 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); } - public void testBuildUpdatedDocument() throws IOException { + public void testMaybeBuildUpdatedDocument() throws IOException { final var apiKey = randomAlphaOfLength(16); final var hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - - final var oldApiKeyDoc = buildApiKeyDoc(hash, randomBoolean() ? -1 : Instant.now().toEpochMilli(), false); - - final Set newUserRoles = randomBoolean() ? Set.of() : Set.of(RoleDescriptorTests.randomRoleDescriptor()); - - final boolean nullKeyRoles = randomBoolean(); - final List newKeyRoles; - if (nullKeyRoles) { - newKeyRoles = null; - } else { - newKeyRoles = List.of(RoleDescriptorTests.randomRoleDescriptor()); - } - - final var metadata = ApiKeyTests.randomMetadata(); - final var version = Version.CURRENT; - final var authentication = randomValueOtherThanMany( + final var oldAuthentication = randomValueOtherThanMany( Authentication::isApiKey, - () -> AuthenticationTestHelper.builder().user(new User("user", "role")).build(false) - ); - - final var keyDocSource = ApiKeyService.buildUpdatedDocument( - oldApiKeyDoc, - authentication, - newUserRoles, - newKeyRoles, - version, - metadata + () -> AuthenticationTestHelper.builder() + .user(AuthenticationTestHelper.userWithRandomMetadataAndDetails("user", "role")) + .build(false) ); - final var updatedApiKeyDoc = ApiKeyDoc.fromXContent( - XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(keyDocSource), XContentType.JSON) + final Set oldUserRoles = randomSet(0, 3, RoleDescriptorTests::randomRoleDescriptor); + final List oldKeyRoles = randomList(3, RoleDescriptorTests::randomRoleDescriptor); + final Map oldMetadata = ApiKeyTests.randomMetadata(); + final Version oldVersion = VersionUtils.randomVersion(random()); + final ApiKeyDoc oldApiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser( + XContentParserConfiguration.EMPTY, + BytesReference.bytes( + ApiKeyService.newDocument( + hash, + randomAlphaOfLength(10), + oldAuthentication, + oldUserRoles, + Instant.now(), + randomBoolean() ? null : Instant.now(), + oldKeyRoles, + oldVersion, + oldMetadata + ) + ), + XContentType.JSON + ) ); - assertEquals(oldApiKeyDoc.docType, updatedApiKeyDoc.docType); - assertEquals(oldApiKeyDoc.name, updatedApiKeyDoc.name); - assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); - assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); - assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); - assertEquals(oldApiKeyDoc.invalidated, updatedApiKeyDoc.invalidated); - - final var service = createApiKeyService(Settings.EMPTY); - final var actualUserRoles = service.parseRoleDescriptorsBytes( - "", - updatedApiKeyDoc.limitedByRoleDescriptorsBytes, - RoleReference.ApiKeyRoleType.LIMITED_BY - ); - assertEquals(newUserRoles.size(), actualUserRoles.size()); - assertEquals(new HashSet<>(newUserRoles), new HashSet<>(actualUserRoles)); + final boolean changeUserRoles = randomBoolean(); + final boolean changeKeyRoles = randomBoolean(); + final boolean changeMetadata = randomBoolean(); + final boolean changeVersion = randomBoolean(); + final boolean changeCreator = randomBoolean(); + final Set newUserRoles = changeUserRoles + ? randomValueOtherThan(oldUserRoles, () -> randomSet(0, 3, RoleDescriptorTests::randomRoleDescriptor)) + : oldUserRoles; + final List newKeyRoles = changeKeyRoles + ? randomValueOtherThan(oldKeyRoles, () -> randomList(0, 3, RoleDescriptorTests::randomRoleDescriptor)) + : (randomBoolean() ? oldKeyRoles : null); + final Map newMetadata = changeMetadata + ? randomValueOtherThanMany(md -> md == null || md.equals(oldMetadata), ApiKeyTests::randomMetadata) + : (randomBoolean() ? oldMetadata : null); + final Version newVersion = changeVersion + ? randomValueOtherThan(oldVersion, () -> VersionUtils.randomVersion(random())) + : oldVersion; + final Authentication newAuthentication = changeCreator + ? randomValueOtherThanMany( + (auth -> auth.isApiKey() || auth.getEffectiveSubject().getUser().equals(oldAuthentication.getEffectiveSubject().getUser())), + () -> AuthenticationTestHelper.builder() + .user(AuthenticationTestHelper.userWithRandomMetadataAndDetails("user", "role")) + .build(false) + ) + : oldAuthentication; + final var request = new UpdateApiKeyRequest(randomAlphaOfLength(10), newKeyRoles, newMetadata); + final var service = createApiKeyService(); - final var actualKeyRoles = service.parseRoleDescriptorsBytes( - "", - updatedApiKeyDoc.roleDescriptorsBytes, - RoleReference.ApiKeyRoleType.ASSIGNED + final XContentBuilder builder = service.maybeBuildUpdatedDocument( + oldApiKeyDoc, + newVersion, + newAuthentication, + request, + newUserRoles ); - if (nullKeyRoles) { - assertEquals( - service.parseRoleDescriptorsBytes("", oldApiKeyDoc.roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED), - actualKeyRoles - ); - } else { - assertEquals(newKeyRoles.size(), actualKeyRoles.size()); - assertEquals(new HashSet<>(newKeyRoles), new HashSet<>(actualKeyRoles)); - } - if (metadata == null) { - assertEquals(oldApiKeyDoc.metadataFlattened, updatedApiKeyDoc.metadataFlattened); - } else { - assertEquals(metadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); - } - assertEquals(authentication.getEffectiveSubject().getUser().principal(), updatedApiKeyDoc.creator.getOrDefault("principal", null)); - assertEquals(authentication.getEffectiveSubject().getUser().fullName(), updatedApiKeyDoc.creator.getOrDefault("fullName", null)); - assertEquals(authentication.getEffectiveSubject().getUser().email(), updatedApiKeyDoc.creator.getOrDefault("email", null)); - assertEquals(authentication.getEffectiveSubject().getUser().metadata(), updatedApiKeyDoc.creator.getOrDefault("metadata", null)); - RealmRef realm = authentication.getEffectiveSubject().getRealm(); - assertEquals(realm.getName(), updatedApiKeyDoc.creator.getOrDefault("realm", null)); - assertEquals(realm.getType(), updatedApiKeyDoc.creator.getOrDefault("realm_type", null)); - if (realm.getDomain() != null) { - @SuppressWarnings("unchecked") - final var actualDomain = (Map) updatedApiKeyDoc.creator.getOrDefault("realm_domain", null); - assertEquals(realm.getDomain().name(), actualDomain.get("name")); + final boolean noop = (changeCreator || changeMetadata || changeKeyRoles || changeUserRoles || changeVersion) == false; + if (noop) { + assertNull(builder); } else { - assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); + final ApiKeyDoc updatedApiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser(XContentParserConfiguration.EMPTY, BytesReference.bytes(builder), XContentType.JSON) + ); + assertEquals(oldApiKeyDoc.docType, updatedApiKeyDoc.docType); + assertEquals(oldApiKeyDoc.name, updatedApiKeyDoc.name); + assertEquals(oldApiKeyDoc.hash, updatedApiKeyDoc.hash); + assertEquals(oldApiKeyDoc.expirationTime, updatedApiKeyDoc.expirationTime); + assertEquals(oldApiKeyDoc.creationTime, updatedApiKeyDoc.creationTime); + assertEquals(oldApiKeyDoc.invalidated, updatedApiKeyDoc.invalidated); + assertEquals(newVersion.id, updatedApiKeyDoc.version); + final var actualUserRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.limitedByRoleDescriptorsBytes, + RoleReference.ApiKeyRoleType.LIMITED_BY + ); + assertEquals(newUserRoles.size(), actualUserRoles.size()); + assertEquals(new HashSet<>(newUserRoles), new HashSet<>(actualUserRoles)); + final var actualKeyRoles = service.parseRoleDescriptorsBytes( + "", + updatedApiKeyDoc.roleDescriptorsBytes, + RoleReference.ApiKeyRoleType.ASSIGNED + ); + if (changeKeyRoles == false) { + assertEquals( + service.parseRoleDescriptorsBytes("", oldApiKeyDoc.roleDescriptorsBytes, RoleReference.ApiKeyRoleType.ASSIGNED), + actualKeyRoles + ); + } else { + assertEquals(newKeyRoles.size(), actualKeyRoles.size()); + assertEquals(new HashSet<>(newKeyRoles), new HashSet<>(actualKeyRoles)); + } + if (changeMetadata == false) { + assertEquals(oldApiKeyDoc.metadataFlattened, updatedApiKeyDoc.metadataFlattened); + } else { + assertEquals(newMetadata, XContentHelper.convertToMap(updatedApiKeyDoc.metadataFlattened, true, XContentType.JSON).v2()); + } + assertEquals(newAuthentication.getEffectiveSubject().getUser().principal(), updatedApiKeyDoc.creator.get("principal")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().fullName(), updatedApiKeyDoc.creator.get("full_name")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().email(), updatedApiKeyDoc.creator.get("email")); + assertEquals(newAuthentication.getEffectiveSubject().getUser().metadata(), updatedApiKeyDoc.creator.get("metadata")); + final RealmRef realm = newAuthentication.getEffectiveSubject().getRealm(); + assertEquals(realm.getName(), updatedApiKeyDoc.creator.get("realm")); + assertEquals(realm.getType(), updatedApiKeyDoc.creator.get("realm_type")); + if (realm.getDomain() != null) { + @SuppressWarnings("unchecked") + final var actualRealmDomain = RealmDomain.fromXContent( + XContentHelper.mapToXContentParser( + XContentParserConfiguration.EMPTY, + (Map) updatedApiKeyDoc.creator.get("realm_domain") + ) + ); + assertEquals(realm.getDomain(), actualRealmDomain); + } else { + assertFalse(updatedApiKeyDoc.creator.containsKey("realm_domain")); + } } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml index 013d28113521b..73ff3fba19b46 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/api_key/30_update.yml @@ -223,7 +223,7 @@ teardown: Authorization: "Basic YXBpX2tleV91c2VyXzE6eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" # api_key_user_1 security.update_api_key: id: "$user1_key_id" - - match: { updated: true } + - match: { updated: false } # Check metadata did not change - do: From bb950414a3a5f0a0b8119586d82cd687a46f0c0a Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 12 Jul 2022 08:38:43 -0500 Subject: [PATCH 27/40] unmute test (#88454) --- .../rest-api-spec/test/security/authz/14_cat_indices.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml index 09d0d416e54da..31fb9e0810b90 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/14_cat_indices.yml @@ -120,8 +120,6 @@ teardown: "Test empty request while single authorized closed index": - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/47875" features: ["allowed_warnings"] - do: From 1e079154a46d37bb337b76c6130928cca86b374d Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Tue, 12 Jul 2022 16:52:32 +0300 Subject: [PATCH 28/40] INFO logging of snapshot restore and completion (#88257) But DEBUG (silent) logging of snapshot restore/completion when done in the context of CCR or searchable snapshots. --- docs/changelog/88257.yaml | 6 ++ .../cluster/ClusterStateDiffIT.java | 1 + .../snapshots/RestoreSnapshotIT.java | 62 +++++++++++++++++++ .../restore/RestoreSnapshotRequest.java | 34 ++++++++++ .../cluster/RestoreInProgress.java | 41 +++++++++++- .../snapshots/RestoreService.java | 14 ++++- .../restore/RestoreSnapshotRequestTests.java | 2 + .../MetadataIndexStateServiceTests.java | 1 + .../allocation/ThrottlingAllocationTests.java | 1 + .../decider/DiskThresholdDeciderTests.java | 2 +- ...storeInProgressAllocationDeciderTests.java | 1 + .../ClusterSerializationTests.java | 1 + .../InternalSnapshotsInfoServiceTests.java | 1 + .../xpack/ccr/CcrRepositoryIT.java | 18 ++++-- .../xpack/ccr/CcrRetentionLeaseIT.java | 3 +- .../ccr/action/TransportPutFollowAction.java | 3 +- ...ransportMountSearchableSnapshotAction.java | 4 +- 17 files changed, 180 insertions(+), 15 deletions(-) create mode 100644 docs/changelog/88257.yaml diff --git a/docs/changelog/88257.yaml b/docs/changelog/88257.yaml new file mode 100644 index 0000000000000..feb5df85feafb --- /dev/null +++ b/docs/changelog/88257.yaml @@ -0,0 +1,6 @@ +pr: 88257 +summary: INFO logging of snapshot restore and completion +area: Snapshot/Restore +type: enhancement +issues: + - 86610 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 5b995482f6fa7..b669cbf11bffc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -727,6 +727,7 @@ public ClusterState.Custom randomCreate(String name) { UUIDs.randomBase64UUID(), new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())), RestoreInProgress.State.fromValue((byte) randomIntBetween(0, 3)), + randomBoolean(), emptyList(), ImmutableOpenMap.of() ) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java index c9c0c4a6cd60c..23c43a16308bd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RestoreSnapshotIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.xcontent.XContentFactory; import java.nio.file.Path; @@ -158,6 +159,67 @@ public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception { assertThat(client.prepareGet(restoredIndexName2, sameSourceIndex ? docId : docId2).get().isExists(), equalTo(true)); } + @TestLogging( + reason = "testing the logging of the start and completion of a snapshot restore", + value = "org.elasticsearch.snapshots.RestoreService:INFO" + ) + public void testRestoreLogging() throws IllegalAccessException { + final MockLogAppender mockLogAppender = new MockLogAppender(); + try { + String indexName = "testindex"; + String repoName = "test-restore-snapshot-repo"; + String snapshotName = "test-restore-snapshot"; + Path absolutePath = randomRepoPath().toAbsolutePath(); + logger.info("Path [{}]", absolutePath); + String restoredIndexName = indexName + "-restored"; + String expectedValue = "expected"; + + mockLogAppender.start(); + Loggers.addAppender(LogManager.getLogger(RestoreService.class), mockLogAppender); + + mockLogAppender.addExpectation( + new MockLogAppender.PatternSeenEventExpectation( + "not seen start of snapshot restore", + "org.elasticsearch.snapshots.RestoreService", + Level.INFO, + "started restore of snapshot \\[.*" + snapshotName + ".*\\] for indices \\[.*" + indexName + ".*\\]" + ) + ); + + mockLogAppender.addExpectation( + new MockLogAppender.PatternSeenEventExpectation( + "not seen completion of snapshot restore", + "org.elasticsearch.snapshots.RestoreService", + Level.INFO, + "completed restore of snapshot \\[.*" + snapshotName + ".*\\]" + ) + ); + + Client client = client(); + // Write a document + String docId = Integer.toString(randomInt()); + indexDoc(indexName, docId, "value", expectedValue); + createRepository(repoName, "fs", absolutePath); + createSnapshot(repoName, snapshotName, Collections.singletonList(indexName)); + + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(repoName, snapshotName) + .setWaitForCompletion(false) + .setRenamePattern(indexName) + .setRenameReplacement(restoredIndexName) + .get(); + + assertThat(restoreSnapshotResponse.status(), equalTo(RestStatus.ACCEPTED)); + ensureGreen(restoredIndexName); + assertThat(client.prepareGet(restoredIndexName, docId).get().isExists(), equalTo(true)); + mockLogAppender.assertAllExpectationsMatched(); + } finally { + Loggers.removeAppender(LogManager.getLogger(RestoreService.class), mockLogAppender); + mockLogAppender.stop(); + } + } + public void testRestoreIncreasesPrimaryTerms() { final String indexName = randomAlphaOfLengthBetween(5, 10).toLowerCase(Locale.ROOT); createIndex(indexName, indexSettingsNoReplicas(2).build()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java index 8eb5f6d48ecc2..c383b2d610e5f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.restore; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -48,6 +49,8 @@ public class RestoreSnapshotRequest extends MasterNodeRequest indices, Map shards) { + public record Entry( + String uuid, + Snapshot snapshot, + State state, + boolean quiet, + List indices, + Map shards + ) { /** * Creates new restore metadata * * @param uuid uuid of the restore * @param snapshot snapshot * @param state current state of the restore process + * @param quiet {@code true} if logging of the start and completion of the snapshot restore should be at {@code DEBUG} log + * level, else it should be at {@code INFO} log level * @param indices list of indices being restored * @param shards map of shards being restored to their current restore status */ - public Entry(String uuid, Snapshot snapshot, State state, List indices, Map shards) { + public Entry( + String uuid, + Snapshot snapshot, + State state, + boolean quiet, + List indices, + Map shards + ) { this.snapshot = Objects.requireNonNull(snapshot); this.state = Objects.requireNonNull(state); + this.quiet = Objects.requireNonNull(quiet); this.indices = Objects.requireNonNull(indices); if (shards == null) { this.shards = Map.of(); @@ -342,10 +360,24 @@ public RestoreInProgress(StreamInput in) throws IOException { uuid = in.readString(); Snapshot snapshot = new Snapshot(in); State state = State.fromValue(in.readByte()); + boolean quiet; + if (in.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) { + quiet = in.readBoolean(); + } else { + // Backwards compatibility: previously there was no logging of the start or completion of a snapshot restore + quiet = true; + } List indices = in.readImmutableList(StreamInput::readString); entriesBuilder.put( uuid, - new Entry(uuid, snapshot, state, indices, in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus)) + new Entry( + uuid, + snapshot, + state, + quiet, + indices, + in.readImmutableMap(ShardId::new, ShardRestoreStatus::readShardRestoreStatus) + ) ); } this.entries = Collections.unmodifiableMap(entriesBuilder); @@ -357,6 +389,9 @@ public void writeTo(StreamOutput out) throws IOException { o.writeString(entry.uuid); entry.snapshot().writeTo(o); o.writeByte(entry.state().value()); + if (out.getVersion().onOrAfter(RestoreSnapshotRequest.VERSION_SUPPORTING_QUIET_PARAMETER)) { + o.writeBoolean(entry.quiet()); + } o.writeStringCollection(entry.indices); o.writeMap(entry.shards); }); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 6a2f49f99a2d7..512fe1766133f 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -238,7 +238,6 @@ public void restoreSnapshot( final BiConsumer updater ) { try { - // Try and fill in any missing repository UUIDs in case they're needed during the restore final StepListener repositoryUuidRefreshListener = new StepListener<>(); refreshRepositoryUuids(refreshRepositoryUuidOnRestore, repositoriesService, repositoryUuidRefreshListener); @@ -737,6 +736,7 @@ public static RestoreInProgress updateRestoreStateWithDeletedIndices(RestoreInPr entry.uuid(), entry.snapshot(), overallState(RestoreInProgress.State.STARTED, shards), + entry.quiet(), entry.indices(), shards ) @@ -873,7 +873,9 @@ public RestoreInProgress applyChanges(final RestoreInProgress oldRestore) { Map shards = Map.copyOf(shardsBuilder); RestoreInProgress.State newState = overallState(RestoreInProgress.State.STARTED, shards); - builder.add(new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.indices(), shards)); + builder.add( + new RestoreInProgress.Entry(entry.uuid(), entry.snapshot(), newState, entry.quiet(), entry.indices(), shards) + ); } else { builder.add(entry); } @@ -1051,6 +1053,7 @@ public ClusterState execute(ClusterState currentState) { boolean changed = false; for (RestoreInProgress.Entry entry : currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY)) { if (entry.state().completed()) { + logger.log(entry.quiet() ? Level.DEBUG : Level.INFO, "completed restore of snapshot [{}]", entry.snapshot()); changed = true; } else { restoreInProgressBuilder.add(entry); @@ -1373,6 +1376,7 @@ && isSystemIndex(snapshotIndexMetadata) == false) { restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), + request.quiet(), List.copyOf(indicesToRestore.keySet()), Map.copyOf(shards) ) @@ -1569,6 +1573,12 @@ public void onFailure(Exception e) { @Override public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + logger.log( + request.quiet() ? Level.DEBUG : Level.INFO, + "started restore of snapshot [{}] for indices {}", + snapshot, + snapshotInfo.indices() + ); listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java index ae730ed4d4a64..922e7e03c7600 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequestTests.java @@ -64,6 +64,7 @@ private RestoreSnapshotRequest randomState(RestoreSnapshotRequest instance) { } instance.partial(randomBoolean()); instance.includeAliases(randomBoolean()); + instance.quiet(randomBoolean()); if (randomBoolean()) { Map indexSettings = new HashMap<>(); @@ -127,6 +128,7 @@ protected RestoreSnapshotRequest mutateInstance(RestoreSnapshotRequest instance) public void testSource() throws IOException { RestoreSnapshotRequest original = createTestInstance(); original.snapshotUuid(null); // cannot be set via the REST API + original.quiet(false); // cannot be set via the REST API XContentBuilder builder = original.toXContent(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Collections.emptyMap())); XContentParser parser = XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, null, BytesReference.bytes(builder).streamInput()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java index 43cad5e994326..21f41912e9514 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java @@ -343,6 +343,7 @@ private static ClusterState addRestoredIndex(final String index, final int numSh "_uuid", snapshot, RestoreInProgress.State.INIT, + false, Collections.singletonList(index), shardsBuilder ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index e616b81bebc8e..4077c878d3c4e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -413,6 +413,7 @@ private ClusterState createRecoveryStateAndInitializeAllocations( restoreUUID, snapshot, RestoreInProgress.State.INIT, + false, new ArrayList<>(snapshotIndices), restoreShards ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 582521729fdea..8793f6f9c63e5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -1212,7 +1212,7 @@ public void testDiskThresholdWithSnapshotShardSizes() { Map shards = Map.of(shardId, new RestoreInProgress.ShardRestoreStatus("node1")); final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add( - new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, List.of("test"), shards) + new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, false, List.of("test"), shards) ); ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName())) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java index 6fda78125b8ed..2562d367cb6f1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java @@ -150,6 +150,7 @@ public void testCanAllocatePrimaryExistingInRestoreInProgress() { recoverySource.restoreUUID(), snapshot, restoreState, + false, singletonList("test"), shards ); diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 1bffc1931d840..f57627649706d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -137,6 +137,7 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception { UUIDs.randomBase64UUID(), new Snapshot("repo2", new SnapshotId("snap2", UUIDs.randomBase64UUID())), RestoreInProgress.State.STARTED, + false, Collections.singletonList("index_name"), Map.of() ) diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index 45d9f37bd054b..72fb1f17a4b1e 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -451,6 +451,7 @@ private ClusterState addUnassignedShards(final ClusterState currentState, String recoverySource.restoreUUID(), recoverySource.snapshot(), RestoreInProgress.State.INIT, + false, Collections.singletonList(indexName), shards ) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 55e0e5a614962..65192908c28ed 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -158,7 +158,8 @@ public void testThatRepositoryRecoversEmptyIndexBasedOnLeaderSettings() throws I .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(TimeValue.MAX_VALUE) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -230,7 +231,8 @@ public void testDocsAreRecovered() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -298,7 +300,8 @@ public void testRateLimitingIsEmployed() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(TimeValue.MAX_VALUE) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -364,7 +367,8 @@ public void testIndividualActionsTimeout() throws Exception { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); try { final RestoreService restoreService = getFollowerCluster().getCurrentMasterNodeInstance(RestoreService.class); @@ -427,7 +431,8 @@ public void testFollowerMappingIsUpdated() throws IOException { .renamePattern("^(.*)$") .renameReplacement(followerIndex) .masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) - .indexSettings(settingsBuilder); + .indexSettings(settingsBuilder) + .quiet(true); List transportServices = new ArrayList<>(); CountDownLatch latch = new CountDownLatch(1); @@ -586,7 +591,8 @@ public void testCcrRepositoryFetchesSnapshotShardSizeFromIndexShardStoreStats() Settings.builder() .put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, followerIndex) .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true) - ); + ) + .quiet(true); restoreService.restoreSnapshot(restoreRequest, PlainActionFuture.newFuture()); waitForRestoreInProgress.get(30L, TimeUnit.SECONDS); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java index a3cd5750f0d39..8d8fc8120c36b 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java @@ -157,7 +157,8 @@ private RestoreSnapshotRequest setUpRestoreSnapshotRequest( .indicesOptions(indicesOptions) .renamePattern("^(.*)$") .renameReplacement(followerIndex) - .masterNodeTimeout(TimeValue.MAX_VALUE); + .masterNodeTimeout(TimeValue.MAX_VALUE) + .quiet(true); } public void testRetentionLeaseIsTakenAtTheStartOfRecovery() throws Exception { diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index a22ac0abc9e03..88301c49c2101 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -194,7 +194,8 @@ private void createFollowerIndex( .renamePattern("^(.*)$") .renameReplacement(request.getFollowerIndex()) .masterNodeTimeout(request.masterNodeTimeout()) - .indexSettings(overrideSettings); + .indexSettings(overrideSettings) + .quiet(true); final Client clientWithHeaders = CcrLicenseChecker.wrapClient( this.client, diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index 1b8bf8c991efc..ae476d6f8539c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -272,7 +272,9 @@ protected void masterOperation( // Pass through the master-node timeout .masterNodeTimeout(request.masterNodeTimeout()) // Fail the restore if the snapshot found above is swapped out from under us before the restore happens - .snapshotUuid(snapshotId.getUUID()), + .snapshotUuid(snapshotId.getUUID()) + // Log snapshot restore at the DEBUG log level + .quiet(true), listener ); }, listener::onFailure), threadPool.executor(ThreadPool.Names.SNAPSHOT_META), null); From 84af49337850922a76ef1bde270f47086ac5e8d1 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Jul 2022 16:08:05 +0200 Subject: [PATCH 29/40] Fix ReactiveStorageDeciderServiceTests testNodeSizeForDataBelowLowWatermark (#88452) Fix this test unexpectedly being off by one by increasing the accuracy of the fp division (better to have a larger dividend and divisor) a little. I could easily reproduce the failure without the fix but with it, the test cases we use at least run accurate with the change. closes #88433 --- .../autoscaling/storage/ReactiveStorageDeciderService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index c168e7b3b08f0..476c6b21df034 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -225,7 +225,7 @@ static long nodeSizeForDataBelowLowWatermark(long bytes, DiskThresholdSettings t } else { double percentThreshold = thresholdSettings.getFreeDiskThresholdLow(); if (percentThreshold >= 0.0 && percentThreshold < 100.0) { - return (long) (bytes / ((100.0 - percentThreshold) / 100)); + return (long) (100 * bytes / (100 - percentThreshold)); } else { return bytes; } From ba46bd4ad8e7e87ce0a7b1847cb1b898dfcf9ffb Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 12 Jul 2022 16:40:54 +0200 Subject: [PATCH 30/40] Avoid noisy exceptions on data nodes when aborting snapshots (#88476) Currently, an abort (especially when triggered an index delete) can manifest as either an aborted snapshot exception, a missing index exception or an NPE. The latter two show up as noise in logs. This change catches effectively all of these cleanly as aborted snapshot exceptions so they don't get logged as warnings and avoids the NPE if a shard was removed from the index service concurrently by using the API that throws on missing shards to look it up. --- .../org/elasticsearch/snapshots/SnapshotShardsService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index c2ab4e808f1dc..2850dbacf73aa 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -337,7 +337,10 @@ private void snapshot( ActionListener listener ) { try { - final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()); + if (snapshotStatus.isAborted()) { + throw new AbortedSnapshotException(); + } + final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); if (indexShard.routingEntry().primary() == false) { throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary"); } From b790256862822a2a435d956971b0030f207f4a45 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Tue, 12 Jul 2022 11:26:40 -0400 Subject: [PATCH 31/40] Track the count of failed invocations since last successful policy snapshot (#88398) Add tracking for the number of invocations that have passed between a successful SLM snapshot and the most recent failure. These stats would be helpful for reporting on SLM policy health. --- docs/changelog/88398.yaml | 5 +++ .../slm/SnapshotLifecyclePolicyMetadata.java | 35 ++++++++++++++++--- .../SnapshotLifecyclePolicyMetadataTests.java | 7 +++- .../xpack/slm/SnapshotLifecycleTask.java | 2 ++ 4 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/88398.yaml diff --git a/docs/changelog/88398.yaml b/docs/changelog/88398.yaml new file mode 100644 index 0000000000000..e694b560e9407 --- /dev/null +++ b/docs/changelog/88398.yaml @@ -0,0 +1,5 @@ +pr: 88398 +summary: Track the count of failed invocations since last successful policy snapshot +area: ILM+SLM +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java index d42d7f62d6a1c..59c65da8cfea0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadata.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.slm; +import org.elasticsearch.Version; import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -40,6 +41,7 @@ public class SnapshotLifecyclePolicyMetadata implements SimpleDiffable PARSER = new ConstructingObjectParser<>( @@ -66,6 +69,7 @@ public class SnapshotLifecyclePolicyMetadata implements SimpleDiffable getHeaders() { @@ -165,6 +177,10 @@ public SnapshotInvocationRecord getLastFailure() { return lastFailure; } + public long getInvocationsSinceLastSuccess() { + return invocationsSinceLastSuccess; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -178,13 +194,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (Objects.nonNull(lastFailure)) { builder.field(LAST_FAILURE.getPreferredName(), lastFailure); } + builder.field(INVOCATIONS_SINCE_LAST_SUCCESS.getPreferredName(), invocationsSinceLastSuccess); builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(policy, headers, version, modifiedDate, lastSuccess, lastFailure); + return Objects.hash(policy, headers, version, modifiedDate, lastSuccess, lastFailure, invocationsSinceLastSuccess); } @Override @@ -201,7 +218,8 @@ public boolean equals(Object obj) { && Objects.equals(version, other.version) && Objects.equals(modifiedDate, other.modifiedDate) && Objects.equals(lastSuccess, other.lastSuccess) - && Objects.equals(lastFailure, other.lastFailure); + && Objects.equals(lastFailure, other.lastFailure) + && Objects.equals(invocationsSinceLastSuccess, other.invocationsSinceLastSuccess); } @Override @@ -222,6 +240,7 @@ private Builder() {} private Long modifiedDate; private SnapshotInvocationRecord lastSuccessDate; private SnapshotInvocationRecord lastFailureDate; + private long invocationsSinceLastSuccess = 0L; public Builder setPolicy(SnapshotLifecyclePolicy policy) { this.policy = policy; @@ -253,6 +272,11 @@ public Builder setLastFailure(SnapshotInvocationRecord lastFailure) { return this; } + public Builder setInvocationsSinceLastSuccess(long invocationsSinceLastSuccess) { + this.invocationsSinceLastSuccess = invocationsSinceLastSuccess; + return this; + } + public SnapshotLifecyclePolicyMetadata build() { return new SnapshotLifecyclePolicyMetadata( Objects.requireNonNull(policy), @@ -260,7 +284,8 @@ public SnapshotLifecyclePolicyMetadata build() { version, Objects.requireNonNull(modifiedDate, "modifiedDate must be set"), lastSuccessDate, - lastFailureDate + lastFailureDate, + invocationsSinceLastSuccess ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java index 59c703c5418c0..58ed29e30d79d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/SnapshotLifecyclePolicyMetadataTests.java @@ -84,11 +84,16 @@ public static SnapshotLifecyclePolicyMetadata createRandomPolicyMetadata(String if (randomBoolean()) { builder.setHeaders(randomHeaders()); } - if (randomBoolean()) { + boolean hasSuccess = randomBoolean(); + if (hasSuccess) { builder.setLastSuccess(randomSnapshotInvocationRecord()); + builder.setInvocationsSinceLastSuccess(0L); } if (randomBoolean()) { builder.setLastFailure(randomSnapshotInvocationRecord()); + if (hasSuccess) { + builder.setInvocationsSinceLastSuccess(randomLongBetween(1, Long.MAX_VALUE)); + } } return builder.build(); } diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java index 1f7f5faf45d74..bd06e35a5b217 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java @@ -276,9 +276,11 @@ public ClusterState execute(ClusterState currentState) throws Exception { exception.map(SnapshotLifecycleTask::exceptionToString).orElse(null) ) ); + newPolicyMetadata.setInvocationsSinceLastSuccess(policyMetadata.getInvocationsSinceLastSuccess() + 1L); } else { stats.snapshotTaken(policyName); newPolicyMetadata.setLastSuccess(new SnapshotInvocationRecord(snapshotName, snapshotStartTime, snapshotFinishTime, null)); + newPolicyMetadata.setInvocationsSinceLastSuccess(0L); } snapLifecycles.put(policyName, newPolicyMetadata.build()); From c636e439e836ea6e6092f4cb2b61fbb27bf4910c Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Tue, 12 Jul 2022 17:34:23 +0200 Subject: [PATCH 32/40] Mute ReactiveStorageIT#testScaleDuringSplitOrClone (#88480) Relates to https://github.com/elastic/elasticsearch/issues/88478. --- .../xpack/autoscaling/storage/ReactiveStorageIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index cdc532f6f47f7..5d785e4751b6e 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -356,6 +356,7 @@ public void testScaleWhileShrinking() throws Exception { ensureGreen(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/88478") public void testScaleDuringSplitOrClone() throws Exception { internalCluster().startMasterOnlyNode(); final String dataNode1Name = internalCluster().startDataOnlyNode(); From 08c54f13294ac74fe8bb9823907fd9ba60e6e46c Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Tue, 12 Jul 2022 20:41:35 +0200 Subject: [PATCH 33/40] Enable TRACE Logging for test and increase timeout (#88477) --- .../gateway/GatewayMetaStatePersistedStateTests.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index dc53c12f3bcda..31242ff3084aa 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.node.Node; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -378,6 +379,7 @@ public void testStatePersistedOnLoad() throws IOException { } } + @TestIssueLogging(value = "org.elasticsearch.gateway:TRACE", issueUrl = "https://github.com/elastic/elasticsearch/issues/87952") public void testDataOnlyNodePersistence() throws Exception { final List cleanup = new ArrayList<>(2); @@ -434,7 +436,7 @@ public void testDataOnlyNodePersistence() throws Exception { ); persistedState.setCurrentTerm(state.term()); persistedState.setLastAcceptedState(state); - assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten())); + assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()), 30, TimeUnit.SECONDS); assertThat( persistedState.getLastAcceptedState().getLastAcceptedConfiguration(), @@ -452,7 +454,7 @@ public void testDataOnlyNodePersistence() throws Exception { ); persistedState.markLastAcceptedStateAsCommitted(); - assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten())); + assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()), 30, TimeUnit.SECONDS); CoordinationMetadata expectedCoordinationMetadata = CoordinationMetadata.builder(coordinationMetadata) .lastCommittedConfiguration(coordinationMetadata.getLastAcceptedConfiguration()) @@ -506,7 +508,7 @@ public void testDataOnlyNodePersistence() throws Exception { assertTrue(wroteState); // must write it at least once assertEquals(currentTerm, persistedState.getCurrentTerm()); assertClusterStateEqual(state, persistedState.getLastAcceptedState()); - assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten())); + assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()), 30, TimeUnit.SECONDS); gateway.close(); assertTrue(cleanup.remove(gateway)); From c271d391f855da27b678b5ebd09404956b68c0fc Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 12 Jul 2022 12:46:04 -0700 Subject: [PATCH 34/40] Move runtime fields base scripts out of scripting fields api package. (#88488) Two runtime fields script classes were placed into the package for the scripting fields api. These have been moved to the script package where the rest of the runtime fields script classes live. This is a purely mechanical change. --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/GeoPointFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 2 +- .../{field => }/SortedNumericDocValuesLongFieldScript.java | 4 ++-- .../{field => }/SortedSetDocValuesStringFieldScript.java | 3 +-- .../field/SortedNumericDocValuesLongFieldScriptTests.java | 1 + .../field/SortedSetDocValuesStringFieldScriptTests.java | 1 + 7 files changed, 8 insertions(+), 7 deletions(-) rename server/src/main/java/org/elasticsearch/script/{field => }/SortedNumericDocValuesLongFieldScript.java (94%) rename server/src/main/java/org/elasticsearch/script/{field => }/SortedSetDocValuesStringFieldScript.java (95%) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index a7cadc7e40a8b..666a6032aa048 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -42,9 +42,9 @@ import org.elasticsearch.script.DateFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; +import org.elasticsearch.script.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.script.field.DateMillisDocValuesField; import org.elasticsearch.script.field.DateNanosDocValuesField; -import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.script.field.ToScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.FieldValues; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index d34a7edf67e22..4e639b6db9bae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -34,8 +34,8 @@ import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; +import org.elasticsearch.script.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.script.field.GeoPointDocValuesField; -import org.elasticsearch.script.field.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index dff6d4b84baf3..f4c3b4df0a05d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -52,9 +52,9 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptCompiler; +import org.elasticsearch.script.SortedSetDocValuesStringFieldScript; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.field.KeywordDocValuesField; -import org.elasticsearch.script.field.SortedSetDocValuesStringFieldScript; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; diff --git a/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java b/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java similarity index 94% rename from server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java rename to server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java index 4b911c7d52a77..9d4e3f5f79fc6 100644 --- a/server/src/main/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/SortedNumericDocValuesLongFieldScript.java @@ -6,11 +6,11 @@ * Side Public License, v 1. */ -package org.elasticsearch.script.field; +package org.elasticsearch.script; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.AbstractLongFieldScript; +import org.elasticsearch.script.field.LongDocValuesField; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; diff --git a/server/src/main/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScript.java b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java similarity index 95% rename from server/src/main/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScript.java rename to server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java index 72313aebde15a..c39d887922e94 100644 --- a/server/src/main/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/SortedSetDocValuesStringFieldScript.java @@ -6,13 +6,12 @@ * Side Public License, v 1. */ -package org.elasticsearch.script.field; +package org.elasticsearch.script; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java index 0cc14dcfa23c4..a44b1aba76de3 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.script.AbstractFieldScript; +import org.elasticsearch.script.SortedNumericDocValuesLongFieldScript; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java index 09b86887c54dc..002ad85ddbdb8 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java @@ -17,6 +17,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.script.AbstractFieldScript; +import org.elasticsearch.script.SortedSetDocValuesStringFieldScript; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESTestCase; From 39de085cb085f116b2ec40961de727deaa4777e8 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 12 Jul 2022 16:01:16 -0500 Subject: [PATCH 35/40] Ingest: Start separating Metadata from IngestSourceAndMetadata (#88401) Pull out the implementation of `Metadata` from `IngestSourceAndMetadata`. `Metadata` will become a base class extended by the update contexts: ingest, update, update by query and reindex. `Metadata` implements a map-like interface, making it easy for a class containing `Metadata` to implement the full `Map` interface. --- .../common/DotExpanderProcessorTests.java | 4 +- .../ingest/common/RenameProcessorTests.java | 6 +- .../elasticsearch/ingest/IngestClientIT.java | 2 +- .../ingest/WriteableIngestDocument.java | 9 +- .../elasticsearch/ingest/IngestDocument.java | 24 +- .../elasticsearch/ingest/IngestService.java | 22 +- .../ingest/IngestSourceAndMetadata.java | 387 +++-------------- .../org/elasticsearch/script/Metadata.java | 400 +++++++++++++++++- .../ingest/SimulateExecutionServiceTests.java | 2 +- .../SimulatePipelineRequestParsingTests.java | 33 +- .../ingest/WriteableIngestDocumentTests.java | 13 +- .../ingest/IngestServiceTests.java | 2 +- .../ingest/IngestSourceAndMetadataTests.java | 151 +++---- .../elasticsearch/script/MetadataTests.java | 72 ++++ .../ingest/TestIngestDocument.java | 21 +- .../elasticsearch/script/TestMetadata.java | 27 ++ .../results/InferenceResultsTestCase.java | 11 +- 17 files changed, 660 insertions(+), 526 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/script/MetadataTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index 1714717d0e6d3..fc17506555edb 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -48,7 +48,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version + assertThat(document.getMetadata().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2")); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1")); @@ -60,7 +60,7 @@ public void testEscapeFields() throws Exception { processor = new DotExpanderProcessor("_tag", null, null, "foo.bar"); processor.execute(document); assertThat(document.getSource().size(), equalTo(1)); - assertThat(document.getMetadataMap().size(), equalTo(1)); // the default version + assertThat(document.getMetadata().size(), equalTo(1)); // the default version assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2)); assertThat(document.getFieldValue("foo.bar.0", Integer.class), equalTo(1)); assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("2")); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index 4cab0b999c248..32566e82baf80 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -140,11 +140,11 @@ public void testRenameAtomicOperationSetFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (k, v) -> { + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (o, k, v) -> { if (v != null) { throw new UnsupportedOperationException(); } - }, "list", (k, v) -> {})); + }, "list", (o, k, v) -> {})); Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); @@ -160,7 +160,7 @@ public void testRenameAtomicOperationRemoveFails() throws Exception { Map metadata = new HashMap<>(); metadata.put("list", Collections.singletonList("item")); - IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (k, v) -> { + IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (o, k, v) -> { if (v == null) { throw new UnsupportedOperationException(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 2e69cdff0fe80..9b05421d479d2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -115,7 +115,7 @@ public void testSimulate() throws Exception { source.put("processed", true); IngestDocument ingestDocument = new IngestDocument("index", "id", Versions.MATCH_ANY, null, null, source); assertThat(simulateDocumentBaseResult.getIngestDocument().getSource(), equalTo(ingestDocument.getSource())); - assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadataMap(), equalTo(ingestDocument.getMetadataMap())); + assertThat(simulateDocumentBaseResult.getIngestDocument().getMetadata().getMap(), equalTo(ingestDocument.getMetadata().getMap())); assertThat(simulateDocumentBaseResult.getFailure(), nullValue()); // cleanup diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index e9e2882763e33..3a7e3c11fa141 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -107,10 +107,11 @@ IngestDocument getIngestDocument() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(DOC_FIELD); - Map metadataMap = ingestDocument.getMetadataMap(); - for (Map.Entry metadata : metadataMap.entrySet()) { - if (metadata.getValue() != null) { - builder.field(metadata.getKey(), metadata.getValue().toString()); + org.elasticsearch.script.Metadata metadata = ingestDocument.getMetadata(); + for (String key : metadata.keySet()) { + Object value = metadata.get(key); + if (value != null) { + builder.field(key, value.toString()); } } if (builder.getRestApiVersion() == RestApiVersion.V_7) { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index e6e8f428efb3a..a77d5d57b3170 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -68,7 +68,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver source ); this.ingestMetadata = new HashMap<>(); - this.ingestMetadata.put(TIMESTAMP, sourceAndMetadata.getTimestamp()); + this.ingestMetadata.put(TIMESTAMP, sourceAndMetadata.getMetadata().getTimestamp()); } /** @@ -76,12 +76,7 @@ public IngestDocument(String index, String id, long version, String routing, Ver */ public IngestDocument(IngestDocument other) { this( - new IngestSourceAndMetadata( - deepCopyMap(other.sourceAndMetadata.getSource()), - deepCopyMap(other.sourceAndMetadata.getMetadata()), - other.getIngestSourceAndMetadata().timestamp, - other.getIngestSourceAndMetadata().validators - ), + new IngestSourceAndMetadata(deepCopyMap(other.sourceAndMetadata.getSource()), other.sourceAndMetadata.getMetadata().clone()), deepCopyMap(other.ingestMetadata) ); } @@ -93,14 +88,12 @@ public IngestDocument(Map sourceAndMetadata, Map Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); this.sourceAndMetadata = new IngestSourceAndMetadata( sm.v1(), - sm.v2(), - IngestSourceAndMetadata.getTimestamp(ingestMetadata), - IngestSourceAndMetadata.VALIDATORS + new org.elasticsearch.script.Metadata(sm.v2(), IngestSourceAndMetadata.getTimestamp(ingestMetadata)) ); this.ingestMetadata = new HashMap<>(ingestMetadata); this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> { if (v instanceof String) { - return this.sourceAndMetadata.getTimestamp(); + return this.sourceAndMetadata.getMetadata().getTimestamp(); } return v; }); @@ -737,18 +730,11 @@ public IngestSourceAndMetadata getIngestSourceAndMetadata() { return sourceAndMetadata; } - /** - * Get all Metadata values in a Map - */ - public Map getMetadataMap() { - return sourceAndMetadata.getMetadata(); - } - /** * Get the strongly typed metadata */ public org.elasticsearch.script.Metadata getMetadata() { - return sourceAndMetadata; + return sourceAndMetadata.getMetadata(); } /** diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 67f0abae7b23d..152c9a80f7d62 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -897,27 +897,27 @@ private void innerExecute( itemDroppedHandler.accept(slot); handler.accept(null); } else { - IngestSourceAndMetadata sourceAndMetadata = ingestDocument.getIngestSourceAndMetadata(); + org.elasticsearch.script.Metadata metadata = ingestDocument.getMetadata(); // it's fine to set all metadata fields all the time, as ingest document holds their starting values // before ingestion, which might also get modified during ingestion. - indexRequest.index(sourceAndMetadata.getIndex()); - indexRequest.id(sourceAndMetadata.getId()); - indexRequest.routing(sourceAndMetadata.getRouting()); - indexRequest.version(sourceAndMetadata.getVersion()); - if (sourceAndMetadata.getVersionType() != null) { - indexRequest.versionType(VersionType.fromString(sourceAndMetadata.getVersionType())); + indexRequest.index(metadata.getIndex()); + indexRequest.id(metadata.getId()); + indexRequest.routing(metadata.getRouting()); + indexRequest.version(metadata.getVersion()); + if (metadata.getVersionType() != null) { + indexRequest.versionType(VersionType.fromString(metadata.getVersionType())); } Number number; - if ((number = sourceAndMetadata.getIfSeqNo()) != null) { + if ((number = metadata.getIfSeqNo()) != null) { indexRequest.setIfSeqNo(number.longValue()); } - if ((number = sourceAndMetadata.getIfPrimaryTerm()) != null) { + if ((number = metadata.getIfPrimaryTerm()) != null) { indexRequest.setIfPrimaryTerm(number.longValue()); } try { boolean ensureNoSelfReferences = ingestDocument.doNoSelfReferencesCheck(); - indexRequest.source(sourceAndMetadata.getSource(), indexRequest.getContentType(), ensureNoSelfReferences); + indexRequest.source(ingestDocument.getSource(), indexRequest.getContentType(), ensureNoSelfReferences); } catch (IllegalArgumentException ex) { // An IllegalArgumentException can be thrown when an ingest // processor creates a source map that is self-referencing. @@ -933,7 +933,7 @@ private void innerExecute( return; } Map map; - if ((map = sourceAndMetadata.getDynamicTemplates()) != null) { + if ((map = metadata.getDynamicTemplates()) != null) { Map mergedDynamicTemplates = new HashMap<>(indexRequest.getDynamicTemplates()); mergedDynamicTemplates.putAll(map); indexRequest.setDynamicTemplates(mergedDynamicTemplates); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java index 68d779be37812..16a79d6c7f074 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; @@ -17,7 +18,6 @@ import java.util.AbstractCollection; import java.util.AbstractMap; import java.util.AbstractSet; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiConsumer; import java.util.stream.Collectors; /** @@ -41,37 +40,10 @@ * * The map is expected to be used by processors, server code should the typed getter and setters where possible. */ -class IngestSourceAndMetadata extends AbstractMap implements Metadata { - protected final ZonedDateTime timestamp; - - /** - * map of key to validating function. Should throw {@link IllegalArgumentException} on invalid value - */ - static final Map> VALIDATORS = Map.of( - IngestDocument.Metadata.INDEX.getFieldName(), - IngestSourceAndMetadata::stringValidator, - IngestDocument.Metadata.ID.getFieldName(), - IngestSourceAndMetadata::stringValidator, - IngestDocument.Metadata.ROUTING.getFieldName(), - IngestSourceAndMetadata::stringValidator, - IngestDocument.Metadata.VERSION.getFieldName(), - IngestSourceAndMetadata::versionValidator, - IngestDocument.Metadata.VERSION_TYPE.getFieldName(), - IngestSourceAndMetadata::versionTypeValidator, - IngestDocument.Metadata.DYNAMIC_TEMPLATES.getFieldName(), - IngestSourceAndMetadata::mapValidator, - IngestDocument.Metadata.IF_SEQ_NO.getFieldName(), - IngestSourceAndMetadata::longValidator, - IngestDocument.Metadata.IF_PRIMARY_TERM.getFieldName(), - IngestSourceAndMetadata::longValidator, - IngestDocument.Metadata.TYPE.getFieldName(), - IngestSourceAndMetadata::stringValidator - ); +class IngestSourceAndMetadata extends AbstractMap { protected final Map source; - protected final Map metadata; - protected final Map> validators; - private EntrySet entrySet; // cache to avoid recreation + protected final Metadata metadata; /** * Create an IngestSourceAndMetadata with the given metadata, source and default validators @@ -85,46 +57,26 @@ class IngestSourceAndMetadata extends AbstractMap implements Met ZonedDateTime timestamp, Map source ) { - this(new HashMap<>(source), metadataMap(index, id, version, routing, versionType), timestamp, VALIDATORS); + this(new HashMap<>(source), new Metadata(index, id, version, routing, versionType, timestamp)); } /** - * Create IngestSourceAndMetadata with custom validators. + * Create IngestSourceAndMetadata from a source and metadata * * @param source the source document map * @param metadata the metadata map - * @param timestamp the time of ingestion - * @param validators validators to run on metadata map, if a key is in this map, the value is stored in metadata. - * if null, use the default validators from {@link #VALIDATORS} */ - IngestSourceAndMetadata( - Map source, - Map metadata, - ZonedDateTime timestamp, - Map> validators - ) { + IngestSourceAndMetadata(Map source, Metadata metadata) { this.source = source != null ? source : new HashMap<>(); - this.metadata = metadata != null ? metadata : new HashMap<>(); - this.timestamp = timestamp; - this.validators = validators != null ? validators : VALIDATORS; - validateMetadata(); - } - - /** - * Create the backing metadata map with the standard contents assuming default validators. - */ - protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); - metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); - metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); - if (routing != null) { - metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - if (versionType != null) { - metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + this.metadata = metadata; + Set badKeys = Sets.intersection(this.metadata.keySet(), this.source.keySet()); + if (badKeys.size() > 0) { + throw new IllegalArgumentException( + "unexpected metadata [" + + badKeys.stream().sorted().map(k -> k + ":" + this.source.get(k)).collect(Collectors.joining(", ")) + + "] in source" + ); } - return metadata; } /** @@ -132,11 +84,12 @@ protected static Map metadataMap(String index, String id, long v */ public static Tuple, Map> splitSourceAndMetadata(Map sourceAndMetadata) { if (sourceAndMetadata instanceof IngestSourceAndMetadata ingestSourceAndMetadata) { - return new Tuple<>(new HashMap<>(ingestSourceAndMetadata.source), new HashMap<>(ingestSourceAndMetadata.metadata)); + return new Tuple<>(new HashMap<>(ingestSourceAndMetadata.source), new HashMap<>(ingestSourceAndMetadata.metadata.getMap())); } Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); Map source = new HashMap<>(sourceAndMetadata); - for (String metadataName : VALIDATORS.keySet()) { + for (IngestDocument.Metadata ingestDocumentMetadata : IngestDocument.Metadata.values()) { + String metadataName = ingestDocumentMetadata.getFieldName(); if (sourceAndMetadata.containsKey(metadataName)) { metadata.put(metadataName, source.remove(metadataName)); } @@ -171,107 +124,17 @@ public Map getSource() { /** * get the metadata map, if externally modified then the guarantees of this class are not enforced */ - public Map getMetadata() { + public Metadata getMetadata() { return metadata; } - // These are available to scripts - public String getIndex() { - return getString(IngestDocument.Metadata.INDEX.getFieldName()); - } - - public void setIndex(String index) { - put(IngestDocument.Metadata.INDEX.getFieldName(), index); - } - - public String getId() { - return getString(IngestDocument.Metadata.ID.getFieldName()); - } - - public void setId(String id) { - put(IngestDocument.Metadata.ID.getFieldName(), id); - } - - public String getRouting() { - return getString(IngestDocument.Metadata.ROUTING.getFieldName()); - } - - public void setRouting(String routing) { - put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - - public String getVersionType() { - return getString(IngestDocument.Metadata.VERSION_TYPE.getFieldName()); - } - - public void setVersionType(String versionType) { - put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), versionType); - } - - public long getVersion() { - Number version = getNumber(IngestDocument.Metadata.VERSION.getFieldName()); - assert version != null : IngestDocument.Metadata.VERSION.getFieldName() + " validation allowed null version"; - return version.longValue(); - } - - public void setVersion(long version) { - put(IngestDocument.Metadata.VERSION.getFieldName(), version); - } - - // timestamp isn't backed by the map - public ZonedDateTime getTimestamp() { - return timestamp; - } - - // These are not available to scripts - public Number getIfSeqNo() { - return getNumber(IngestDocument.Metadata.IF_SEQ_NO.getFieldName()); - } - - public Number getIfPrimaryTerm() { - return getNumber(IngestDocument.Metadata.IF_PRIMARY_TERM.getFieldName()); - } - - @SuppressWarnings("unchecked") - public Map getDynamicTemplates() { - return (Map) metadata.get(IngestDocument.Metadata.DYNAMIC_TEMPLATES.getFieldName()); - } - - /** - * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata - */ - protected void validateMetadata() { - int numMetadata = 0; - for (Map.Entry> entry : validators.entrySet()) { - String key = entry.getKey(); - if (metadata.containsKey(key)) { - numMetadata++; - } - entry.getValue().accept(key, metadata.get(key)); - if (source.containsKey(key)) { - throw new IllegalArgumentException("Unexpected metadata key [" + key + "] in source with value [" + source.get(key) + "]"); - } - } - if (numMetadata < metadata.size()) { - Set keys = new HashSet<>(metadata.keySet()); - keys.removeAll(validators.keySet()); - throw new IllegalArgumentException( - "Unexpected metadata keys [" - + keys.stream().sorted().map(k -> k + ":" + metadata.get(k)).collect(Collectors.joining(", ")) - + "]" - ); - } - } - /** * Returns an entrySet that respects the validators of the map. */ @Override public Set> entrySet() { - if (entrySet == null) { - entrySet = new EntrySet(source.entrySet(), metadata.entrySet()); - } - return entrySet; + // Make a copy of the Metadata.keySet() to avoid a ConcurrentModificationException when removing a value from the iterator + return new EntrySet(source.entrySet(), new HashSet<>(metadata.keySet())); } /** @@ -280,9 +143,7 @@ public Set> entrySet() { */ @Override public Object put(String key, Object value) { - BiConsumer validator = validators.get(key); - if (validator != null) { - validator.accept(key, value); + if (metadata.isAvailable(key)) { return metadata.put(key, value); } return source.put(key, value); @@ -295,11 +156,9 @@ public Object put(String key, Object value) { @Override public Object remove(Object key) { // uses map directly to avoid AbstractMaps linear time implementation using entrySet() - if (key instanceof String strKey) { - BiConsumer validator = validators.get(key); - if (validator != null) { - validator.accept(strKey, null); - return metadata.remove(key); + if (key instanceof String str) { + if (metadata.isAvailable(str)) { + return metadata.remove(str); } } return source.remove(key); @@ -312,12 +171,9 @@ public Object remove(Object key) { @Override public void clear() { // AbstractMap uses entrySet().clear(), it should be quicker to run through the validators, then call the wrapped maps clear - validators.forEach((k, v) -> { - if (metadata.containsKey(k)) { - v.accept(k, null); - } - }); - metadata.clear(); + for (String key : metadata.keySet()) { + metadata.remove(key); + } source.clear(); } @@ -336,42 +192,23 @@ public boolean containsValue(Object value) { @Override public boolean containsKey(Object key) { // uses map directly to avoid AbstractMaps linear time implementation using entrySet() - return metadata.containsKey(key) || source.containsKey(key); + if (key instanceof String str) { + return metadata.containsKey(str) || source.containsKey(key); + } + return source.containsKey(key); } @Override public Object get(Object key) { // uses map directly to avoid AbstractMaps linear time implementation using entrySet() - if (validators.get(key) != null) { - return metadata.get(key); + if (key instanceof String str) { + if (metadata.isAvailable(str)) { + return metadata.get(str); + } } return source.get(key); } - /** - * Get the String version of the value associated with {@code key}, or null - */ - public String getString(Object key) { - return Objects.toString(get(key), null); - } - - /** - * Get the {@link Number} associated with key, or null - * @throws IllegalArgumentException if the value is not a {@link Number} - */ - public Number getNumber(Object key) { - Object value = get(key); - if (value == null) { - return null; - } - if (value instanceof Number number) { - return number; - } - throw new IllegalArgumentException( - "unexpected type for [" + key + "] with value [" + value + "], expected Number, got [" + value.getClass().getName() + "]" - ); - } - /** * Set of entries of the wrapped map that calls the appropriate validator before changing an entries value or removing an entry. * @@ -382,32 +219,31 @@ public Number getNumber(Object key) { */ class EntrySet extends AbstractSet> { Set> sourceSet; - Set> metadataSet; + Set metadataKeys; - EntrySet(Set> sourceSet, Set> metadataSet) { + EntrySet(Set> sourceSet, Set metadataKeys) { this.sourceSet = sourceSet; - this.metadataSet = metadataSet; + this.metadataKeys = metadataKeys; } @Override public Iterator> iterator() { - return new EntrySetIterator(sourceSet.iterator(), metadataSet.iterator()); + return new EntrySetIterator(sourceSet.iterator(), metadataKeys.iterator()); } @Override public int size() { - return sourceSet.size() + metadataSet.size(); + return sourceSet.size() + metadataKeys.size(); } @Override public boolean remove(Object o) { - if (metadataSet.contains(o)) { - if (o instanceof Map.Entry entry) { - if (entry.getKey()instanceof String key) { - BiConsumer validator = validators.get(key); - if (validator != null) { - validator.accept(key, null); - return metadataSet.remove(o); + if (o instanceof Map.Entry entry) { + if (entry.getKey()instanceof String key) { + if (metadata.containsKey(key)) { + if (Objects.equals(entry.getValue(), metadata.get(key))) { + metadata.remove(key); + return true; } } } @@ -424,25 +260,25 @@ public boolean remove(Object o) { */ class EntrySetIterator implements Iterator> { final Iterator> sourceIter; - final Iterator> metadataIter; + final Iterator metadataKeyIter; boolean sourceCur = true; - Entry cur; + Map.Entry cur; - EntrySetIterator(Iterator> sourceIter, Iterator> metadataIter) { + EntrySetIterator(Iterator> sourceIter, Iterator metadataKeyIter) { this.sourceIter = sourceIter; - this.metadataIter = metadataIter; + this.metadataKeyIter = metadataKeyIter; } @Override public boolean hasNext() { - return sourceIter.hasNext() || metadataIter.hasNext(); + return sourceIter.hasNext() || metadataKeyIter.hasNext(); } @Override public Map.Entry next() { sourceCur = sourceIter.hasNext(); - return cur = new Entry(sourceCur ? sourceIter.next() : metadataIter.next(), sourceCur); + return cur = sourceCur ? sourceIter.next() : new Entry(metadataKeyIter.next()); } /** @@ -458,145 +294,48 @@ public void remove() { if (sourceCur) { sourceIter.remove(); } else { - BiConsumer validator = validators.get(cur.getKey()); - if (validator != null) { - validator.accept(cur.getKey(), null); - } - metadataIter.remove(); + metadata.remove(cur.getKey()); } } } /** - * Wrapped Map.Entry that calls the key's validator on {@link #setValue(Object)} + * Map.Entry that stores metadata key and calls into {@link #metadata} for {@link #setValue} */ class Entry implements Map.Entry { - final Map.Entry entry; - final boolean isSource; + final String key; - Entry(Map.Entry entry, boolean isSource) { - this.entry = entry; - this.isSource = isSource; + Entry(String key) { + this.key = key; } @Override public String getKey() { - return entry.getKey(); + return key; } @Override public Object getValue() { - return entry.getValue(); + return metadata.get(key); } - /** - * Associate the value with the Entry's key in the linked Map. If the Entry's key has a validator, it is applied before association - * @throws IllegalArgumentException if value does not pass validation for the Entry's key - */ @Override public Object setValue(Object value) { - if (isSource == false) { - BiConsumer validator = validators.get(entry.getKey()); - if (validator != null) { - validator.accept(entry.getKey(), value); - } - } - return entry.setValue(value); - } - } - - /** - * Allow a String or null - */ - protected static void stringValidator(String key, Object value) { - if (value == null || value instanceof String) { - return; - } - throw new IllegalArgumentException( - key + " must be null or a String but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Allow Numbers that can be represented as longs without loss of precision - */ - protected static void longValidator(String key, Object value) { - if (value == null) { - return; // Allow null version for now - } - if (value instanceof Number number) { - long version = number.longValue(); - // did we round? - if (number.doubleValue() == version) { - return; - } - } - throw new IllegalArgumentException( - key + " may only be set to an int or a long but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); - } - - /** - * Version must be non-null and representable as a long without loss of precision - */ - protected static void versionValidator(String key, Object value) { - if (value == null) { - throw new IllegalArgumentException(key + " cannot be null"); - } - longValidator(key, value); - } - - /** - * Allow lower case Strings that map to VersionType values, or null - */ - protected static void versionTypeValidator(String key, Object value) { - if (value == null) { - return; - } - if (value instanceof String versionType) { - try { - VersionType.fromString(versionType); - return; - } catch (IllegalArgumentException ignored) {} - } - throw new IllegalArgumentException( - key - + " must be a null or one of [" - + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) - + "] but was [" - + value - + "] with type [" - + value.getClass().getName() - + "]" - ); - } - - /** - * Allow maps - */ - protected static void mapValidator(String key, Object value) { - if (value == null || value instanceof Map) { - return; + return metadata.put(key, value); } - throw new IllegalArgumentException( - key + " must be a null or a Map but was [" + value + "] with type [" + value.getClass().getName() + "]" - ); } @Override public boolean equals(Object o) { if (this == o) return true; - if ((o instanceof IngestSourceAndMetadata) == false) return false; + if (o == null || getClass() != o.getClass()) return false; if (super.equals(o) == false) return false; IngestSourceAndMetadata that = (IngestSourceAndMetadata) o; - return Objects.equals(timestamp, that.timestamp) - && source.equals(that.source) - && metadata.equals(that.metadata) - && validators.equals(that.validators); + return Objects.equals(source, that.source) && Objects.equals(metadata, that.metadata); } @Override public int hashCode() { - return Objects.hash(timestamp, source, metadata, validators); + return Objects.hash(source, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index e598a211c109a..8118e4f5f0cb7 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -8,53 +8,411 @@ package org.elasticsearch.script; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.ingest.IngestDocument; + import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** - * Ingest and update metadata available to write scripts + * Ingest and update metadata available to write scripts. + * + * Provides a map-like interface for backwards compatibility with the ctx map. + * - {@link #put(String, Object)} + * - {@link #get(String)} + * - {@link #remove(String)} + * - {@link #containsKey(String)} + * - {@link #containsValue(Object)} + * - {@link #keySet()} for iteration + * - {@link #size()} + * - {@link #isAvailable(String)} for determining if a key is a metadata key + * + * Provides getters and setters for script usage. + * + * Validates all updates whether originating in map-like interface or setters. */ -public interface Metadata { +public class Metadata { + protected static final String INDEX = "_index"; + protected static final String ID = "_id"; + protected static final String ROUTING = "_routing"; + protected static final String VERSION_TYPE = "_version_type"; + protected static final String VERSION = "_version"; + protected static final String TYPE = "_type"; // type is deprecated so it's supported in the map but not available as a getter + protected static final String IF_SEQ_NO = "_if_seq_no"; + protected static final String IF_PRIMARY_TERM = "_if_primary_term"; + protected static final String DYNAMIC_TEMPLATES = "_dynamic_templates"; + + protected static final Map VALIDATORS = Map.of( + INDEX, + Metadata::stringValidator, + ID, + Metadata::stringValidator, + ROUTING, + Metadata::stringValidator, + VERSION_TYPE, + Metadata::versionTypeValidator, + VERSION, + Metadata::notNullLongValidator, + TYPE, + Metadata::stringValidator, + IF_SEQ_NO, + Metadata::longValidator, + IF_PRIMARY_TERM, + Metadata::longValidator, + DYNAMIC_TEMPLATES, + Metadata::mapValidator + ); + + protected final Map map; + protected final Map validators; + + // timestamp is new to ingest metadata, so it doesn't need to be backed by the map for back compat + protected final ZonedDateTime timestamp; + + public Metadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { + this(metadataMap(index, id, version, routing, versionType), timestamp, VALIDATORS); + } + + public Metadata(Map map, ZonedDateTime timestamp) { + this(map, timestamp, VALIDATORS); + } + + Metadata(Map map, ZonedDateTime timestamp, Map validators) { + this.map = map; + this.timestamp = timestamp; + this.validators = validators; + validateMetadata(); + } + /** - * The destination index + * Create the backing metadata map with the standard contents assuming default validators. */ - String getIndex(); + protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); + metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); + metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); + } + if (versionType != null) { + metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + } + return metadata; + } + + /** + * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata + */ + protected void validateMetadata() { + int numMetadata = 0; + for (Map.Entry entry : validators.entrySet()) { + String key = entry.getKey(); + if (map.containsKey(key)) { + numMetadata++; + } + entry.getValue().accept(MapOperation.INIT, key, map.get(key)); + } + if (numMetadata < map.size()) { + Set keys = new HashSet<>(map.keySet()); + keys.removeAll(validators.keySet()); + throw new IllegalArgumentException( + "Unexpected metadata keys [" + keys.stream().sorted().map(k -> k + ":" + map.get(k)).collect(Collectors.joining(", ")) + "]" + ); + } + } + + // These are available to scripts + public String getIndex() { + return getString(INDEX); + } + + public void setIndex(String index) { + put(INDEX, index); + } + + public String getId() { + return getString(ID); + } + + public void setId(String id) { + put(ID, id); + } + + public String getRouting() { + return getString(ROUTING); + } - void setIndex(String index); + public void setRouting(String routing) { + put(ROUTING, routing); + } + + public String getVersionType() { + return getString(VERSION_TYPE); + } + + public void setVersionType(String versionType) { + put(VERSION_TYPE, versionType); + } + + public long getVersion() { + return getNumber(VERSION).longValue(); + } + + public void setVersion(long version) { + put(VERSION, version); + } + + public ZonedDateTime getTimestamp() { + return timestamp; + } + + // These are not available to scripts + public Number getIfSeqNo() { + return getNumber(IF_SEQ_NO); + } + + public Number getIfPrimaryTerm() { + return getNumber(IF_PRIMARY_TERM); + } + + @SuppressWarnings("unchecked") + public Map getDynamicTemplates() { + return (Map) get(DYNAMIC_TEMPLATES); + } /** - * The document id + * Get the String version of the value associated with {@code key}, or null */ - String getId(); + protected String getString(String key) { + return Objects.toString(get(key), null); + } - void setId(String id); + /** + * Get the {@link Number} associated with key, or null + * @throws IllegalArgumentException if the value is not a {@link Number} + */ + protected Number getNumber(String key) { + Object value = get(key); + if (value == null) { + return null; + } + if (value instanceof Number number) { + return number; + } + throw new IllegalStateException( + "unexpected type for [" + key + "] with value [" + value + "], expected Number, got [" + value.getClass().getName() + "]" + ); + } /** - * The document routing string + * Is this key a Metadata key? A {@link #remove}d key would return false for {@link #containsKey(String)} but true for + * this call. */ - String getRouting(); + public boolean isAvailable(String key) { + return validators.containsKey(key); + } - void setRouting(String routing); + /** + * Create the mapping from key to value. + * @throws IllegalArgumentException if {@link #isAvailable(String)} is false or the key cannot be updated to the value. + */ + public Object put(String key, Object value) { + Validator v = validators.getOrDefault(key, this::badKey); + v.accept(MapOperation.UPDATE, key, value); + return map.put(key, value); + } /** - * The version of the document + * Does the metadata contain the key? */ - long getVersion(); + public boolean containsKey(String key) { + return map.containsKey(key); + } - void setVersion(long version); + /** + * Does the metadata contain the value. + */ + public boolean containsValue(Object value) { + return map.containsValue(value); + } /** - * The version type of the document, {@link org.elasticsearch.index.VersionType} as a lower-case string. + * Get the value associated with {@param key} */ - String getVersionType(); + public Object get(String key) { + return map.get(key); + } /** - * Set the version type of the document. - * @param versionType {@link org.elasticsearch.index.VersionType} as a lower-case string + * Remove the mapping associated with {@param key} + * @throws IllegalArgumentException if {@link #isAvailable(String)} is false or the key cannot be removed. */ - void setVersionType(String versionType); + public Object remove(String key) { + Validator v = validators.getOrDefault(key, this::badKey); + v.accept(MapOperation.REMOVE, key, null); + return map.remove(key); + } /** - * Timestamp of this ingestion or update + * Return the list of keys with mappings */ - ZonedDateTime getTimestamp(); + public Set keySet() { + return Collections.unmodifiableSet(map.keySet()); + } + + /** + * The number of metadata keys currently mapped. + */ + public int size() { + return map.size(); + } + + @Override + public Metadata clone() { + return new Metadata(new HashMap<>(map), timestamp, new HashMap<>(validators)); + } + + /** + * Get the backing map, if modified then the guarantees of this class may not hold + */ + public Map getMap() { + return map; + } + + /** + * Allow a String or null. + * @throws IllegalArgumentException if {@param value} is neither a {@link String} nor null + */ + protected static void stringValidator(MapOperation op, String key, Object value) { + if (op == MapOperation.REMOVE || value == null || value instanceof String) { + return; + } + throw new IllegalArgumentException( + key + " must be null or a String but was [" + value + "] with type [" + value.getClass().getName() + "]" + ); + } + + /** + * Allow Numbers that can be represented as longs without loss of precision or null + * @throws IllegalArgumentException if the value cannot be represented as a long + */ + protected static void longValidator(MapOperation op, String key, Object value) { + if (op == MapOperation.REMOVE || value == null) { + return; + } + if (value instanceof Number number) { + long version = number.longValue(); + // did we round? + if (number.doubleValue() == version) { + return; + } + } + throw new IllegalArgumentException( + key + " may only be set to an int or a long but was [" + value + "] with type [" + value.getClass().getName() + "]" + ); + } + + /** + * Same as {@link #longValidator(MapOperation, String, Object)} but {@param value} cannot be null. + * @throws IllegalArgumentException if value is null or cannot be represented as a long. + */ + protected static void notNullLongValidator(MapOperation op, String key, Object value) { + if (op == MapOperation.REMOVE || value == null) { + throw new IllegalArgumentException(key + " cannot be removed or set to null"); + } + longValidator(op, key, value); + } + + /** + * Allow maps. + * @throws IllegalArgumentException if {@param value} is not a {@link Map} + */ + protected static void mapValidator(MapOperation op, String key, Object value) { + if (op == MapOperation.REMOVE || value == null || value instanceof Map) { + return; + } + throw new IllegalArgumentException( + key + " must be a null or a Map but was [" + value + "] with type [" + value.getClass().getName() + "]" + ); + } + + /** + * Allow lower case Strings that map to VersionType values, or null. + * @throws IllegalArgumentException if {@param value} cannot be converted via {@link VersionType#fromString(String)} + */ + protected static void versionTypeValidator(MapOperation op, String key, Object value) { + if (op == MapOperation.REMOVE || value == null) { + return; + } + if (value instanceof String versionType) { + try { + VersionType.fromString(versionType); + return; + } catch (IllegalArgumentException ignored) {} + } + throw new IllegalArgumentException( + key + + " must be a null or one of [" + + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) + + "] but was [" + + value + + "] with type [" + + value.getClass().getName() + + "]" + ); + } + + private void badKey(MapOperation op, String key, Object value) { + throw new IllegalArgumentException( + "unexpected metadata key [" + + key + + "], expected one of [" + + validators.keySet().stream().sorted().collect(Collectors.joining(", ")) + + "]" + ); + } + + /** + * The operation being performed on the value in the map. + * INIT: Initial value - the metadata value as passed into this class + * UPDATE: the metadata is being set to a different value + * REMOVE: the metadata mapping is being removed + */ + public enum MapOperation { + INIT, + UPDATE, + REMOVE + } + + /** + * A "TriConsumer" that tests if the {@link MapOperation}, the metadata key and value are valid. + * + * throws IllegalArgumentException if the given triple is invalid + */ + @FunctionalInterface + public interface Validator { + void accept(MapOperation op, String key, Object value); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Metadata metadata = (Metadata) o; + return Objects.equals(map, metadata.map) && Objects.equals(timestamp, metadata.timestamp); + } + + @Override + public int hashCode() { + return Objects.hash(map, timestamp); + } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index 958c59f67a838..f609114832a70 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -378,7 +378,7 @@ public boolean isAsync() { for (int id = 0; id < numDocs; id++) { SimulateDocumentBaseResult result = (SimulateDocumentBaseResult) response.getResults().get(id); assertThat( - result.getIngestDocument().getMetadataMap().get(IngestDocument.Metadata.ID.getFieldName()), + result.getIngestDocument().getMetadata().get(IngestDocument.Metadata.ID.getFieldName()), equalTo(Integer.toString(id)) ); assertThat(result.getIngestDocument().getSourceAndMetadata().get("processed"), is(true)); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index e906a2d619cb0..bdc14e32c2a6f 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -100,9 +100,8 @@ public void testParseUsingPipelineStore() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadataMap(); - assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); + assertThat(ingestDocument.getMetadata().get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); + assertThat(ingestDocument.getMetadata().get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(ingestDocument.getSource(), equalTo(expectedDocument.get(Fields.SOURCE))); } @@ -196,14 +195,14 @@ public void testParseWithProvidedPipeline() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadataMap(); - assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); - assertThat(metadataMap.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); - assertThat(metadataMap.get(VERSION.getFieldName()), equalTo(expectedDocument.get(VERSION.getFieldName()))); - assertThat(metadataMap.get(VERSION_TYPE.getFieldName()), equalTo(expectedDocument.get(VERSION_TYPE.getFieldName()))); - assertThat(metadataMap.get(IF_SEQ_NO.getFieldName()), equalTo(expectedDocument.get(IF_SEQ_NO.getFieldName()))); - assertThat(metadataMap.get(IF_PRIMARY_TERM.getFieldName()), equalTo(expectedDocument.get(IF_PRIMARY_TERM.getFieldName()))); + org.elasticsearch.script.Metadata metadata = ingestDocument.getMetadata(); + assertThat(metadata.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); + assertThat(metadata.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); + assertThat(metadata.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); + assertThat(metadata.get(VERSION.getFieldName()), equalTo(expectedDocument.get(VERSION.getFieldName()))); + assertThat(metadata.get(VERSION_TYPE.getFieldName()), equalTo(expectedDocument.get(VERSION_TYPE.getFieldName()))); + assertThat(metadata.get(IF_SEQ_NO.getFieldName()), equalTo(expectedDocument.get(IF_SEQ_NO.getFieldName()))); + assertThat(metadata.get(IF_PRIMARY_TERM.getFieldName()), equalTo(expectedDocument.get(IF_PRIMARY_TERM.getFieldName()))); assertThat(ingestDocument.getSource(), equalTo(expectedDocument.get(Fields.SOURCE))); } @@ -349,12 +348,12 @@ public void testIngestPipelineWithDocumentsWithType() throws Exception { Iterator> expectedDocsIterator = expectedDocs.iterator(); for (IngestDocument ingestDocument : actualRequest.documents()) { Map expectedDocument = expectedDocsIterator.next(); - Map metadataMap = ingestDocument.getMetadataMap(); - assertThat(metadataMap.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); - assertThat(metadataMap.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); - assertThat(metadataMap.get(VERSION.getFieldName()), equalTo(expectedDocument.get(VERSION.getFieldName()))); - assertThat(metadataMap.get(VERSION_TYPE.getFieldName()), equalTo(expectedDocument.get(VERSION_TYPE.getFieldName()))); + org.elasticsearch.script.Metadata metadata = ingestDocument.getMetadata(); + assertThat(metadata.get(INDEX.getFieldName()), equalTo(expectedDocument.get(INDEX.getFieldName()))); + assertThat(metadata.get(ID.getFieldName()), equalTo(expectedDocument.get(ID.getFieldName()))); + assertThat(metadata.get(ROUTING.getFieldName()), equalTo(expectedDocument.get(ROUTING.getFieldName()))); + assertThat(metadata.get(VERSION.getFieldName()), equalTo(expectedDocument.get(VERSION.getFieldName()))); + assertThat(metadata.get(VERSION_TYPE.getFieldName()), equalTo(expectedDocument.get(VERSION_TYPE.getFieldName()))); assertThat(ingestDocument.getSource(), equalTo(expectedDocument.get(Fields.SOURCE))); } assertThat(actualRequest.pipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index 03007d1e84712..6d70802bab021 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -148,19 +148,18 @@ public void testToXContent() throws IOException { Map toXContentSource = (Map) toXContentDoc.get("_source"); Map toXContentIngestMetadata = (Map) toXContentDoc.get("_ingest"); - Map metadataMap = ingestDocument.getMetadataMap(); - for (Map.Entry metadata : metadataMap.entrySet()) { - String fieldName = metadata.getKey(); - if (metadata.getValue() == null) { + for (String fieldName : ingestDocument.getMetadata().keySet()) { + Object value = ingestDocument.getMetadata().get(fieldName); + if (value == null) { assertThat(toXContentDoc.containsKey(fieldName), is(false)); } else { - assertThat(toXContentDoc.get(fieldName), equalTo(metadata.getValue().toString())); + assertThat(toXContentDoc.get(fieldName), equalTo(value.toString())); } } - Map sourceAndMetadata = Maps.newMapWithExpectedSize(toXContentSource.size() + metadataMap.size()); + Map sourceAndMetadata = Maps.newMapWithExpectedSize(toXContentSource.size() + ingestDocument.getMetadata().size()); sourceAndMetadata.putAll(toXContentSource); - sourceAndMetadata.putAll(metadataMap); + ingestDocument.getMetadata().keySet().forEach(k -> sourceAndMetadata.put(k, ingestDocument.getMetadata().get(k))); IngestDocument serializedIngestDocument = new IngestDocument(sourceAndMetadata, toXContentIngestMetadata); // TODO(stu): is this test correct? Comparing against ingestDocument fails due to incorrectly failed byte array comparisons assertThat(serializedIngestDocument, equalTo(serializedIngestDocument)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 7fcb2b07a1f81..39d93a0691856 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2230,7 +2230,7 @@ private class IngestDocumentMatcher implements ArgumentMatcher { public boolean matches(IngestDocument other) { // ingest metadata and IngestSourceAndMetadata will not be the same (timestamp differs every time) return Objects.equals(ingestDocument.getSource(), other.getSource()) - && Objects.equals(ingestDocument.getMetadataMap(), other.getMetadataMap()); + && Objects.equals(ingestDocument.getMetadata().getMap(), other.getMetadata().getMap()); } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java index f0085abda82ee..34a05e9ef2e03 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestSourceAndMetadataTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.ingest; import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Metadata; +import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; @@ -17,15 +19,13 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.ingest.TestIngestDocument.replaceValidator; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.notNullValue; public class IngestSourceAndMetadataTests extends ESTestCase { IngestSourceAndMetadata map; + Metadata md; public void testSettersAndGetters() { Map metadata = new HashMap<>(); @@ -37,65 +37,32 @@ public void testSettersAndGetters() { metadata.put("_if_primary_term", 10000); metadata.put("_version_type", "internal"); metadata.put("_dynamic_templates", Map.of("foo", "bar")); - map = new IngestSourceAndMetadata(new HashMap<>(), metadata, null, null); - assertEquals("myIndex", map.getIndex()); - map.setIndex("myIndex2"); - assertEquals("myIndex2", map.getIndex()); + map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)); + md = map.getMetadata(); + assertEquals("myIndex", md.getIndex()); + md.setIndex("myIndex2"); + assertEquals("myIndex2", md.getIndex()); - assertEquals("myId", map.getId()); - map.setId("myId2"); - assertEquals("myId2", map.getId()); + assertEquals("myId", md.getId()); + md.setId("myId2"); + assertEquals("myId2", md.getId()); - assertEquals("myRouting", map.getRouting()); - map.setRouting("myRouting2"); - assertEquals("myRouting2", map.getRouting()); + assertEquals("myRouting", md.getRouting()); + md.setRouting("myRouting2"); + assertEquals("myRouting2", md.getRouting()); - assertEquals(20, map.getVersion()); - map.setVersion(10); - assertEquals(10, map.getVersion()); + assertEquals(20, md.getVersion()); + md.setVersion(10); + assertEquals(10, md.getVersion()); - assertEquals("internal", map.getVersionType()); - map.setVersionType("external_gte"); - assertEquals("external_gte", map.getVersionType()); + assertEquals("internal", md.getVersionType()); + md.setVersionType("external_gte"); + assertEquals("external_gte", md.getVersionType()); - assertEquals(Map.of("foo", "bar"), map.getDynamicTemplates()); + assertEquals(Map.of("foo", "bar"), md.getDynamicTemplates()); - assertEquals(500, map.getIfSeqNo()); - assertEquals(10000, map.getIfPrimaryTerm()); - } - - public void testGetString() { - Map metadata = new HashMap<>(); - metadata.put("_routing", "myRouting"); - Map source = new HashMap<>(); - source.put("str", "myStr"); - source.put("toStr", new Object() { - @Override - public String toString() { - return "myToString()"; - } - }); - source.put("missing", null); - map = new IngestSourceAndMetadata(source, metadata, null, replaceValidator("_version", IngestSourceAndMetadata::longValidator)); - assertNull(map.getString("missing")); - assertNull(map.getString("no key")); - assertEquals("myToString()", map.getString("toStr")); - assertEquals("myStr", map.getString("str")); - assertEquals("myRouting", map.getString("_routing")); - } - - public void testGetNumber() { - Map metadata = new HashMap<>(); - metadata.put("_version", Long.MAX_VALUE); - Map source = new HashMap<>(); - source.put("number", "NaN"); - source.put("missing", null); - map = new IngestSourceAndMetadata(source, metadata, null, null); - assertEquals(Long.MAX_VALUE, map.getNumber("_version")); - IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.getNumber("number")); - assertEquals("unexpected type for [number] with value [NaN], expected Number, got [java.lang.String]", err.getMessage()); - assertNull(map.getNumber("missing")); - assertNull(map.getNumber("no key")); + assertEquals(500, md.getIfSeqNo()); + assertEquals(10000, md.getIfPrimaryTerm()); } public void testInvalidMetadata() { @@ -103,7 +70,7 @@ public void testInvalidMetadata() { metadata.put("_version", Double.MAX_VALUE); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), metadata, null, null) + () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) ); assertThat(err.getMessage(), containsString("_version may only be set to an int or a long but was [")); assertThat(err.getMessage(), containsString("] with type [java.lang.Double]")); @@ -114,9 +81,9 @@ public void testSourceInMetadata() { source.put("_version", 25); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(source, source, null, null) + () -> new IngestSourceAndMetadata(source, new Metadata(source, null)) ); - assertEquals("Unexpected metadata key [_version] in source with value [25]", err.getMessage()); + assertEquals("unexpected metadata [_version:25] in source", err.getMessage()); } public void testExtraMetadata() { @@ -126,7 +93,7 @@ public void testExtraMetadata() { metadata.put("routing", "myRouting"); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestSourceAndMetadata(new HashMap<>(), metadata, null, null) + () -> new IngestSourceAndMetadata(new HashMap<>(), new Metadata(metadata, null)) ); assertEquals("Unexpected metadata keys [routing:myRouting, version:567]", err.getMessage()); } @@ -135,7 +102,7 @@ public void testPutSource() { Map metadata = new HashMap<>(); metadata.put("_version", 123); Map source = new HashMap<>(); - map = new IngestSourceAndMetadata(source, metadata, null, null); + map = new IngestSourceAndMetadata(source, new Metadata(metadata, null)); } public void testRemove() { @@ -143,11 +110,11 @@ public void testRemove() { String canRemove = "canRemove"; Map metadata = new HashMap<>(); metadata.put(cannotRemove, "value"); - map = new IngestSourceAndMetadata(new HashMap<>(), metadata, null, Map.of(cannotRemove, (k, v) -> { + map = new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, Map.of(cannotRemove, (o, k, v) -> { if (v == null) { throw new IllegalArgumentException(k + " cannot be null or removed"); } - }, canRemove, (k, v) -> {})); + }, canRemove, (o, k, v) -> {}))); String msg = "cannotRemove cannot be null or removed"; IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.remove(cannotRemove)); assertEquals(msg, err.getMessage()); @@ -208,7 +175,8 @@ public void testEntryAndIterator() { source.put("foo", "bar"); source.put("baz", "qux"); source.put("noz", "zon"); - map = new IngestSourceAndMetadata(source, metadata, null, replaceValidator("_version", IngestSourceAndMetadata::longValidator)); + map = new IngestSourceAndMetadata(source, TestMetadata.withNullableVersion(metadata)); + md = map.getMetadata(); for (Map.Entry entry : map.entrySet()) { if ("foo".equals(entry.getKey())) { @@ -218,7 +186,7 @@ public void testEntryAndIterator() { } } assertEquals("changed", map.get("foo")); - assertEquals("external_gte", map.getVersionType()); + assertEquals("external_gte", md.getVersionType()); assertEquals(5, map.entrySet().size()); assertEquals(5, map.size()); @@ -235,7 +203,7 @@ public void testEntryAndIterator() { } } - assertNull(map.getVersionType()); + assertNull(md.getVersionType()); assertFalse(map.containsKey("baz")); assertTrue(map.containsKey("_version")); assertTrue(map.containsKey("foo")); @@ -247,7 +215,7 @@ public void testEntryAndIterator() { } public void testContainsValue() { - map = new IngestSourceAndMetadata(Map.of("myField", "fieldValue"), Map.of("_version", 5678), null, null); + map = new IngestSourceAndMetadata(Map.of("myField", "fieldValue"), new Metadata(Map.of("_version", 5678), null)); assertTrue(map.containsValue(5678)); assertFalse(map.containsValue(5679)); assertTrue(map.containsValue("fieldValue")); @@ -256,39 +224,40 @@ public void testContainsValue() { public void testValidators() { map = new IngestSourceAndMetadata("myIndex", "myId", 1234, "myRouting", VersionType.EXTERNAL, null, new HashMap<>()); + md = map.getMetadata(); IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> map.put("_index", 555)); assertEquals("_index must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); - assertEquals("myIndex", map.getIndex()); + assertEquals("myIndex", md.getIndex()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_id", 555)); assertEquals("_id must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); - assertEquals("myId", map.getId()); + assertEquals("myId", md.getId()); map.put("_id", "myId2"); - assertEquals("myId2", map.getId()); + assertEquals("myId2", md.getId()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_routing", 555)); assertEquals("_routing must be null or a String but was [555] with type [java.lang.Integer]", err.getMessage()); - assertEquals("myRouting", map.getRouting()); + assertEquals("myRouting", md.getRouting()); map.put("_routing", "myRouting2"); - assertEquals("myRouting2", map.getRouting()); + assertEquals("myRouting2", md.getRouting()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version", "five-five-five")); assertEquals( "_version may only be set to an int or a long but was [five-five-five] with type [java.lang.String]", err.getMessage() ); - assertEquals(1234, map.getVersion()); + assertEquals(1234, md.getVersion()); map.put("_version", 555); - assertEquals(555, map.getVersion()); + assertEquals(555, md.getVersion()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version_type", "vt")); assertEquals( "_version_type must be a null or one of [internal, external, external_gte] but was [vt] with type [java.lang.String]", err.getMessage() ); - assertEquals("external", map.getVersionType()); + assertEquals("external", md.getVersionType()); map.put("_version_type", "internal"); - assertEquals("internal", map.getVersionType()); + assertEquals("internal", md.getVersionType()); err = expectThrows(IllegalArgumentException.class, () -> map.put("_version_type", VersionType.EXTERNAL.toString())); assertEquals( "_version_type must be a null or one of [internal, external, external_gte] but was [EXTERNAL] with type [java.lang.String]", @@ -300,8 +269,8 @@ public void testValidators() { + " [org.elasticsearch.index.VersionType$2]", err.getMessage() ); - assertEquals("internal", map.getVersionType()); - err = expectThrows(IllegalArgumentException.class, () -> map.setVersionType(VersionType.EXTERNAL.toString())); + assertEquals("internal", md.getVersionType()); + err = expectThrows(IllegalArgumentException.class, () -> md.setVersionType(VersionType.EXTERNAL.toString())); assertEquals( "_version_type must be a null or one of [internal, external, external_gte] but was [EXTERNAL] with type [java.lang.String]", err.getMessage() @@ -311,33 +280,27 @@ public void testValidators() { assertEquals("_dynamic_templates must be a null or a Map but was [5] with type [java.lang.String]", err.getMessage()); Map dt = Map.of("a", "b"); map.put("_dynamic_templates", dt); - assertThat(dt, equalTo(map.getDynamicTemplates())); - } - - public void testDefaultValidatorForAllMetadata() { - for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { - assertThat(IngestSourceAndMetadata.VALIDATORS, hasEntry(equalTo(m.getFieldName()), notNullValue())); - } - assertEquals(IngestDocument.Metadata.values().length, IngestSourceAndMetadata.VALIDATORS.size()); + assertThat(dt, equalTo(md.getDynamicTemplates())); } public void testHandlesAllVersionTypes() { - Map md = new HashMap<>(); - md.put("_version", 1234); - map = new IngestSourceAndMetadata(new HashMap<>(), md, null, null); - assertNull(map.getVersionType()); + Map mdRawMap = new HashMap<>(); + mdRawMap.put("_version", 1234); + map = new IngestSourceAndMetadata(new HashMap<>(), new Metadata(mdRawMap, null)); + md = map.getMetadata(); + assertNull(md.getVersionType()); for (VersionType vt : VersionType.values()) { - map.setVersionType(VersionType.toString(vt)); + md.setVersionType(VersionType.toString(vt)); assertEquals(VersionType.toString(vt), map.get("_version_type")); } for (VersionType vt : VersionType.values()) { map.put("_version_type", VersionType.toString(vt)); - assertEquals(vt.toString().toLowerCase(Locale.ROOT), map.getVersionType()); + assertEquals(vt.toString().toLowerCase(Locale.ROOT), md.getVersionType()); } - map.setVersionType(null); - assertNull(map.getVersionType()); + md.setVersionType(null); + assertNull(md.getVersionType()); } private static class TestEntry implements Map.Entry { diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java new file mode 100644 index 0000000000000..5469bd91ec9f4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/MetadataTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.test.ESTestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.notNullValue; + +public class MetadataTests extends ESTestCase { + Metadata md; + + public void testDefaultValidatorForAllMetadata() { + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + assertThat(Metadata.VALIDATORS, hasEntry(equalTo(m.getFieldName()), notNullValue())); + } + assertEquals(IngestDocument.Metadata.values().length, Metadata.VALIDATORS.size()); + } + + public void testGetString() { + Map metadata = new HashMap<>(); + metadata.put("a", "A"); + metadata.put("b", new Object() { + @Override + public String toString() { + return "myToString()"; + } + }); + metadata.put("c", null); + metadata.put("d", 1234); + md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + assertNull(md.getString("c")); + assertNull(md.getString("no key")); + assertEquals("myToString()", md.getString("b")); + assertEquals("A", md.getString("a")); + assertEquals("1234", md.getString("d")); + } + + public void testGetNumber() { + Map metadata = new HashMap<>(); + metadata.put("a", Long.MAX_VALUE); + metadata.put("b", Double.MAX_VALUE); + metadata.put("c", "NaN"); + metadata.put("d", null); + md = new TestMetadata(metadata, allowAllValidators("a", "b", "c", "d")); + assertEquals(Long.MAX_VALUE, md.getNumber("a")); + assertEquals(Double.MAX_VALUE, md.getNumber("b")); + IllegalStateException err = expectThrows(IllegalStateException.class, () -> md.getNumber("c")); + assertEquals("unexpected type for [c] with value [NaN], expected Number, got [java.lang.String]", err.getMessage()); + assertNull(md.getNumber("d")); + assertNull(md.getNumber("no key")); + } + + private static Map allowAllValidators(String... keys) { + Map validators = new HashMap<>(); + for (String key : keys) { + validators.put(key, (o, k, v) -> {}); + } + return validators; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index b6b6949a07290..ffd9ca324f63b 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -11,11 +11,12 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Metadata; +import org.elasticsearch.script.TestMetadata; import org.elasticsearch.test.ESTestCase; import java.util.HashMap; import java.util.Map; -import java.util.function.BiConsumer; /** * Construct ingest documents for testing purposes @@ -36,10 +37,8 @@ public static IngestDocument withNullableVersion(Map sourceAndMe * _versions. Normally null _version is not allowed, but many tests don't care about that invariant. */ public static IngestDocument ofIngestWithNullableVersion(Map sourceAndMetadata, Map ingestMetadata) { - Map> validators = replaceValidator(VERSION, IngestSourceAndMetadata::longValidator); Tuple, Map> sm = IngestSourceAndMetadata.splitSourceAndMetadata(sourceAndMetadata); - IngestSourceAndMetadata withNullableVersion = new IngestSourceAndMetadata(sm.v1(), sm.v2(), null, validators); - return new IngestDocument(withNullableVersion, ingestMetadata); + return new IngestDocument(new IngestSourceAndMetadata(sm.v1(), TestMetadata.withNullableVersion(sm.v2())), ingestMetadata); } /** @@ -53,22 +52,12 @@ public static IngestDocument withDefaultVersion(Map sourceAndMet return new IngestDocument(sourceAndMetadata, new HashMap<>()); } - /** - * Return the default validator map with a single validator replaced, if that validator was already present in the default validators - * map - */ - protected static Map> replaceValidator(String key, BiConsumer validator) { - Map> validators = new HashMap<>(IngestSourceAndMetadata.VALIDATORS); - validators.computeIfPresent(key, (k, v) -> validator); - return validators; - } - /** * Create an IngestDocument with a metadata map and validators. The metadata map is passed by reference, not copied, so callers * can observe changes to the map directly. */ - public static IngestDocument ofMetadataWithValidator(Map metadata, Map> validators) { - return new IngestDocument(new IngestSourceAndMetadata(new HashMap<>(), metadata, null, validators), new HashMap<>()); + public static IngestDocument ofMetadataWithValidator(Map metadata, Map validators) { + return new IngestDocument(new IngestSourceAndMetadata(new HashMap<>(), new TestMetadata(metadata, validators)), new HashMap<>()); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java new file mode 100644 index 0000000000000..87a45ea857549 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import java.util.HashMap; +import java.util.Map; + +/** + * An implementation of {@link Metadata} with customizable {@link org.elasticsearch.script.Metadata.Validator}s for use in testing. + */ +public class TestMetadata extends Metadata { + public TestMetadata(Map map, Map validators) { + super(map, null, validators); + } + + public static TestMetadata withNullableVersion(Map map) { + Map updatedValidators = new HashMap<>(VALIDATORS); + updatedValidators.replace(VERSION, Metadata::longValidator); + return new TestMetadata(map, updatedValidators); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java index 1663c63eeeae3..58e58bd9eb1f0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResultsTestCase.java @@ -55,14 +55,15 @@ public void testWriteToDocAndSerialize() throws IOException { InferenceResults.writeResult(inferenceResult, document, parentField, modelId); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { builder.startObject(); - Map metadataMap = document.getMetadataMap(); - for (Map.Entry metadata : metadataMap.entrySet()) { - if (metadata.getValue() != null) { - builder.field(metadata.getKey(), metadata.getValue().toString()); + org.elasticsearch.script.Metadata metadata = document.getMetadata(); + for (String key : metadata.keySet()) { + Object value = metadata.get(key); + if (value != null) { + builder.field(key, value.toString()); } } Map source = IngestDocument.deepCopyMap(document.getSourceAndMetadata()); - metadataMap.keySet().forEach(mD -> source.remove(mD)); + metadata.keySet().forEach(source::remove); builder.field("_source", source); builder.field("_ingest", document.getIngestMetadata()); builder.endObject(); From 9c7b3895f3c37a9616c5364be3c7c3369c72294b Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 12 Jul 2022 16:19:33 -0500 Subject: [PATCH 36/40] Remove getIngestSourceAndMetadata --- .../common/ScriptProcessorFactoryTests.java | 2 +- .../elasticsearch/ingest/IngestDocument.java | 7 ----- .../elasticsearch/script/TestMetadata.java | 27 ------------------- 3 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java index 6fb39fa0fb803..7476eb2216dc6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java @@ -159,7 +159,7 @@ public void testInlineIsCompiled() throws Exception { assertThat(processor.getScript().getParams(), equalTo(Collections.emptyMap())); assertNotNull(processor.getPrecompiledIngestScriptFactory()); IngestDocument doc = TestIngestDocument.emptyIngestDocument(); - Map ctx = TestIngestDocument.emptyIngestDocument().getIngestSourceAndMetadata(); + Map ctx = TestIngestDocument.emptyIngestDocument().getSourceAndMetadata(); processor.getPrecompiledIngestScriptFactory().newInstance(null, doc.getMetadata(), ctx).execute(); assertThat(ctx.get("foo"), equalTo("bar")); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 81ecfa39bccd0..c80e477606b0b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -718,13 +718,6 @@ public Map getSourceAndMetadata() { return sourceAndMetadata; } - /** - * Get source and metadata map as {@link IngestCtxMap} - */ - public IngestCtxMap getIngestSourceAndMetadata() { - return sourceAndMetadata; - } - /** * Get the strongly typed metadata */ diff --git a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java b/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java deleted file mode 100644 index 87a45ea857549..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/script/TestMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.script; - -import java.util.HashMap; -import java.util.Map; - -/** - * An implementation of {@link Metadata} with customizable {@link org.elasticsearch.script.Metadata.Validator}s for use in testing. - */ -public class TestMetadata extends Metadata { - public TestMetadata(Map map, Map validators) { - super(map, null, validators); - } - - public static TestMetadata withNullableVersion(Map map) { - Map updatedValidators = new HashMap<>(VALIDATORS); - updatedValidators.replace(VERSION, Metadata::longValidator); - return new TestMetadata(map, updatedValidators); - } -} From 5a45f97a1c5f3a472fcce1a066cfd8ff94651ecb Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 12 Jul 2022 18:13:15 -0500 Subject: [PATCH 37/40] Don't clone properties, add MetadataTests --- .../org/elasticsearch/script/Metadata.java | 5 +- .../elasticsearch/script/MetadataTests.java | 207 ++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index c435dff5ca7fa..6750c79061475 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -52,7 +52,7 @@ public class Metadata { public Metadata(Map map, Map> properties) { this.map = map; - this.properties = properties; + this.properties = Collections.unmodifiableMap(properties); validateMetadata(); } @@ -223,7 +223,8 @@ public int size() { @Override public Metadata clone() { - return new Metadata(new HashMap<>(map), new HashMap<>(properties)); + // properties is an UnmodifiableMap, no need to create a copy + return new Metadata(new HashMap<>(map), properties); } /** diff --git a/server/src/test/java/org/elasticsearch/script/MetadataTests.java b/server/src/test/java/org/elasticsearch/script/MetadataTests.java index 1b3d5574b496b..843c5a6c1c97c 100644 --- a/server/src/test/java/org/elasticsearch/script/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/script/MetadataTests.java @@ -10,11 +10,14 @@ import org.elasticsearch.test.ESTestCase; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; public class MetadataTests extends ESTestCase { Metadata md; + private static final Metadata.FieldProperty STRING_PROP = new Metadata.FieldProperty<>(String.class, true, true, null); public void testGetString() { Map metadata = new HashMap<>(); @@ -57,4 +60,208 @@ private static Map> allowAllValidators(String. } return validators; } + + public void testValidateMetadata() { + IllegalArgumentException err = expectThrows( + IllegalArgumentException.class, + () -> new Metadata(Map.of("foo", 1), Map.of("foo", STRING_PROP)) + ); + assertEquals("foo [1] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); + + err = expectThrows( + IllegalArgumentException.class, + () -> new Metadata(Map.of("foo", "abc", "bar", "def"), Map.of("foo", STRING_PROP)) + ); + assertEquals("Unexpected metadata keys [bar:def]", err.getMessage()); + } + + public void testIsAvailable() { + md = new Metadata(Map.of("bar", "baz"), Map.of("foo", STRING_PROP, "bar", STRING_PROP)); + assertTrue(md.isAvailable("bar")); + assertTrue(md.isAvailable("foo")); + } + + public void testPut() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP)); + assertEquals("bar", md.get("foo")); + md.put("foo", "baz"); + assertEquals("baz", md.get("foo")); + md.put("foo", null); + assertNull(md.get("foo")); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> md.put("foo", 1)); + assertEquals("foo [1] is wrong type, expected assignable to [java.lang.String], not [java.lang.Integer]", err.getMessage()); + } + + public void testContainsKey() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + assertTrue(md.containsKey("foo")); + assertFalse(md.containsKey("baz")); + assertTrue(md.isAvailable("baz")); + } + + public void testContainsValue() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + assertTrue(md.containsValue("bar")); + assertFalse(md.containsValue("foo")); + } + + public void testRemove() { + md = new Metadata( + new HashMap<>(Map.of("foo", "bar", "baz", "qux")), + Map.of("foo", STRING_PROP, "baz", new Metadata.FieldProperty<>(String.class, false, true, null)) + ); + assertTrue(md.containsKey("foo")); + md.remove("foo"); + assertFalse(md.containsKey("foo")); + + assertTrue(md.containsKey("baz")); + IllegalArgumentException err = expectThrows(IllegalArgumentException.class, () -> md.remove("baz")); + assertEquals("baz cannot be removed", err.getMessage()); + } + + public void testKeySetAndSize() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar", "baz", "qux")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + Set expected = Set.of("foo", "baz"); + assertEquals(expected, md.keySet()); + assertEquals(2, md.size()); + md.remove("foo"); + assertEquals(Set.of("baz"), md.keySet()); + assertEquals(1, md.size()); + md.put("foo", "abc"); + assertEquals(expected, md.keySet()); + assertEquals(2, md.size()); + md.remove("foo"); + md.remove("baz"); + assertEquals(Collections.emptySet(), md.keySet()); + assertEquals(0, md.size()); + } + + public void testTestClone() { + md = new Metadata(new HashMap<>(Map.of("foo", "bar", "baz", "qux")), Map.of("foo", STRING_PROP, "baz", STRING_PROP)); + Metadata md2 = md.clone(); + md2.remove("foo"); + md.remove("baz"); + assertEquals("bar", md.get("foo")); + assertNull(md2.get("foo")); + assertNull(md.get("baz")); + assertEquals("qux", md2.get("baz")); + } + + public void testFieldPropertyType() { + Metadata.FieldProperty aProp = new Metadata.FieldProperty<>(A.class, true, true, null); + aProp.check(Metadata.MapOperation.UPDATE, "a", new A()); + aProp.check(Metadata.MapOperation.INIT, "a", new A()); + aProp.check(Metadata.MapOperation.UPDATE, "a", new B()); + aProp.check(Metadata.MapOperation.INIT, "a", new B()); + + IllegalArgumentException err = expectThrows( + IllegalArgumentException.class, + () -> aProp.check(Metadata.MapOperation.UPDATE, "a", new C()) + ); + String expected = "a [I'm C] is wrong type, expected assignable to [org.elasticsearch.script.MetadataTests$A], not" + + " [org.elasticsearch.script.MetadataTests$C]"; + assertEquals(expected, err.getMessage()); + err = expectThrows(IllegalArgumentException.class, () -> aProp.check(Metadata.MapOperation.INIT, "a", new C())); + assertEquals(expected, err.getMessage()); + } + + static class A {} + + static class B extends A {} + + static class C { + @Override + public String toString() { + return "I'm C"; + } + } + + public void testFieldPropertyNullable() { + Metadata.FieldProperty cantNull = new Metadata.FieldProperty<>(String.class, false, true, null); + Metadata.FieldProperty canNull = new Metadata.FieldProperty<>(String.class, true, true, null); + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be null", err.getMessage()); + canNull.check(op, "a", null); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be null", err.getMessage()); + canNull.check(Metadata.MapOperation.UPDATE, "a", null); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + err = expectThrows(IllegalArgumentException.class, () -> cantNull.check(op, "a", null)); + assertEquals("a cannot be removed", err.getMessage()); + canNull.check(Metadata.MapOperation.REMOVE, "a", null); + } + } + + public void testFieldPropertyWritable() { + Metadata.FieldProperty writable = new Metadata.FieldProperty<>(String.class, true, true, null); + Metadata.FieldProperty readonly = new Metadata.FieldProperty<>(String.class, true, false, null); + + String key = "a"; + String value = "abc"; + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + writable.check(op, key, value); + readonly.check(op, key, value); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + err = expectThrows(IllegalArgumentException.class, () -> readonly.check(op, key, value)); + assertEquals("a cannot be updated", err.getMessage()); + writable.check(op, key, value); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + err = expectThrows(IllegalArgumentException.class, () -> readonly.check(op, key, value)); + assertEquals("a cannot be removed", err.getMessage()); + writable.check(op, key, value); + } + } + + public void testFieldPropertyExtendedValidation() { + Metadata.FieldProperty any = new Metadata.FieldProperty<>(Number.class, true, true, null); + Metadata.FieldProperty odd = new Metadata.FieldProperty<>(Number.class, true, true, (k, v) -> { + if (v.intValue() % 2 == 0) { + throw new IllegalArgumentException("not odd"); + } + }); + + String key = "a"; + int value = 2; + + IllegalArgumentException err; + { + Metadata.MapOperation op = Metadata.MapOperation.INIT; + any.check(op, key, value); + err = expectThrows(IllegalArgumentException.class, () -> odd.check(op, key, value)); + assertEquals("not odd", err.getMessage()); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.UPDATE; + any.check(op, key, value); + err = expectThrows(IllegalArgumentException.class, () -> odd.check(op, key, value)); + assertEquals("not odd", err.getMessage()); + } + + { + Metadata.MapOperation op = Metadata.MapOperation.REMOVE; + any.check(op, key, value); + odd.check(op, key, value); + } + } } From dc1fc58051c251aa78456a721affbb5bb67171f3 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Jul 2022 10:01:35 -0500 Subject: [PATCH 38/40] IngestMetadata -> IngestDocMetadata, remove splitSourceAndMetadta, fix validate javadoc --- .../elasticsearch/ingest/IngestCtxMap.java | 76 +--------------- .../ingest/IngestDocMetadata.java | 90 +++++++++++++++++++ .../elasticsearch/ingest/IngestDocument.java | 30 +++---- .../java/org/elasticsearch/script/CtxMap.java | 24 ----- .../org/elasticsearch/script/Metadata.java | 2 +- .../ingest/IngestCtxMapTests.java | 14 +-- .../ingest/TestIngestCtxMetadata.java | 2 +- .../ingest/TestIngestDocument.java | 17 ++-- 8 files changed, 125 insertions(+), 130 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index d1fddd11c3a9c..b648051669567 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -8,16 +8,13 @@ package org.elasticsearch.ingest; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.CtxMap; import org.elasticsearch.script.Metadata; import java.time.ZonedDateTime; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; /** * Map containing ingest source and metadata. @@ -46,7 +43,7 @@ class IngestCtxMap extends CtxMap { ZonedDateTime timestamp, Map source ) { - super(new HashMap<>(source), new IngestMetadata(index, id, version, routing, versionType, timestamp)); + super(new HashMap<>(source), new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); } /** @@ -76,75 +73,4 @@ public static ZonedDateTime getTimestamp(Map ingestMetadata) { return null; } - static class IngestMetadata extends Metadata { - private static final FieldProperty UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null); - static final Map> PROPERTIES = Map.of( - INDEX, - UPDATABLE_STRING, - ID, - UPDATABLE_STRING, - ROUTING, - UPDATABLE_STRING, - VERSION_TYPE, - new FieldProperty<>(String.class, true, true, (k, v) -> { - try { - VersionType.fromString(v); - return; - } catch (IllegalArgumentException ignored) {} - throw new IllegalArgumentException( - k - + " must be a null or one of [" - + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) - + "] but was [" - + v - + "] with type [" - + v.getClass().getName() - + "]" - ); - }), - VERSION, - new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), - TYPE, - new FieldProperty<>(String.class, true, false, null), - IF_SEQ_NO, - new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), - IF_PRIMARY_TERM, - new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), - DYNAMIC_TEMPLATES, - new FieldProperty<>(Map.class, true, true, null) - ); - - protected final ZonedDateTime timestamp; - - IngestMetadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { - this(metadataMap(index, id, version, routing, versionType), timestamp); - } - - IngestMetadata(Map metadata, ZonedDateTime timestamp) { - super(metadata, PROPERTIES); - this.timestamp = timestamp; - } - - /** - * Create the backing metadata map with the standard contents assuming default validators. - */ - protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { - Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); - metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); - metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); - metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); - if (routing != null) { - metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); - } - if (versionType != null) { - metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); - } - return metadata; - } - - @Override - public ZonedDateTime getTimestamp() { - return timestamp; - } - } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java new file mode 100644 index 0000000000000..0897f1a3175e4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocMetadata.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.script.Metadata; + +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +class IngestDocMetadata extends Metadata { + private static final FieldProperty UPDATABLE_STRING = new FieldProperty<>(String.class, true, true, null); + static final Map> PROPERTIES = Map.of( + INDEX, + UPDATABLE_STRING, + ID, + UPDATABLE_STRING, + ROUTING, + UPDATABLE_STRING, + VERSION_TYPE, + new FieldProperty<>(String.class, true, true, (k, v) -> { + try { + VersionType.fromString(v); + return; + } catch (IllegalArgumentException ignored) {} + throw new IllegalArgumentException( + k + + " must be a null or one of [" + + Arrays.stream(VersionType.values()).map(vt -> VersionType.toString(vt)).collect(Collectors.joining(", ")) + + "] but was [" + + v + + "] with type [" + + v.getClass().getName() + + "]" + ); + }), + VERSION, + new FieldProperty<>(Number.class, false, true, FieldProperty.LONGABLE_NUMBER), + TYPE, + new FieldProperty<>(String.class, true, false, null), + IF_SEQ_NO, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + IF_PRIMARY_TERM, + new FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER), + DYNAMIC_TEMPLATES, + new FieldProperty<>(Map.class, true, true, null) + ); + + protected final ZonedDateTime timestamp; + + IngestDocMetadata(String index, String id, long version, String routing, VersionType versionType, ZonedDateTime timestamp) { + this(metadataMap(index, id, version, routing, versionType), timestamp); + } + + IngestDocMetadata(Map metadata, ZonedDateTime timestamp) { + super(metadata, PROPERTIES); + this.timestamp = timestamp; + } + + /** + * Create the backing metadata map with the standard contents assuming default validators. + */ + protected static Map metadataMap(String index, String id, long version, String routing, VersionType versionType) { + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + metadata.put(IngestDocument.Metadata.INDEX.getFieldName(), index); + metadata.put(IngestDocument.Metadata.ID.getFieldName(), id); + metadata.put(IngestDocument.Metadata.VERSION.getFieldName(), version); + if (routing != null) { + metadata.put(IngestDocument.Metadata.ROUTING.getFieldName(), routing); + } + if (versionType != null) { + metadata.put(IngestDocument.Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); + } + return metadata; + } + + @Override + public ZonedDateTime getTimestamp() { + return timestamp; + } +} diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index c80e477606b0b..715ba748e6049 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.util.LazyMap; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; @@ -77,21 +76,22 @@ public IngestDocument(IngestDocument other) { * Constructor to create an IngestDocument from its constituent maps. The maps are shallow copied. */ public IngestDocument(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( - sourceAndMetadata, - Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) - ); - this.sourceAndMetadata = new IngestCtxMap( - sm.v1(), - new IngestCtxMap.IngestMetadata(sm.v2(), IngestCtxMap.getTimestamp(ingestMetadata)) - ); - this.ingestMetadata = new HashMap<>(ingestMetadata); - this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> { - if (v instanceof String) { - return this.sourceAndMetadata.getMetadata().getTimestamp(); + Map source; + Map metadata; + if (sourceAndMetadata instanceof IngestCtxMap ingestCtxMap) { + source = new HashMap<>(ingestCtxMap.getSource()); + metadata = new HashMap<>(ingestCtxMap.getMetadata().getMap()); + } else { + metadata = Maps.newHashMapWithExpectedSize(Metadata.METADATA_NAMES.size()); + source = new HashMap<>(sourceAndMetadata); + for (String key : Metadata.METADATA_NAMES) { + if (sourceAndMetadata.containsKey(key)) { + metadata.put(key, source.remove(key)); + } } - return v; - }); + } + this.ingestMetadata = new HashMap<>(ingestMetadata); + this.sourceAndMetadata = new IngestCtxMap(source, new IngestDocMetadata(metadata, IngestCtxMap.getTimestamp(ingestMetadata))); } /** diff --git a/server/src/main/java/org/elasticsearch/script/CtxMap.java b/server/src/main/java/org/elasticsearch/script/CtxMap.java index 7301052a27a7e..d66514127043a 100644 --- a/server/src/main/java/org/elasticsearch/script/CtxMap.java +++ b/server/src/main/java/org/elasticsearch/script/CtxMap.java @@ -8,9 +8,7 @@ package org.elasticsearch.script; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.core.Tuple; import java.util.AbstractCollection; import java.util.AbstractMap; @@ -52,28 +50,6 @@ protected CtxMap(Map source, Metadata metadata) { } } - /** - * Returns a new metadata map and the existing source map with metadata removed. - */ - public static Tuple, Map> splitSourceAndMetadata( - Map sourceAndMetadata, - Set metadataKeys - ) { - if (sourceAndMetadata instanceof CtxMap ctxMap) { - return new Tuple<>(new HashMap<>(ctxMap.source), new HashMap<>(ctxMap.metadata.getMap())); - } - - Map metadata = Maps.newHashMapWithExpectedSize(metadataKeys.size()); - Map source = new HashMap<>(sourceAndMetadata); - - for (String metadataKey : metadataKeys) { - if (sourceAndMetadata.containsKey(metadataKey)) { - metadata.put(metadataKey, source.remove(metadataKey)); - } - } - return new Tuple<>(source, metadata); - } - /** * get the source map, if externally modified then the guarantees of this class are not enforced */ diff --git a/server/src/main/java/org/elasticsearch/script/Metadata.java b/server/src/main/java/org/elasticsearch/script/Metadata.java index 6750c79061475..f84e6d5502b61 100644 --- a/server/src/main/java/org/elasticsearch/script/Metadata.java +++ b/server/src/main/java/org/elasticsearch/script/Metadata.java @@ -57,7 +57,7 @@ public Metadata(Map map, Map> propertie } /** - * Check that all metadata map contains only valid metadata and no extraneous keys and source map contains no metadata + * Check that all metadata map contains only valid metadata and no extraneous keys */ protected void validateMetadata() { int numMetadata = 0; diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java index 3327d30420478..dab30f6c13b91 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestCtxMapTests.java @@ -36,7 +36,7 @@ public void testSettersAndGetters() { metadata.put("_if_primary_term", 10000); metadata.put("_version_type", "internal"); metadata.put("_dynamic_templates", Map.of("foo", "bar")); - map = new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)); md = map.getMetadata(); assertEquals("myIndex", md.getIndex()); md.setIndex("myIndex2"); @@ -69,7 +69,7 @@ public void testInvalidMetadata() { metadata.put("_version", Double.MAX_VALUE); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)) ); assertThat(err.getMessage(), containsString("_version may only be set to an int or a long but was [")); assertThat(err.getMessage(), containsString("] with type [java.lang.Double]")); @@ -80,7 +80,7 @@ public void testSourceInMetadata() { source.put("_version", 25); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(source, new IngestCtxMap.IngestMetadata(source, null)) + () -> new IngestCtxMap(source, new IngestDocMetadata(source, null)) ); assertEquals("unexpected metadata [_version:25] in source", err.getMessage()); } @@ -92,7 +92,7 @@ public void testExtraMetadata() { metadata.put("routing", "myRouting"); IllegalArgumentException err = expectThrows( IllegalArgumentException.class, - () -> new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(metadata, null)) + () -> new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(metadata, null)) ); assertEquals("Unexpected metadata keys [routing:myRouting, version:567]", err.getMessage()); } @@ -101,7 +101,7 @@ public void testPutSource() { Map metadata = new HashMap<>(); metadata.put("_version", 123); Map source = new HashMap<>(); - map = new IngestCtxMap(source, new IngestCtxMap.IngestMetadata(metadata, null)); + map = new IngestCtxMap(source, new IngestDocMetadata(metadata, null)); } public void testRemove() { @@ -221,7 +221,7 @@ public void testEntryAndIterator() { } public void testContainsValue() { - map = new IngestCtxMap(Map.of("myField", "fieldValue"), new IngestCtxMap.IngestMetadata(Map.of("_version", 5678), null)); + map = new IngestCtxMap(Map.of("myField", "fieldValue"), new IngestDocMetadata(Map.of("_version", 5678), null)); assertTrue(map.containsValue(5678)); assertFalse(map.containsValue(5679)); assertTrue(map.containsValue("fieldValue")); @@ -295,7 +295,7 @@ public void testValidators() { public void testHandlesAllVersionTypes() { Map mdRawMap = new HashMap<>(); mdRawMap.put("_version", 1234); - map = new IngestCtxMap(new HashMap<>(), new IngestCtxMap.IngestMetadata(mdRawMap, null)); + map = new IngestCtxMap(new HashMap<>(), new IngestDocMetadata(mdRawMap, null)); md = map.getMetadata(); assertNull(md.getVersionType()); for (VersionType vt : VersionType.values()) { diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java index 2040fa5ee9322..1e83eaca5c55e 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestCtxMetadata.java @@ -19,7 +19,7 @@ public TestIngestCtxMetadata(Map map, Map map) { - Map> updatedProperties = new HashMap<>(IngestCtxMap.IngestMetadata.PROPERTIES); + Map> updatedProperties = new HashMap<>(IngestDocMetadata.PROPERTIES); updatedProperties.replace(VERSION, new Metadata.FieldProperty<>(Number.class, true, true, FieldProperty.LONGABLE_NUMBER)); return new TestIngestCtxMetadata(map, updatedProperties); } diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java index 9dbb74ea80608..3998cf6db1aa5 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestIngestDocument.java @@ -9,15 +9,14 @@ package org.elasticsearch.ingest; import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.VersionType; import org.elasticsearch.script.Metadata; import org.elasticsearch.test.ESTestCase; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; /** * Construct ingest documents for testing purposes @@ -38,11 +37,15 @@ public static IngestDocument withNullableVersion(Map sourceAndMe * _versions. Normally null _version is not allowed, but many tests don't care about that invariant. */ public static IngestDocument ofIngestWithNullableVersion(Map sourceAndMetadata, Map ingestMetadata) { - Tuple, Map> sm = IngestCtxMap.splitSourceAndMetadata( - sourceAndMetadata, - Arrays.stream(IngestDocument.Metadata.values()).map(IngestDocument.Metadata::getFieldName).collect(Collectors.toSet()) - ); - return new IngestDocument(new IngestCtxMap(sm.v1(), TestIngestCtxMetadata.withNullableVersion(sm.v2())), ingestMetadata); + Map source = new HashMap<>(sourceAndMetadata); + Map metadata = Maps.newHashMapWithExpectedSize(IngestDocument.Metadata.values().length); + for (IngestDocument.Metadata m : IngestDocument.Metadata.values()) { + String key = m.getFieldName(); + if (sourceAndMetadata.containsKey(key)) { + metadata.put(key, source.remove(key)); + } + } + return new IngestDocument(new IngestCtxMap(source, TestIngestCtxMetadata.withNullableVersion(metadata)), ingestMetadata); } /** From a08afb8d9a0d201709ec8594e892dd77162c324e Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Jul 2022 13:03:49 -0500 Subject: [PATCH 39/40] Don't copy source --- .../main/java/org/elasticsearch/ingest/IngestCtxMap.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index b648051669567..e3cdc13e1c11e 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -13,7 +13,6 @@ import org.elasticsearch.script.Metadata; import java.time.ZonedDateTime; -import java.util.HashMap; import java.util.Map; /** @@ -32,7 +31,9 @@ class IngestCtxMap extends CtxMap { /** - * Create an IngestCtxMap with the given metadata, source and default validators + * Create an IngestCtxMap with the given metadata, source and default validators. + * + * Source is not copied, callers can observe changes to source. */ IngestCtxMap( String index, @@ -43,7 +44,7 @@ class IngestCtxMap extends CtxMap { ZonedDateTime timestamp, Map source ) { - super(new HashMap<>(source), new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); + super(source, new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); } /** From 8eb42c6f46f7c8a3590c7039c00800ed416c2d52 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 13 Jul 2022 13:50:16 -0500 Subject: [PATCH 40/40] Revert "Don't copy source" This reverts commit a08afb8d9a0d201709ec8594e892dd77162c324e. --- .../main/java/org/elasticsearch/ingest/IngestCtxMap.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java index e3cdc13e1c11e..b648051669567 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestCtxMap.java @@ -13,6 +13,7 @@ import org.elasticsearch.script.Metadata; import java.time.ZonedDateTime; +import java.util.HashMap; import java.util.Map; /** @@ -31,9 +32,7 @@ class IngestCtxMap extends CtxMap { /** - * Create an IngestCtxMap with the given metadata, source and default validators. - * - * Source is not copied, callers can observe changes to source. + * Create an IngestCtxMap with the given metadata, source and default validators */ IngestCtxMap( String index, @@ -44,7 +43,7 @@ class IngestCtxMap extends CtxMap { ZonedDateTime timestamp, Map source ) { - super(source, new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); + super(new HashMap<>(source), new IngestDocMetadata(index, id, version, routing, versionType, timestamp)); } /**