Script: Metadata for update context

build-tools/src/integTest/groovy/org/elasticsearch/gradle/LoggedExecFuncTest.groovy

@@ -35,7 +35,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest {
         import org.elasticsearch.gradle.LoggedExec
         tasks.register('loggedExec', LoggedExec) {
           commandLine 'ls', '-lh'
-          spoolOutput = $spooling
+          getSpoolOutput().set($spooling)
         }
         """
         when:
@@ -54,7 +54,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest {
         import org.elasticsearch.gradle.LoggedExec
         tasks.register('loggedExec', LoggedExec) {
           commandLine 'ls', 'wtf'
-          spoolOutput = $spooling
+          getSpoolOutput().set($spooling)        
         }
         """
         when:
@@ -97,7 +97,7 @@ class LoggedExecFuncTest extends AbstractGradleFuncTest {
         tasks.register('loggedExec', LoggedExec) {
           commandLine 'echo', 'HELLO'
           getCaptureOutput().set(true)
-          spoolOutput = true
+          getSpoolOutput().set(true)
         }
         """
         when:

build-tools/src/main/java/org/elasticsearch/gradle/LoggedExec.java

@@ -56,7 +56,6 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     protected FileSystemOperations fileSystemOperations;
     private ProjectLayout projectLayout;
     private ExecOperations execOperations;
-    private boolean spoolOutput;
 
     @Input
     @Optional
@@ -84,6 +83,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
     @Input
     abstract public Property<File> getWorkingDir();
 
+    @Internal
+    abstract public Property<Boolean> getSpoolOutput();
+
     private String output;
 
     @Inject
@@ -95,14 +97,16 @@ public LoggedExec(ProjectLayout projectLayout, ExecOperations execOperations, Fi
         // For now mimic default behaviour of Gradle Exec task here
         getEnvironment().putAll(System.getenv());
         getCaptureOutput().convention(false);
+        getSpoolOutput().convention(false);
     }
 
     @TaskAction
     public void run() {
+        boolean spoolOutput = getSpoolOutput().get();
         if (spoolOutput && getCaptureOutput().get()) {
             throw new GradleException("Capturing output is not supported when spoolOutput is true.");
         }
-        if (getCaptureOutput().getOrElse(false) && getIndentingConsoleOutput().isPresent()) {
+        if (getCaptureOutput().get() && getIndentingConsoleOutput().isPresent()) {
             throw new GradleException("Capturing output is not supported when indentingConsoleOutput is configured.");
         }
         Consumer<Logger> outputLogger;
@@ -156,7 +160,9 @@ public void run() {
         if (getLogger().isInfoEnabled() == false) {
             if (exitValue != 0) {
                 try {
-                    getLogger().error("Output for " + getExecutable().get() + ":");
+                    if (getIndentingConsoleOutput().isPresent() == false) {
+                        getLogger().error("Output for " + getExecutable().get() + ":");
+                    }
                     outputLogger.accept(getLogger());
                 } catch (Exception e) {
                     throw new GradleException("Failed to read exec output", e);
@@ -173,10 +179,6 @@ private String byteStreamToString(OutputStream out) {
         return ((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8);
     }
 
-    public void setSpoolOutput(boolean spoolOutput) {
-        this.spoolOutput = spoolOutput;
-    }
-
     public static ExecResult exec(ExecOperations execOperations, Action<ExecSpec> action) {
         return genericExec(execOperations::exec, action);
     }

docs/changelog/88257.yaml

@@ -0,0 +1,6 @@
+pr: 88257
+summary: INFO logging of snapshot restore and completion
+area: Snapshot/Restore
+type: enhancement
+issues:
+ - 86610

docs/changelog/88276.yaml

@@ -0,0 +1,5 @@
+pr: 88276
+summary: Updatable API keys - logging audit trail event
+area: Audit
+type: enhancement
+issues: []

docs/changelog/88333.yaml

@@ -0,0 +1,5 @@
+pr: 88333
+summary: "Script: Metadata for update context"
+area: Infra/Scripting
+type: enhancement
+issues: []

docs/changelog/88346.yaml

@@ -0,0 +1,5 @@
+pr: 88346
+summary: Updatable API keys - noop check
+area: Security
+type: enhancement
+issues: []

docs/changelog/88399.yaml

@@ -0,0 +1,6 @@
+pr: 88399
+summary: Improve error when sorting on incompatible types
+area: Search
+type: enhancement
+issues:
+ - 73146

docs/changelog/88456.yaml

@@ -0,0 +1,5 @@
+pr: 88456
+summary: Audit API key ID when create or grant API keys
+area: Audit
+type: enhancement
+issues: []

docs/reference/aggregations/metrics/avg-aggregation.asciidoc

@@ -4,7 +4,7 @@
 <titleabbrev>Avg</titleabbrev>
 ++++
 
-A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric fields in the documents.
+A `single-value` metrics aggregation that computes the average of numeric values that are extracted from the aggregated documents. These values can be extracted either from specific numeric or <<histogram,histogram>> fields in the documents.
 
 Assuming the data consists of documents representing exams grades (between 0
 and 100) of students we can average their scores with:

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java

@@ -140,11 +140,11 @@ public void testRenameAtomicOperationSetFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
         metadata.put("list", Collections.singletonList("item"));
 
-        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (k, v) -> {
+        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("new_field", (o, k, v) -> {
             if (v != null) {
                 throw new UnsupportedOperationException();
             }
-        }, "list", (k, v) -> {}));
+        }, "list", (o, k, v) -> {}));
         Processor processor = createRenameProcessor("list", "new_field", false);
         try {
             processor.execute(ingestDocument);
@@ -160,7 +160,7 @@ public void testRenameAtomicOperationRemoveFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
         metadata.put("list", Collections.singletonList("item"));
 
-        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (k, v) -> {
+        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(metadata, Map.of("list", (o, k, v) -> {
             if (v == null) {
                 throw new UnsupportedOperationException();
             }

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.update.txt

@@ -18,3 +18,17 @@ class org.elasticsearch.painless.api.Json {
   String dump(def)
   String dump(def, boolean)
 }
+
+class org.elasticsearch.script.Metadata {
+    String getIndex()
+    String getId()
+    String getRouting()
+    long getVersion()
+    String getOp()
+    void setOp(String)
+    ZonedDateTime getTimestamp()
+}
+
+class org.elasticsearch.script.UpdateScript {
+    Metadata metadata()
+}

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml

@@ -123,3 +123,47 @@
   - match: { error.root_cause.0.type: "illegal_argument_exception" }
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "Iterable object is self-referencing itself" }
+
+---
+"Script Update Metadata":
+  - skip:
+      version: " - 8.3.99"
+      reason: "update metadata introduced in 8.4.0"
+
+  - do:
+      update:
+        index:    test_1
+        id:       "2"
+        body:
+          script:
+            source: "ctx._source.bar = metadata().id + '-extra'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      get:
+        index: test_1
+        id: "2"
+
+  - match: { _source.bar: 2-extra }
+  - match: { found: true }
+
+  - do:
+      update:
+        index:    test_1
+        id:       "2"
+        body:
+          script:
+            source: "metadata().op = 'delete'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      catch: missing
+      get:
+        index: test_1
+        id: "2"
+
+  - match: { found: false }

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml

@@ -93,3 +93,97 @@
         id: "4"
 
   - match: { _source.within_one_minute: true }
+
+---
+"Script Upsert Metadata":
+  - skip:
+      version: " - 8.3.99"
+      reason: "update metadata introduced in 8.4.0"
+
+  - do:
+      catch: /routing is unavailable for insert/
+      update:
+        index:    test_1
+        id:       "1"
+        body:
+          script:
+            source: "ctx._source.foo = metadata().routing"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      update:
+        index:    test_1
+        id:       "2"
+        body:
+          script:
+            source: "ctx._source.foo = metadata().index + '_1'; ctx._source.bar = 'nothing'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      get:
+        index: test_1
+        id: "2"
+
+  - match: { _source.foo: test_1_1 }
+  - match: { _source.bar: nothing }
+
+  - do:
+      update:
+        index:    test_1
+        id:       "3"
+        body:
+          script:
+            source: "metadata().op = 'noop'; ctx._source.bar = 'skipped?'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      catch: missing
+      get:
+        index: test_1
+        id: "3"
+
+  - match: { found: false }
+
+  - do:
+      update:
+        index:    test_1
+        id:       "3"
+        body:
+          script:
+            source: "metadata().op = 'create'; ctx._source.bar = 'skipped?'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      get:
+        index: test_1
+        id: "3"
+
+  - match: { found: true }
+  - match: { _source.bar: "skipped?" }
+
+  # update
+  - do:
+      update:
+        index:    test_1
+        id:       "2"
+        body:
+          script:
+            source: "ctx._source.bar = metadata().op + '-extra'"
+            lang: "painless"
+          upsert: {}
+          scripted_upsert: true
+
+  - do:
+      get:
+        index: test_1
+        id: "2"
+
+  - match: { _source.bar: index-extra }

modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/GeoBoundingBoxQueryLegacyGeoShapeIT.java

@@ -22,11 +22,6 @@
 
 public class GeoBoundingBoxQueryLegacyGeoShapeIT extends GeoBoundingBoxQueryIntegTestCase {
 
-    @Override
-    protected boolean addMockGeoShapeFieldMapper() {
-        return false;
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class);

modules/legacy-geo/src/internalClusterTest/java/org/elasticsearch/legacygeo/search/LegacyGeoShapeIT.java

@@ -28,11 +28,6 @@
 
 public class LegacyGeoShapeIT extends GeoShapeIntegTestCase {
 
-    @Override
-    protected boolean addMockGeoShapeFieldMapper() {
-        return false;
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(TestLegacyGeoShapeFieldMapperPlugin.class);

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureQueryBuilderTests.java

@@ -19,7 +19,6 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -54,7 +53,7 @@ protected void initializeAdditionalMappings(MapperService mapperService) throws
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Arrays.asList(MapperExtrasPlugin.class, TestGeoShapeFieldMapperPlugin.class);
+        return Arrays.asList(MapperExtrasPlugin.class);
     }
 
     @Override

modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenTests.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -20,7 +19,7 @@ public class ChildrenTests extends BaseAggregationTestCase<ChildrenAggregationBu
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class);
+        return Arrays.asList(ParentJoinPlugin.class);
     }
 
     @Override

modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ParentTests.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.join.ParentJoinPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -20,7 +19,7 @@ public class ParentTests extends BaseAggregationTestCase<ParentAggregationBuilde
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return Arrays.asList(ParentJoinPlugin.class, TestGeoShapeFieldMapperPlugin.class);
+        return Arrays.asList(ParentJoinPlugin.class);
     }
 
     @Override