Skip to content

Commit b2a9d54

Browse files
PeteGillinElasticPeteGillinElastic
committed
Prepend project ID to indices in _nodes/stats API
In multi-project mode, this prepends the project ID to the index name in the indices response in the `_nodes/stats` API, because multiple projects might have the indices with the same name.
1 parent 1dfb70e commit b2a9d54

File tree

20 files changed

+279
-25
lines changed

20 files changed

+279
-25
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,11 @@ public void testIndicesMetrics() {
116116
.findFirst()
117117
.orElseThrow();
118118
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, indexNode);
119-
var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal();
119+
var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false, false).getIndexing().getTotal();
120120
telemetry.resetMeter();
121121
long numStandardIndices = randomIntBetween(1, 5);
122122
long numStandardDocs = populateStandardIndices(numStandardIndices);
123-
var indexing1 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal();
123+
var indexing1 = indicesService.stats(CommonStatsFlags.ALL, false, false).getIndexing().getTotal();
124124
collectThenAssertMetrics(
125125
telemetry,
126126
1,
@@ -145,7 +145,7 @@ public void testIndicesMetrics() {
145145

146146
long numTimeSeriesIndices = randomIntBetween(1, 5);
147147
long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices);
148-
var indexing2 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal();
148+
var indexing2 = indicesService.stats(CommonStatsFlags.ALL, false, false).getIndexing().getTotal();
149149
collectThenAssertMetrics(
150150
telemetry,
151151
2,
@@ -170,7 +170,7 @@ public void testIndicesMetrics() {
170170

171171
long numLogsdbIndices = randomIntBetween(1, 5);
172172
long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices);
173-
var indexing3 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal();
173+
var indexing3 = indicesService.stats(CommonStatsFlags.ALL, false, false).getIndexing().getTotal();
174174
collectThenAssertMetrics(
175175
telemetry,
176176
3,
@@ -240,7 +240,7 @@ public void testIndicesMetrics() {
240240
// search and fetch
241241
String preference = "_only_local";
242242
client(searchNode).prepareSearch("standard*").setPreference(preference).setSize(100).get().decRef();
243-
var search1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal();
243+
var search1 = indicesService.stats(CommonStatsFlags.ALL, false, false).getSearch().getTotal();
244244
collectThenAssertMetrics(
245245
telemetry,
246246
1,
@@ -267,7 +267,7 @@ public void testIndicesMetrics() {
267267
);
268268

269269
client(searchNode).prepareSearch("time*").setPreference(preference).setSize(100).get().decRef();
270-
var search2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal();
270+
var search2 = indicesService.stats(CommonStatsFlags.ALL, false, false).getSearch().getTotal();
271271
collectThenAssertMetrics(
272272
telemetry,
273273
2,
@@ -293,7 +293,7 @@ public void testIndicesMetrics() {
293293
)
294294
);
295295
client(searchNode).prepareSearch("logs*").setPreference(preference).setSize(100).get().decRef();
296-
var search3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal();
296+
var search3 = indicesService.stats(CommonStatsFlags.ALL, false, false).getSearch().getTotal();
297297
collectThenAssertMetrics(
298298
telemetry,
299299
3,

‎server/src/main/java/org/elasticsearch/TransportVersions.java

+1
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ static TransportVersion def(int id) {
228228
public static final TransportVersion DENSE_VECTOR_OFF_HEAP_STATS = def(9_062_00_0);
229229
public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00);
230230
public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
231+
public static final TransportVersion NODES_STATS_SUPPORTS_MULTI_PROJECT = def(9_065_0_00);
231232

232233
/*
233234
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java

+7-4
Original file line numberDiff line numberDiff line change
@@ -156,14 +156,17 @@ public String getDescription() {
156156
};
157157
}
158158

159-
public boolean includeShardsStats() {
160-
return nodesStatsRequestParameters.includeShardsStats();
161-
}
162-
163159
public void setIncludeShardsStats(boolean includeShardsStats) {
164160
nodesStatsRequestParameters.setIncludeShardsStats(includeShardsStats);
165161
}
166162

163+
/**
164+
* Sets whether to include the project IDs in the {@link NodeStats} if this ES instance is multi-project.
165+
*/
166+
public void setIncludeProjectIdsIfMultiProject(boolean includeProjectIdsIfMultiProject) {
167+
nodesStatsRequestParameters.setIncludeProjectIdsIfMultiProject(includeProjectIdsIfMultiProject);
168+
}
169+
167170
public NodesStatsRequestParameters getNodesStatsRequestParameters() {
168171
return nodesStatsRequestParameters;
169172
}

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestParameters.java

+23
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public class NodesStatsRequestParameters implements Writeable {
3131
private CommonStatsFlags indices = new CommonStatsFlags();
3232
private final EnumSet<Metric> requestedMetrics;
3333
private boolean includeShardsStats = true;
34+
private boolean includeProjectIdsIfMultiProject = false;
3435

3536
public NodesStatsRequestParameters() {
3637
this.requestedMetrics = EnumSet.noneOf(Metric.class);
@@ -44,6 +45,11 @@ public NodesStatsRequestParameters(StreamInput in) throws IOException {
4445
} else {
4546
includeShardsStats = true;
4647
}
48+
if (in.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_SUPPORTS_MULTI_PROJECT)) {
49+
includeProjectIdsIfMultiProject = in.readBoolean();
50+
} else {
51+
includeProjectIdsIfMultiProject = false;
52+
}
4753
}
4854

4955
@Override
@@ -53,6 +59,9 @@ public void writeTo(StreamOutput out) throws IOException {
5359
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
5460
out.writeBoolean(includeShardsStats);
5561
}
62+
if (out.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_SUPPORTS_MULTI_PROJECT)) {
63+
out.writeBoolean(includeProjectIdsIfMultiProject);
64+
}
5665
}
5766

5867
public CommonStatsFlags indices() {
@@ -75,6 +84,20 @@ public void setIncludeShardsStats(boolean includeShardsStats) {
7584
this.includeShardsStats = includeShardsStats;
7685
}
7786

87+
/**
88+
* Returns whether to include the project IDs in the {@link NodeStats} if this ES instance is multi-project.
89+
*/
90+
public boolean includeProjectIdsIfMultiProject() {
91+
return includeProjectIdsIfMultiProject;
92+
}
93+
94+
/**
95+
* Sets whether to include the project IDs in the {@link NodeStats} if this ES instance is multi-project.
96+
*/
97+
public void setIncludeProjectIdsIfMultiProject(boolean includeProjectIdsIfMultiProject) {
98+
this.includeProjectIdsIfMultiProject = includeProjectIdsIfMultiProject;
99+
}
100+
78101
/**
79102
* An enumeration of the "core" sections of metrics that may be requested
80103
* from the nodes stats endpoint. Eventually this list will be pluggable.

‎server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java

+1
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ protected NodeStats nodeOperation(NodeStatsRequest request, Task task) {
161161
return nodeService.stats(
162162
nodesStatsRequestParameters.indices(),
163163
nodesStatsRequestParameters.includeShardsStats(),
164+
nodesStatsRequestParameters.includeProjectIdsIfMultiProject(),
164165
metrics.contains(Metric.OS),
165166
metrics.contains(Metric.PROCESS),
166167
metrics.contains(Metric.JVM),

‎server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

+1
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
241241
NodeStats nodeStats = nodeService.stats(
242242
CommonStatsFlags.NONE,
243243
false,
244+
false,
244245
true,
245246
true,
246247
true,

‎server/src/main/java/org/elasticsearch/health/node/tracker/DiskHealthTracker.java

+1
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ private DiskUsage getDiskUsage() {
111111
false,
112112
false,
113113
false,
114+
false,
114115
true,
115116
false,
116117
false,

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java

+17-2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.cluster.metadata.IndexMetadata;
4040
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
4141
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
42+
import org.elasticsearch.cluster.metadata.ProjectId;
4243
import org.elasticsearch.cluster.metadata.ProjectMetadata;
4344
import org.elasticsearch.cluster.node.DiscoveryNode;
4445
import org.elasticsearch.cluster.project.ProjectResolver;
@@ -83,6 +84,7 @@
8384
import org.elasticsearch.env.ShardLockObtainFailedException;
8485
import org.elasticsearch.gateway.MetaStateService;
8586
import org.elasticsearch.gateway.MetadataStateFormat;
87+
import org.elasticsearch.index.AbstractIndexComponent;
8688
import org.elasticsearch.index.CloseUtils;
8789
import org.elasticsearch.index.Index;
8890
import org.elasticsearch.index.IndexMode;
@@ -466,7 +468,7 @@ public boolean awaitClose(long timeout, TimeUnit timeUnit) throws InterruptedExc
466468
return closeLatch.await(timeout, timeUnit);
467469
}
468470

469-
public NodeIndicesStats stats(CommonStatsFlags flags, boolean includeShardsStats) {
471+
public NodeIndicesStats stats(CommonStatsFlags flags, boolean includeShardsStats, boolean includeProjectIdsIfMultiProject) {
470472
CommonStats commonStats = new CommonStats(flags);
471473
// the cumulative statistics also account for shards that are no longer on this node, which is tracked by oldShardsStats
472474
for (Flag flag : flags.getFlags()) {
@@ -482,7 +484,13 @@ public NodeIndicesStats stats(CommonStatsFlags flags, boolean includeShardsStats
482484
}
483485
}
484486

485-
return new NodeIndicesStats(commonStats, statsByIndex(this, flags), statsByShard(this, flags), includeShardsStats);
487+
return new NodeIndicesStats(
488+
commonStats,
489+
statsByIndex(this, flags),
490+
statsByShard(this, flags),
491+
(includeProjectIdsIfMultiProject && projectResolver.supportsMultipleProjects()) ? projectsByIndex() : null,
492+
includeShardsStats
493+
);
486494
}
487495

488496
static Map<Index, CommonStats> statsByIndex(final IndicesService indicesService, final CommonStatsFlags flags) {
@@ -564,6 +572,13 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index
564572
);
565573
}
566574

575+
private Map<Index, ProjectId> projectsByIndex() {
576+
return indices.values()
577+
.stream()
578+
.map(AbstractIndexComponent::index)
579+
.collect(Collectors.toMap(index -> index, index -> clusterService.state().metadata().projectFor(index).id()));
580+
}
581+
567582
/**
568583
* Checks if changes (adding / removing) indices, shards and so on are allowed.
569584
*

‎server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java

+40-4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
import org.elasticsearch.action.admin.indices.stats.CommonStats;
1616
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
1717
import org.elasticsearch.action.admin.indices.stats.ShardStats;
18+
import org.elasticsearch.cluster.metadata.Metadata;
19+
import org.elasticsearch.cluster.metadata.ProjectId;
1820
import org.elasticsearch.common.collect.Iterators;
1921
import org.elasticsearch.common.io.stream.StreamInput;
2022
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -66,6 +68,7 @@ public class NodeIndicesStats implements Writeable, ChunkedToXContent {
6668
private final CommonStats stats;
6769
private final Map<Index, List<IndexShardStats>> statsByShard;
6870
private final Map<Index, CommonStats> statsByIndex;
71+
private final Map<Index, ProjectId> projectsByIndex;
6972

7073
public NodeIndicesStats(StreamInput in) throws IOException {
7174
stats = new CommonStats(in);
@@ -87,12 +90,24 @@ public NodeIndicesStats(StreamInput in) throws IOException {
8790
} else {
8891
statsByIndex = new HashMap<>();
8992
}
93+
94+
if (in.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_SUPPORTS_MULTI_PROJECT)) {
95+
boolean hasProjectsByIndex = in.readBoolean();
96+
projectsByIndex = hasProjectsByIndex ? in.readMap(Index::new, ProjectId::readFrom) : null;
97+
} else {
98+
projectsByIndex = null;
99+
}
90100
}
91101

102+
/**
103+
* Constructs an instance. If {@code projectsByIndex} argument is non-null, then the index-to-project map will be stored, and the
104+
* project IDs will be prepended to the index names when converting this instance to XContent (except when it is the default project).
105+
*/
92106
public NodeIndicesStats(
93107
CommonStats oldStats,
94108
Map<Index, CommonStats> statsByIndex,
95109
Map<Index, List<IndexShardStats>> statsByShard,
110+
@Nullable Map<Index, ProjectId> projectsByIndex,
96111
boolean includeShardsStats
97112
) {
98113
if (includeShardsStats) {
@@ -114,6 +129,7 @@ public NodeIndicesStats(
114129
for (CommonStats indexStats : statsByIndex.values()) {
115130
stats.add(indexStats);
116131
}
132+
this.projectsByIndex = projectsByIndex;
117133
}
118134

119135
@Nullable
@@ -228,19 +244,28 @@ public void writeTo(StreamOutput out) throws IOException {
228244
if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_STATS_BY_INDEX)) {
229245
out.writeMap(statsByIndex);
230246
}
247+
if (out.getTransportVersion().onOrAfter(TransportVersions.NODES_STATS_SUPPORTS_MULTI_PROJECT)) {
248+
out.writeBoolean(projectsByIndex != null);
249+
if (projectsByIndex != null) {
250+
out.writeMap(projectsByIndex);
251+
}
252+
}
231253
}
232254

233255
@Override
234256
public boolean equals(Object o) {
235257
if (this == o) return true;
236258
if (o == null || getClass() != o.getClass()) return false;
237259
NodeIndicesStats that = (NodeIndicesStats) o;
238-
return stats.equals(that.stats) && statsByShard.equals(that.statsByShard) && statsByIndex.equals(that.statsByIndex);
260+
return stats.equals(that.stats)
261+
&& statsByShard.equals(that.statsByShard)
262+
&& statsByIndex.equals(that.statsByIndex)
263+
&& Objects.equals(projectsByIndex, that.projectsByIndex);
239264
}
240265

241266
@Override
242267
public int hashCode() {
243-
return Objects.hash(stats, statsByShard, statsByIndex);
268+
return Objects.hash(stats, statsByShard, statsByIndex, projectsByIndex);
244269
}
245270

246271
@Override
@@ -260,7 +285,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
260285
case INDICES -> ChunkedToXContentHelper.object(
261286
Fields.INDICES,
262287
Iterators.map(createCommonStatsByIndex().entrySet().iterator(), entry -> (builder, params) -> {
263-
builder.startObject(entry.getKey().getName());
288+
builder.startObject(xContentKey(entry.getKey()));
264289
entry.getValue().toXContent(builder, outerParams);
265290
return builder.endObject();
266291
})
@@ -271,7 +296,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
271296
Iterators.flatMap(
272297
statsByShard.entrySet().iterator(),
273298
entry -> ChunkedToXContentHelper.array(
274-
entry.getKey().getName(),
299+
xContentKey(entry.getKey()),
275300
Iterators.flatMap(
276301
entry.getValue().iterator(),
277302
indexShardStats -> Iterators.concat(
@@ -291,6 +316,17 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
291316
);
292317
}
293318

319+
private String xContentKey(Index index) {
320+
if (projectsByIndex == null) {
321+
return index.getName();
322+
}
323+
ProjectId projectId = projectsByIndex.get(index);
324+
if (Objects.equals(projectId, Metadata.DEFAULT_PROJECT_ID)) {
325+
return index.getName();
326+
}
327+
return projectId + "/" + index.getName();
328+
}
329+
294330
private Map<Index, CommonStats> createCommonStatsByIndex() {
295331
Map<Index, CommonStats> statsMap = new HashMap<>();
296332

‎server/src/main/java/org/elasticsearch/monitor/metrics/NodeMetrics.java

+1
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,7 @@ private NodeStats getNodeStats() {
750750
true,
751751
false,
752752
false,
753+
false,
753754
true,
754755
false,
755756
true,

‎server/src/main/java/org/elasticsearch/node/NodeService.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ private static Map<String, Integer> findComponentVersions(PluginsService pluginS
164164
public NodeStats stats(
165165
CommonStatsFlags indices,
166166
boolean includeShardsStats,
167+
boolean includeProjectIdsIfMultiProject,
167168
boolean os,
168169
boolean process,
169170
boolean jvm,
@@ -185,7 +186,7 @@ public NodeStats stats(
185186
return new NodeStats(
186187
transportService.getLocalNode(),
187188
System.currentTimeMillis(),
188-
indices.anySet() ? indicesService.stats(indices, includeShardsStats) : null,
189+
indices.anySet() ? indicesService.stats(indices, includeShardsStats, includeProjectIdsIfMultiProject) : null,
189190
os ? monitorService.osService().stats() : null,
190191
process ? monitorService.processService().stats() : null,
191192
jvm ? monitorService.jvmService().stats() : null,

‎server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesStatsAction.java

+3
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
8383
// level parameter validation
8484
nodesStatsRequest.setIncludeShardsStats(NodeStatsLevel.of(request, NodeStatsLevel.NODE) != NodeStatsLevel.NODE);
8585

86+
// In multi-project systems, this API should include the project IDs in the response where relevant:
87+
nodesStatsRequest.setIncludeProjectIdsIfMultiProject(true);
88+
8689
if (metricNames.size() == 1 && metricNames.contains("_all")) {
8790
if (request.hasParam("index_metric")) {
8891
throw new IllegalArgumentException(

‎server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.cluster.coordination.ClusterStateSerializationStats;
1818
import org.elasticsearch.cluster.coordination.PendingClusterStateStats;
1919
import org.elasticsearch.cluster.coordination.PublishClusterStateStats;
20+
import org.elasticsearch.cluster.metadata.ProjectId;
2021
import org.elasticsearch.cluster.node.DiscoveryNode;
2122
import org.elasticsearch.cluster.node.DiscoveryNodeUtils;
2223
import org.elasticsearch.cluster.routing.RecoverySource;
@@ -688,7 +689,8 @@ public static NodeStats createNodeStats() {
688689
statsByShard.put(indexTest, indexShardStats);
689690

690691
CommonStats oldStats = new CommonStats(CommonStatsFlags.ALL);
691-
nodeIndicesStats = new NodeIndicesStats(oldStats, statsByIndex, statsByShard, true);
692+
Map<Index, ProjectId> projectsByIndex = randomBoolean() ? Map.of(indexTest, randomProjectIdOrDefault()) : null;
693+
nodeIndicesStats = new NodeIndicesStats(oldStats, statsByIndex, statsByShard, projectsByIndex, true);
692694
}
693695
OsStats osStats = null;
694696
if (frequently()) {

0 commit comments

Comments
 (0)