From 20cd372ff8c37f79b4d5ab1356c563afd3731c51 Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Wed, 31 May 2023 17:43:30 +0200 Subject: [PATCH 01/10] Mapper: Improve numeric field exact searches. In UnsignedLongFieldMapper we allow the use of docValues in order to improve performances in the case where the query is combined with more restrictive filters --- .../unsignedlong/UnsignedLongFieldMapper.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 04554b577b481..54b7ca6c2788c 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -255,7 +255,13 @@ public Query termQuery(Object value, SearchExecutionContext context) { if (longValue == null) { return new MatchNoDocsQuery(); } - return LongPoint.newExactQuery(name(), unsignedToSortableSignedLong(longValue)); + long lValue = unsignedToSortableSignedLong(longValue); + Query query = LongPoint.newExactQuery(name(), lValue); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), lValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -263,10 +269,19 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexed(); long[] lvalues = new long[values.size()]; int upTo = 0; + long lowerValue = Long.MAX_VALUE; + long upperValue = Long.MIN_VALUE; for (Object value : values) { Long longValue = parseTerm(value); if (longValue != null) { - lvalues[upTo++] = unsignedToSortableSignedLong(longValue); + long currentValue = unsignedToSortableSignedLong(longValue); + lvalues[upTo++] = currentValue; + if (currentValue > upperValue) { + upperValue = currentValue; + } + if (currentValue < lowerValue) { + lowerValue = currentValue; + } } } if (upTo == 0) { @@ -275,7 +290,15 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { if (upTo != lvalues.length) { lvalues = Arrays.copyOf(lvalues, upTo); } - return LongPoint.newSetQuery(name(), lvalues); + Query query = LongPoint.newSetQuery(name(), lvalues); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), lowerValue, upperValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), lowerValue, upperValue, query); + } + } + return query; } @Override From 735f59d83ccc7e6fcf8f06029721a65d25db58cb Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Tue, 27 Jun 2023 13:46:21 +0200 Subject: [PATCH 02/10] WIP passing hasDocValues in term(s)Query for NumberField --- .../index/mapper/NumberFieldMapper.java | 88 +++++++++++++------ 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7590fd36d849a..d971a4d8b8f3f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -327,7 +327,7 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException { } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { float v = parseToFloat(value); if (isIndexed) { return HalfFloatPoint.newExactQuery(field, v); @@ -337,13 +337,28 @@ public Query termQuery(String field, Object value, boolean isIndexed) { } @Override - public Query termsQuery(String field, Collection values) { + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; + float lowerValue = Long.MAX_VALUE; + float upperValue = Long.MIN_VALUE; for (Object value : values) { - v[pos++] = parseToFloat(value); + float floatValue = parseToFloat(value); + v[pos++] = floatValue; + if (floatValue > upperValue) { + upperValue = floatValue; + } + if (floatValue < lowerValue) { + lowerValue = floatValue; + } } - return HalfFloatPoint.newSetQuery(field, v); + + Query query = HalfFloatPoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = FloatField.newRangeQuery(name(), lowerValue, upperValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -480,7 +495,7 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException { } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { float v = parse(value, false); if (isIndexed) { return FloatPoint.newExactQuery(field, v); @@ -490,7 +505,7 @@ public Query termQuery(String field, Object value, boolean isIndexed) { } @Override - public Query termsQuery(String field, Collection values) { + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; for (Object value : values) { @@ -626,7 +641,7 @@ public FieldValues compile(String fieldName, Script script, ScriptCompil } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { double v = parse(value, false); if (isIndexed) { return DoublePoint.newExactQuery(field, v); @@ -636,7 +651,7 @@ public Query termQuery(String field, Object value, boolean isIndexed) { } @Override - public Query termsQuery(String field, Collection values) { + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { double[] v = values.stream().mapToDouble(value -> parse(value, false)).toArray(); return DoublePoint.newSetQuery(field, v); } @@ -761,13 +776,13 @@ public Short parse(XContentParser parser, boolean coerce) throws IOException { } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { - return INTEGER.termQuery(field, value, isIndexed); + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { + return INTEGER.termQuery(field, value, isIndexed, hasDocValues); } @Override - public Query termsQuery(String field, Collection values) { - return INTEGER.termsQuery(field, values); + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { + return INTEGER.termsQuery(field, values, isIndexed, hasDocValues); } @Override @@ -850,13 +865,13 @@ public Short parse(XContentParser parser, boolean coerce) throws IOException { } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { - return INTEGER.termQuery(field, value, isIndexed); + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { + return INTEGER.termQuery(field, value, isIndexed, hasDocValues); } @Override - public Query termsQuery(String field, Collection values) { - return INTEGER.termsQuery(field, values); + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { + return INTEGER.termsQuery(field, values, isIndexed, hasDocValues); } @Override @@ -939,7 +954,7 @@ public Integer parse(XContentParser parser, boolean coerce) throws IOException { } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { if (hasDecimalPart(value)) { return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); } @@ -952,13 +967,22 @@ public Query termQuery(String field, Object value, boolean isIndexed) { } @Override - public Query termsQuery(String field, Collection values) { + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { int[] v = new int[values.size()]; int upTo = 0; + int lowerValue = Integer.MAX_VALUE; + int upperValue = Integer.MIN_VALUE; for (Object value : values) { if (hasDecimalPart(value) == false) { - v[upTo++] = parse(value, true); + int currentValue = parse(value, true); + v[upTo++] = currentValue; + if (currentValue > upperValue) { + upperValue = currentValue; + } + if (currentValue < lowerValue) { + lowerValue = currentValue; + } } } @@ -968,7 +992,19 @@ public Query termsQuery(String field, Collection values) { if (upTo != v.length) { v = Arrays.copyOf(v, upTo); } - return IntPoint.newSetQuery(field, v); +// return IntPoint.newSetQuery(field, v); + + Query query; + if (isIndexed) { + query = IntPoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + } + return query; } @Override @@ -1091,7 +1127,7 @@ public FieldValues compile(String fieldName, Script script, ScriptCompil } @Override - public Query termQuery(String field, Object value, boolean isIndexed) { + public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { if (hasDecimalPart(value)) { return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); } @@ -1104,7 +1140,7 @@ public Query termQuery(String field, Object value, boolean isIndexed) { } @Override - public Query termsQuery(String field, Collection values) { + public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { long[] v = new long[values.size()]; int upTo = 0; @@ -1228,9 +1264,9 @@ public final TypeParser parser() { return parser; } - public abstract Query termQuery(String field, Object value, boolean isIndexed); + public abstract Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues); - public abstract Query termsQuery(String field, Collection values); + public abstract Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues); public abstract Query rangeQuery( String field, @@ -1552,14 +1588,14 @@ public boolean isSearchable() { @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); - return type.termQuery(name(), value, isIndexed()); + return type.termQuery(name(), value, isIndexed(), hasDocValues()); } @Override public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); if (isIndexed()) { - return type.termsQuery(name(), values); + return type.termsQuery(name(), values, isIndexed(), hasDocValues()); } else { return super.termsQuery(values, context); } From dadf9e33c424ec510862a096a3db2e8989fe7f66 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:44:35 +0200 Subject: [PATCH 03/10] allow NumberFieldMapper exact searches to use docValues --- .../mapper/extras/ScaledFloatFieldMapper.java | 9 +- .../index/mapper/NumberFieldMapper.java | 166 +++++++++++++----- 2 files changed, 129 insertions(+), 46 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index bd45ff7a19bfe..f36cb3ab5d9f1 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -257,7 +257,7 @@ public boolean isSearchable() { public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); long scaledValue = Math.round(scale(value)); - return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed()); + return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed(), hasDocValues()); } @Override @@ -269,7 +269,12 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { long scaledValue = Math.round(scale(value)); scaledValues.add(scaledValue); } - return NumberFieldMapper.NumberType.LONG.termsQuery(name(), Collections.unmodifiableList(scaledValues)); + return NumberFieldMapper.NumberType.LONG.termsQuery( + name(), + Collections.unmodifiableList(scaledValues), + isIndexed(), + hasDocValues() + ); } else { return super.termsQuery(values, context); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index d971a4d8b8f3f..c553ade11aa02 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -328,11 +328,16 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException { @Override public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { - float v = parseToFloat(value); + float fValue = parseToFloat(value); if (isIndexed) { - return HalfFloatPoint.newExactQuery(field, v); + Query query = HalfFloatPoint.newExactQuery(field, fValue); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), NumericUtils.floatToSortableInt(fValue)); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } else { - return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v)); + return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(fValue)); } } @@ -340,22 +345,26 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; - float lowerValue = Long.MAX_VALUE; - float upperValue = Long.MIN_VALUE; + float lowerValue = Float.MAX_VALUE; + float upperValue = Float.MIN_VALUE; for (Object value : values) { - float floatValue = parseToFloat(value); - v[pos++] = floatValue; - if (floatValue > upperValue) { - upperValue = floatValue; + float fValue = parseToFloat(value); + v[pos++] = fValue; + if (fValue > upperValue) { + upperValue = fValue; } - if (floatValue < lowerValue) { - lowerValue = floatValue; + if (fValue < lowerValue) { + lowerValue = fValue; } } Query query = HalfFloatPoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = FloatField.newRangeQuery(name(), lowerValue, upperValue); + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + name(), + NumericUtils.floatToSortableInt(lowerValue), + NumericUtils.floatToSortableInt(upperValue) + ); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -496,11 +505,16 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException { @Override public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { - float v = parse(value, false); + float fValue = parse(value, false); if (isIndexed) { - return FloatPoint.newExactQuery(field, v); + Query query = FloatPoint.newExactQuery(field, fValue); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue)); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } else { - return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v)); + return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue)); } } @@ -508,10 +522,29 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; + + float lowerValue = Float.MAX_VALUE; + float upperValue = Float.MIN_VALUE; for (Object value : values) { - v[pos++] = parse(value, false); + float fValue = parse(value, false); + v[pos++] = fValue; + if (fValue > upperValue) { + upperValue = fValue; + } + if (fValue < lowerValue) { + lowerValue = fValue; + } } - return FloatPoint.newSetQuery(field, v); + Query query = FloatPoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + name(), + NumericUtils.floatToSortableInt(lowerValue), + NumericUtils.floatToSortableInt(upperValue) + ); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -644,7 +677,11 @@ public FieldValues compile(String fieldName, Script script, ScriptCompil public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) { double v = parse(value, false); if (isIndexed) { - return DoublePoint.newExactQuery(field, v); + Query query = DoublePoint.newExactQuery(field, v); + if (hasDocValues) { + SortedNumericDocValuesField.newSlowExactQuery(name(), NumericUtils.doubleToSortableLong(v)); + } + return query; } else { return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v)); } @@ -652,8 +689,30 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha @Override public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { - double[] v = values.stream().mapToDouble(value -> parse(value, false)).toArray(); - return DoublePoint.newSetQuery(field, v); + double[] v = new double[values.size()]; + int pos = 0; + double lowerValue = Double.MAX_VALUE; + double upperValue = Double.MIN_VALUE; + for (Object value : values) { + double dValue = parse(value, false); + v[pos++] = dValue; + if (dValue > upperValue) { + upperValue = dValue; + } + if (dValue < lowerValue) { + lowerValue = dValue; + } + } + Query query = DoublePoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( + name(), + NumericUtils.doubleToSortableLong(lowerValue), + NumericUtils.doubleToSortableLong(upperValue) + ); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override @@ -958,11 +1017,16 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (hasDecimalPart(value)) { return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); } - int v = parse(value, true); + int iValue = parse(value, true); if (isIndexed) { - return IntPoint.newExactQuery(field, v); + Query query = IntPoint.newExactQuery(field, iValue); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), iValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } else { - return SortedNumericDocValuesField.newSlowExactQuery(field, v); + return SortedNumericDocValuesField.newSlowExactQuery(field, iValue); } } @@ -975,13 +1039,13 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b int upperValue = Integer.MIN_VALUE; for (Object value : values) { if (hasDecimalPart(value) == false) { - int currentValue = parse(value, true); - v[upTo++] = currentValue; - if (currentValue > upperValue) { - upperValue = currentValue; + int iValue = parse(value, true); + v[upTo++] = iValue; + if (iValue > upperValue) { + upperValue = iValue; } - if (currentValue < lowerValue) { - lowerValue = currentValue; + if (iValue < lowerValue) { + lowerValue = iValue; } } } @@ -992,17 +1056,11 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b if (upTo != v.length) { v = Arrays.copyOf(v, upTo); } -// return IntPoint.newSetQuery(field, v); - Query query; - if (isIndexed) { - query = IntPoint.newSetQuery(field, v); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); - query = new IndexOrDocValuesQuery(query, dvQuery); - } - } else { - query = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + Query query = IntPoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + query = new IndexOrDocValuesQuery(query, dvQuery); } return query; } @@ -1131,11 +1189,16 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (hasDecimalPart(value)) { return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); } - long v = parse(value, true); + long lValue = parse(value, true); if (isIndexed) { - return LongPoint.newExactQuery(field, v); + Query query = LongPoint.newExactQuery(field, lValue); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), lValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } else { - return SortedNumericDocValuesField.newSlowExactQuery(field, v); + return SortedNumericDocValuesField.newSlowExactQuery(field, lValue); } } @@ -1144,9 +1207,18 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b long[] v = new long[values.size()]; int upTo = 0; + long lowerValue = Long.MAX_VALUE; + long upperValue = Long.MIN_VALUE; for (Object value : values) { if (hasDecimalPart(value) == false) { - v[upTo++] = parse(value, true); + long lValue = parse(value, true); + v[upTo++] = lValue; + if (lValue > upperValue) { + upperValue = lValue; + } + if (lValue < lowerValue) { + lowerValue = lValue; + } } } @@ -1156,7 +1228,13 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b if (upTo != v.length) { v = Arrays.copyOf(v, upTo); } - return LongPoint.newSetQuery(field, v); + + Query query = LongPoint.newSetQuery(field, v); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + return query; } @Override From 9f394b283c55624a9445a759379427731b1b4e87 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Wed, 13 Sep 2023 08:53:41 +0200 Subject: [PATCH 04/10] updated tests --- .../index/mapper/NumberFieldTypeTests.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 7084535e59d47..95d955c2f79b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -493,6 +493,7 @@ public void testHalfFloatRange() throws IOException { public void testNegativeZero() { final boolean isIndexed = randomBoolean(); + final boolean hasDocValues = randomBoolean(); assertEquals( NumberType.DOUBLE.rangeQuery("field", null, -0d, true, true, false, MOCK_CONTEXT, isIndexed), NumberType.DOUBLE.rangeQuery("field", null, +0d, true, false, false, MOCK_CONTEXT, isIndexed) @@ -506,9 +507,18 @@ public void testNegativeZero() { NumberType.HALF_FLOAT.rangeQuery("field", null, +0f, true, false, false, MOCK_CONTEXT, isIndexed) ); - assertNotEquals(NumberType.DOUBLE.termQuery("field", -0d, isIndexed), NumberType.DOUBLE.termQuery("field", +0d, isIndexed)); - assertNotEquals(NumberType.FLOAT.termQuery("field", -0f, isIndexed), NumberType.FLOAT.termQuery("field", +0f, isIndexed)); - assertNotEquals(NumberType.HALF_FLOAT.termQuery("field", -0f, isIndexed), NumberType.HALF_FLOAT.termQuery("field", +0f, isIndexed)); + assertNotEquals( + NumberType.DOUBLE.termQuery("field", -0d, isIndexed, hasDocValues), + NumberType.DOUBLE.termQuery("field", +0d, isIndexed, hasDocValues) + ); + assertNotEquals( + NumberType.FLOAT.termQuery("field", -0f, isIndexed, hasDocValues), + NumberType.FLOAT.termQuery("field", +0f, isIndexed, hasDocValues) + ); + assertNotEquals( + NumberType.HALF_FLOAT.termQuery("field", -0f, isIndexed, hasDocValues), + NumberType.HALF_FLOAT.termQuery("field", +0f, isIndexed, hasDocValues) + ); } // Make sure we construct the IndexOrDocValuesQuery objects with queries that match From 51f47021372d714ae2e60df3ef936153e0a6b4a5 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 22 Sep 2023 11:48:02 +0200 Subject: [PATCH 05/10] Update docs/changelog/99801.yaml --- docs/changelog/99801.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/99801.yaml diff --git a/docs/changelog/99801.yaml b/docs/changelog/99801.yaml new file mode 100644 index 0000000000000..d92e99c67e2b7 --- /dev/null +++ b/docs/changelog/99801.yaml @@ -0,0 +1,5 @@ +pr: 99801 +summary: Feature/allow numeric field exact searches to use docvalues +area: Search +type: enhancement +issues: [] From 8f9a86af1bebcadd5ad70ee1d3f494c05fe80ed2 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 22 Sep 2023 12:29:01 +0200 Subject: [PATCH 06/10] full impl --- .../index/mapper/NumberFieldMapper.java | 90 +++++-------------- .../unsignedlong/UnsignedLongFieldMapper.java | 4 +- 2 files changed, 26 insertions(+), 68 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 484d04cde1bcb..07b4c94c802bc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -77,6 +77,7 @@ import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.IntStream; /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ public class NumberFieldMapper extends FieldMapper { @@ -325,7 +326,7 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (isIndexed) { Query query = HalfFloatPoint.newExactQuery(field, fValue); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), NumericUtils.floatToSortableInt(fValue)); + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue)); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -338,26 +339,16 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; - float lowerValue = Float.MAX_VALUE; - float upperValue = Float.MIN_VALUE; for (Object value : values) { - float fValue = parseToFloat(value); - v[pos++] = fValue; - if (fValue > upperValue) { - upperValue = fValue; - } - if (fValue < lowerValue) { - lowerValue = fValue; - } + v[pos++] = parseToFloat(value); } - Query query = HalfFloatPoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( - name(), - NumericUtils.floatToSortableInt(lowerValue), - NumericUtils.floatToSortableInt(upperValue) - ); + long[] longValues = IntStream.range(0, v.length) + .map(HalfFloatPoint::halfFloatToSortableShort) + .mapToLong(i -> (long) i) + .toArray(); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, longValues); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -515,26 +506,16 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { float[] v = new float[values.size()]; int pos = 0; - - float lowerValue = Float.MAX_VALUE; - float upperValue = Float.MIN_VALUE; for (Object value : values) { - float fValue = parse(value, false); - v[pos++] = fValue; - if (fValue > upperValue) { - upperValue = fValue; - } - if (fValue < lowerValue) { - lowerValue = fValue; - } + v[pos++] = parse(value, false); } Query query = FloatPoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( - name(), - NumericUtils.floatToSortableInt(lowerValue), - NumericUtils.floatToSortableInt(upperValue) - ); + long[] longValues = IntStream.range(0, v.length) + .map(NumericUtils::floatToSortableInt) + .mapToLong(i -> (long) i) + .toArray(); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, longValues); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -672,7 +653,7 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (isIndexed) { Query query = DoublePoint.newExactQuery(field, v); if (hasDocValues) { - SortedNumericDocValuesField.newSlowExactQuery(name(), NumericUtils.doubleToSortableLong(v)); + SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v)); } return query; } else { @@ -682,26 +663,12 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha @Override public Query termsQuery(String field, Collection values, boolean isIndexed, boolean hasDocValues) { - double[] v = new double[values.size()]; - int pos = 0; - double lowerValue = Double.MAX_VALUE; - double upperValue = Double.MIN_VALUE; - for (Object value : values) { - double dValue = parse(value, false); - v[pos++] = dValue; - if (dValue > upperValue) { - upperValue = dValue; - } - if (dValue < lowerValue) { - lowerValue = dValue; - } - } + double[] v = values.stream().mapToDouble(value -> parse(value, false)).toArray(); Query query = DoublePoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery( - name(), - NumericUtils.doubleToSortableLong(lowerValue), - NumericUtils.doubleToSortableLong(upperValue) + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery( + field, + Arrays.stream(v).mapToLong(NumericUtils::doubleToSortableLong).toArray() ); query = new IndexOrDocValuesQuery(query, dvQuery); } @@ -1014,7 +981,7 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (isIndexed) { Query query = IntPoint.newExactQuery(field, iValue); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), iValue); + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, iValue); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -1052,7 +1019,7 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b Query query = IntPoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, Arrays.stream(v).asLongStream().toArray()); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -1186,7 +1153,7 @@ public Query termQuery(String field, Object value, boolean isIndexed, boolean ha if (isIndexed) { Query query = LongPoint.newExactQuery(field, lValue); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), lValue); + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, lValue); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; @@ -1200,18 +1167,9 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b long[] v = new long[values.size()]; int upTo = 0; - long lowerValue = Long.MAX_VALUE; - long upperValue = Long.MIN_VALUE; for (Object value : values) { if (hasDecimalPart(value) == false) { - long lValue = parse(value, true); - v[upTo++] = lValue; - if (lValue > upperValue) { - upperValue = lValue; - } - if (lValue < lowerValue) { - lowerValue = lValue; - } + v[upTo++] = parse(value, true); } } @@ -1224,7 +1182,7 @@ public Query termsQuery(String field, Collection values, boolean isIndexed, b Query query = LongPoint.newSetQuery(field, v); if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, lowerValue, upperValue); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, v); query = new IndexOrDocValuesQuery(query, dvQuery); } return query; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 3174380e9977a..7c989489651f7 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -257,7 +257,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { } long lValue = unsignedToSortableSignedLong(longValue); Query query = LongPoint.newExactQuery(name(), lValue); - if (hasDocValues()) { + if (super.hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(name(), lValue); query = new IndexOrDocValuesQuery(query, dvQuery); } @@ -291,7 +291,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { lvalues = Arrays.copyOf(lvalues, upTo); } Query query = LongPoint.newSetQuery(name(), lvalues); - if (hasDocValues()) { + if (super.hasDocValues()) { Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), lowerValue, upperValue); query = new IndexOrDocValuesQuery(query, dvQuery); if (context.indexSortedOnField(name())) { From 8a09649855ebadd89d5cb5783dcbc47c94f2cb6a Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 22 Sep 2023 14:54:43 +0200 Subject: [PATCH 07/10] ci/cd fixes --- .../mapper/extras/ScaledFloatFieldTypeTests.java | 14 +++++++++----- .../unsignedlong/UnsignedLongFieldMapper.java | 16 ++-------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 603b19623a0e7..0323e98d23071 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -44,7 +45,10 @@ public void testTermQuery() { ); double value = (randomDouble() * 2 - 1) * 10000; long scaledValue = Math.round(value * ft.getScalingFactor()); - assertEquals(LongPoint.newExactQuery("scaled_float", scaledValue), ft.termQuery(value, MOCK_CONTEXT)); + Query indexQuery = LongPoint.newExactQuery("scaled_float", scaledValue); + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery("scaled_float", scaledValue); + Query query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + assertEquals(query, ft.termQuery(value, MOCK_CONTEXT)); MappedFieldType ft2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100, false); ElasticsearchException e2 = expectThrows(ElasticsearchException.class, () -> ft2.termQuery("42", MOCK_CONTEXT_DISALLOW_EXPENSIVE)); @@ -63,10 +67,10 @@ public void testTermsQuery() { long scaledValue1 = Math.round(value1 * ft.getScalingFactor()); double value2 = (randomDouble() * 2 - 1) * 10000; long scaledValue2 = Math.round(value2 * ft.getScalingFactor()); - assertEquals( - LongPoint.newSetQuery("scaled_float", scaledValue1, scaledValue2), - ft.termsQuery(Arrays.asList(value1, value2), MOCK_CONTEXT) - ); + Query indexQuery = LongPoint.newSetQuery("scaled_float", scaledValue1, scaledValue2); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery("scaled_float", scaledValue1, scaledValue2); + Query query = new IndexOrDocValuesQuery(indexQuery, dvQuery); + assertEquals(query, ft.termsQuery(Arrays.asList(value1, value2), MOCK_CONTEXT)); MappedFieldType ft2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100, false); ElasticsearchException e2 = expectThrows( diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 7c989489651f7..30fc5e5f49e8a 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -269,19 +269,10 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexed(); long[] lvalues = new long[values.size()]; int upTo = 0; - long lowerValue = Long.MAX_VALUE; - long upperValue = Long.MIN_VALUE; for (Object value : values) { Long longValue = parseTerm(value); if (longValue != null) { - long currentValue = unsignedToSortableSignedLong(longValue); - lvalues[upTo++] = currentValue; - if (currentValue > upperValue) { - upperValue = currentValue; - } - if (currentValue < lowerValue) { - lowerValue = currentValue; - } + lvalues[upTo++] = unsignedToSortableSignedLong(longValue); } } if (upTo == 0) { @@ -292,11 +283,8 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { } Query query = LongPoint.newSetQuery(name(), lvalues); if (super.hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), lowerValue, upperValue); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(name(), lvalues); query = new IndexOrDocValuesQuery(query, dvQuery); - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), lowerValue, upperValue, query); - } } return query; } From 7be9c166b5a5aa6fa3f1dd756534ba7a1b1f2e99 Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:24:28 +0200 Subject: [PATCH 08/10] aligned tests --- .../index/query/TermQueryBuilderTests.java | 2 ++ .../mapper/AggregateDoubleMetricFieldTypeTests.java | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java index 364a4f52f0fca..4b1ebd62a5cf0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; @@ -87,6 +88,7 @@ protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, S either(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class)) .or(instanceOf(MatchNoDocsQuery.class)) .or(instanceOf(AutomatonQuery.class)) + .or(instanceOf(IndexOrDocValuesQuery.class)) ); MappedFieldType mapper = context.getFieldType(queryBuilder.fieldName()); if (query instanceof TermQuery termQuery) { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index c32e7e583c787..2d35dd9b7200b 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -8,6 +8,7 @@ import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; @@ -15,6 +16,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -71,7 +73,16 @@ public void testTermQuery() { public void testTermsQuery() { final MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap(), Metric.max); Query query = fieldType.termsQuery(asList(55.2, 500.3), MOCK_CONTEXT); - assertThat(query, equalTo(DoublePoint.newSetQuery("foo.max", 55.2, 500.3))); + Query indexQuery = DoublePoint.newSetQuery("foo.max", 55.2, 500.3); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery( + "foo.max", + NumericUtils.doubleToSortableLong(55.2), + NumericUtils.doubleToSortableLong(500.3) + ); + assertThat(query, instanceOf(IndexOrDocValuesQuery.class)); + IndexOrDocValuesQuery indexOrDocValuesQuery = (IndexOrDocValuesQuery) query; + assertThat(indexOrDocValuesQuery.getIndexQuery(), equalTo(indexQuery)); + assertThat(indexOrDocValuesQuery.getRandomAccessQuery(), equalTo(dvQuery)); } public void testRangeQuery() { From 80a18eaec800609c94223e993f659df9c807b90a Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Mon, 25 Sep 2023 11:40:01 +0200 Subject: [PATCH 09/10] finished aligning tests --- .../index/mapper/NumberFieldTypeTests.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 741053fc3a549..276dc8326a2aa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -94,15 +94,21 @@ public void testIsFieldWithinQuery() throws IOException { public void testIntegerTermsQueryWithDecimalPart() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.INTEGER); - assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), MOCK_CONTEXT)); - assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), MOCK_CONTEXT)); + Query indexQuery = IntPoint.newSetQuery("field", 1); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery("field", 1); + Query indexOrDocValuesQuery = new IndexOrDocValuesQuery(indexQuery, dvQuery); + assertEquals(indexOrDocValuesQuery, ft.termsQuery(Arrays.asList(1, 2.1), MOCK_CONTEXT)); + assertEquals(indexOrDocValuesQuery, ft.termsQuery(Arrays.asList(1.0, 2.1), MOCK_CONTEXT)); assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), MOCK_CONTEXT) instanceof MatchNoDocsQuery); } public void testLongTermsQueryWithDecimalPart() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.LONG); - assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), MOCK_CONTEXT)); - assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), MOCK_CONTEXT)); + Query indexQuery = LongPoint.newSetQuery("field", 1); + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery("field", 1); + Query indexOrDocValuesQuery = new IndexOrDocValuesQuery(indexQuery, dvQuery); + assertEquals(indexOrDocValuesQuery, ft.termsQuery(Arrays.asList(1, 2.1), MOCK_CONTEXT)); + assertEquals(indexOrDocValuesQuery, ft.termsQuery(Arrays.asList(1.0, 2.1), MOCK_CONTEXT)); assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), MOCK_CONTEXT) instanceof MatchNoDocsQuery); } @@ -145,7 +151,9 @@ private static MappedFieldType unsearchable() { public void testTermQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG); - assertEquals(LongPoint.newExactQuery("field", 42), ft.termQuery("42", MOCK_CONTEXT)); + Query indexQuery = LongPoint.newExactQuery("field", 42); + Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery("field", 42); + assertEquals(new IndexOrDocValuesQuery(indexQuery, dvQuery), ft.termQuery("42", MOCK_CONTEXT)); ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG, false); assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 42), ft.termQuery("42", MOCK_CONTEXT)); From cceb136f5a9b1fbc6e6ce8cf9d0206be8a3b01dc Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Tue, 26 Sep 2023 09:42:38 +0200 Subject: [PATCH 10/10] aligned UnsignedLongFieldType tests --- .../UnsignedLongFieldTypeTests.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index e5f85f8b87b12..1946e52826671 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -8,7 +8,10 @@ package org.elasticsearch.xpack.unsignedlong; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; @@ -27,10 +30,21 @@ public class UnsignedLongFieldTypeTests extends FieldTypeTestCase { public void testTermQuery() { UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); - - assertEquals(LongPoint.newExactQuery("my_unsigned_long", -9223372036854775808L), ft.termQuery(0, null)); - assertEquals(LongPoint.newExactQuery("my_unsigned_long", 0L), ft.termQuery("9223372036854775808", null)); - assertEquals(LongPoint.newExactQuery("my_unsigned_long", 9223372036854775807L), ft.termQuery("18446744073709551615", null)); + Query dvQuery1 = SortedNumericDocValuesField.newSlowExactQuery("my_unsigned_long", -9223372036854775808L); + Query dvQuery2 = SortedNumericDocValuesField.newSlowExactQuery("my_unsigned_long", 0L); + Query dvQuery3 = SortedNumericDocValuesField.newSlowExactQuery("my_unsigned_long", 9223372036854775807L); + assertEquals( + new IndexOrDocValuesQuery(LongPoint.newExactQuery("my_unsigned_long", -9223372036854775808L), dvQuery1), + ft.termQuery(0, null) + ); + assertEquals( + new IndexOrDocValuesQuery(LongPoint.newExactQuery("my_unsigned_long", 0L), dvQuery2), + ft.termQuery("9223372036854775808", null) + ); + assertEquals( + new IndexOrDocValuesQuery(LongPoint.newExactQuery("my_unsigned_long", 9223372036854775807L), dvQuery3), + ft.termQuery("18446744073709551615", null) + ); assertEquals(new MatchNoDocsQuery(), ft.termQuery(-1L, null)); assertEquals(new MatchNoDocsQuery(), ft.termQuery(10.5, null)); @@ -41,9 +55,9 @@ public void testTermQuery() { public void testTermsQuery() { UnsignedLongFieldType ft = new UnsignedLongFieldType("my_unsigned_long"); - + Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery("my_unsigned_long", -9223372036854775808L, 0L, 9223372036854775807L); assertEquals( - LongPoint.newSetQuery("my_unsigned_long", -9223372036854775808L, 0L, 9223372036854775807L), + new IndexOrDocValuesQuery(LongPoint.newSetQuery("my_unsigned_long", -9223372036854775808L, 0L, 9223372036854775807L), dvQuery), ft.termsQuery(List.of("0", "9223372036854775808", "18446744073709551615"), null) );