Skip to content

Feature/allow numeric field exact searches to use docvalues #99801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 18 commits into
base: main
Choose a base branch
from
Draft

Feature/allow numeric field exact searches to use docvalues #99801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
20cd372
Mapper: Improve numeric field exact searches.
piergm May 31, 2023
7842306
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Jun 23, 2023
4b23315
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Jun 26, 2023
fc7892d
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Jun 27, 2023
a9e926d
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Jun 27, 2023
2552199
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Sep 8, 2023
735f59d
WIP passing hasDocValues in term(s)Query for NumberField
piergm Jun 27, 2023
dadf9e3
allow NumberFieldMapper exact searches to use docValues
piergm Sep 8, 2023
9f394b2
updated tests
piergm Sep 13, 2023
ae9d035
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Sep 22, 2023
51f4702
Update docs/changelog/99801.yaml
piergm Sep 22, 2023
8f9a86a
full impl
piergm Sep 22, 2023
8a09649
ci/cd fixes
piergm Sep 22, 2023
7be9c16
aligned tests
piergm Sep 22, 2023
f1346b9
Merge branch 'elastic:main' into feature/Allow-numeric-field-exact-se…
piergm Sep 25, 2023
80a18ea
finished aligning tests
piergm Sep 25, 2023
cceb136
aligned UnsignedLongFieldType tests
piergm Sep 26, 2023
45814ad
merged main
piergm May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/99801.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 99801
summary: Feature/allow numeric field exact searches to use docvalues
area: Search
type: enhancement
issues: []
9 changes: 7 additions & 2 deletions ...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ public boolean isSearchable() {
public Query termQuery(Object value, SearchExecutionContext context) {
failIfNotIndexedNorDocValuesFallback(context);
long scaledValue = Math.round(scale(value));
return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed());
return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed(), hasDocValues());
}

@Override
Expand All @@ -270,7 +270,12 @@ public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
long scaledValue = Math.round(scale(value));
scaledValues.add(scaledValue);
}
return NumberFieldMapper.NumberType.LONG.termsQuery(name(), Collections.unmodifiableList(scaledValues));
return NumberFieldMapper.NumberType.LONG.termsQuery(
name(),
Collections.unmodifiableList(scaledValues),
isIndexed(),
hasDocValues()
);
} else {
return super.termsQuery(values, context);
}
Expand Down
14 changes: 9 additions & 5 deletions ...extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
Expand Down Expand Up @@ -46,7 +47,10 @@ public void testTermQuery() {
);
double value = (randomDouble() * 2 - 1) * 10000;
long scaledValue = Math.round(value * ft.getScalingFactor());
assertEquals(LongPoint.newExactQuery("scaled_float", scaledValue), ft.termQuery(value, MOCK_CONTEXT));
Query indexQuery = LongPoint.newExactQuery("scaled_float", scaledValue);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery("scaled_float", scaledValue);
Query query = new IndexOrDocValuesQuery(indexQuery, dvQuery);
assertEquals(query, ft.termQuery(value, MOCK_CONTEXT));

MappedFieldType ft2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100, false);
ElasticsearchException e2 = expectThrows(ElasticsearchException.class, () -> ft2.termQuery("42", MOCK_CONTEXT_DISALLOW_EXPENSIVE));
Expand All @@ -65,10 +69,10 @@ public void testTermsQuery() {
long scaledValue1 = Math.round(value1 * ft.getScalingFactor());
double value2 = (randomDouble() * 2 - 1) * 10000;
long scaledValue2 = Math.round(value2 * ft.getScalingFactor());
assertEquals(
LongPoint.newSetQuery("scaled_float", scaledValue1, scaledValue2),
ft.termsQuery(Arrays.asList(value1, value2), MOCK_CONTEXT)
);
Query indexQuery = LongPoint.newSetQuery("scaled_float", scaledValue1, scaledValue2);
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery("scaled_float", scaledValue1, scaledValue2);
Query query = new IndexOrDocValuesQuery(indexQuery, dvQuery);
assertEquals(query, ft.termsQuery(Arrays.asList(value1, value2), MOCK_CONTEXT));

MappedFieldType ft2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100, false);
ElasticsearchException e2 = expectThrows(
Expand Down
156 changes: 114 additions & 42 deletions server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.IntStream;

/** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */
public class NumberFieldMapper extends FieldMapper {
Expand Down Expand Up @@ -318,26 +319,40 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException {
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
float v = parseToFloat(value);
validateFiniteValue(v);
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
float fValue = parseToFloat(value);
validateFiniteValue(fValue);
if (isIndexed) {
return HalfFloatPoint.newExactQuery(field, v);
Query query = HalfFloatPoint.newExactQuery(field, fValue);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue));
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
} else {
return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v));
return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(fValue));
}
}

@Override
public Query termsQuery(String field, Collection<?> values) {
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
float[] v = new float[values.size()];
int pos = 0;
for (Object value : values) {
float float_value = parseToFloat(value);
validateFiniteValue(float_value);
v[pos++] = float_value;
}
return HalfFloatPoint.newSetQuery(field, v);
Query query = HalfFloatPoint.newSetQuery(field, v);
if (hasDocValues) {
long[] longValues = IntStream.range(0, v.length)
.map(HalfFloatPoint::halfFloatToSortableShort)
.mapToLong(i -> (long) i)
.toArray();
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, longValues);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
}

@Override
Expand Down Expand Up @@ -500,23 +515,37 @@ public Float parse(XContentParser parser, boolean coerce) throws IOException {
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
float v = parseToFloat(value);
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
float fValue = parseToFloat(value);
if (isIndexed) {
return FloatPoint.newExactQuery(field, v);
Query query = FloatPoint.newExactQuery(field, fValue);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue));
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
} else {
return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v));
return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(fValue));
}
}

@Override
public Query termsQuery(String field, Collection<?> values) {
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
float[] v = new float[values.size()];
int pos = 0;
for (Object value : values) {
v[pos++] = parse(value, false);
}
return FloatPoint.newSetQuery(field, v);
Query query = FloatPoint.newSetQuery(field, v);
if (hasDocValues) {
long[] longValues = IntStream.range(0, v.length)
.map(NumericUtils::floatToSortableInt)
.mapToLong(i -> (long) i)
.toArray();
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, longValues);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
}

@Override
Expand Down Expand Up @@ -664,19 +693,31 @@ public FieldValues<Number> compile(String fieldName, Script script, ScriptCompil
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
double v = parse(value, false);
if (isIndexed) {
return DoublePoint.newExactQuery(field, v);
Query query = DoublePoint.newExactQuery(field, v);
if (hasDocValues) {
SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
}
return query;
} else {
return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
}
}

@Override
public Query termsQuery(String field, Collection<?> values) {
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
double[] v = values.stream().mapToDouble(value -> parse(value, false)).toArray();
return DoublePoint.newSetQuery(field, v);
Query query = DoublePoint.newSetQuery(field, v);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(
field,
Arrays.stream(v).mapToLong(NumericUtils::doubleToSortableLong).toArray()
);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
}

@Override
Expand Down Expand Up @@ -815,13 +856,13 @@ public Short parse(XContentParser parser, boolean coerce) throws IOException {
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
return INTEGER.termQuery(field, value, isIndexed);
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
return INTEGER.termQuery(field, value, isIndexed, hasDocValues);
}

@Override
public Query termsQuery(String field, Collection<?> values) {
return INTEGER.termsQuery(field, values);
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
return INTEGER.termsQuery(field, values, isIndexed, hasDocValues);
}

@Override
Expand Down Expand Up @@ -920,13 +961,13 @@ public Short parse(XContentParser parser, boolean coerce) throws IOException {
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
return INTEGER.termQuery(field, value, isIndexed);
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
return INTEGER.termQuery(field, value, isIndexed, hasDocValues);
}

@Override
public Query termsQuery(String field, Collection<?> values) {
return INTEGER.termsQuery(field, values);
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
return INTEGER.termsQuery(field, values, isIndexed, hasDocValues);
}

@Override
Expand Down Expand Up @@ -1025,26 +1066,40 @@ public Integer parse(XContentParser parser, boolean coerce) throws IOException {
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
if (hasDecimalPart(value)) {
return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
}
int v = parse(value, true);
int iValue = parse(value, true);
if (isIndexed) {
return IntPoint.newExactQuery(field, v);
Query query = IntPoint.newExactQuery(field, iValue);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, iValue);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
} else {
return SortedNumericDocValuesField.newSlowExactQuery(field, v);
return SortedNumericDocValuesField.newSlowExactQuery(field, iValue);
}
}

@Override
public Query termsQuery(String field, Collection<?> values) {
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
int[] v = new int[values.size()];
int upTo = 0;

int lowerValue = Integer.MAX_VALUE;
int upperValue = Integer.MIN_VALUE;
for (Object value : values) {
if (hasDecimalPart(value) == false) {
v[upTo++] = parse(value, true);
int iValue = parse(value, true);
v[upTo++] = iValue;
if (iValue > upperValue) {
upperValue = iValue;
}
if (iValue < lowerValue) {
lowerValue = iValue;
}
}
}

Expand All @@ -1054,7 +1109,13 @@ public Query termsQuery(String field, Collection<?> values) {
if (upTo != v.length) {
v = Arrays.copyOf(v, upTo);
}
return IntPoint.newSetQuery(field, v);

Query query = IntPoint.newSetQuery(field, v);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, Arrays.stream(v).asLongStream().toArray());
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
}

@Override
Expand Down Expand Up @@ -1193,20 +1254,25 @@ public FieldValues<Number> compile(String fieldName, Script script, ScriptCompil
}

@Override
public Query termQuery(String field, Object value, boolean isIndexed) {
public Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues) {
if (hasDecimalPart(value)) {
return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
}
long v = parse(value, true);
long lValue = parse(value, true);
if (isIndexed) {
return LongPoint.newExactQuery(field, v);
Query query = LongPoint.newExactQuery(field, lValue);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, lValue);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
} else {
return SortedNumericDocValuesField.newSlowExactQuery(field, v);
return SortedNumericDocValuesField.newSlowExactQuery(field, lValue);
}
}

@Override
public Query termsQuery(String field, Collection<?> values) {
public Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues) {
long[] v = new long[values.size()];
int upTo = 0;

Expand All @@ -1222,7 +1288,13 @@ public Query termsQuery(String field, Collection<?> values) {
if (upTo != v.length) {
v = Arrays.copyOf(v, upTo);
}
return LongPoint.newSetQuery(field, v);

Query query = LongPoint.newSetQuery(field, v);
if (hasDocValues) {
Query dvQuery = SortedNumericDocValuesField.newSlowSetQuery(field, v);
query = new IndexOrDocValuesQuery(query, dvQuery);
}
return query;
}

@Override
Expand Down Expand Up @@ -1346,9 +1418,9 @@ public final TypeParser parser() {
return parser;
}

public abstract Query termQuery(String field, Object value, boolean isIndexed);
public abstract Query termQuery(String field, Object value, boolean isIndexed, boolean hasDocValues);

public abstract Query termsQuery(String field, Collection<?> values);
public abstract Query termsQuery(String field, Collection<?> values, boolean isIndexed, boolean hasDocValues);

public abstract Query rangeQuery(
String field,
Expand Down Expand Up @@ -1674,14 +1746,14 @@ public boolean isSearchable() {
@Override
public Query termQuery(Object value, SearchExecutionContext context) {
failIfNotIndexedNorDocValuesFallback(context);
return type.termQuery(name(), value, isIndexed());
return type.termQuery(name(), value, isIndexed(), hasDocValues());
}

@Override
public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
failIfNotIndexedNorDocValuesFallback(context);
if (isIndexed()) {
return type.termsQuery(name(), values);
return type.termsQuery(name(), values, isIndexed(), hasDocValues());
} else {
return super.termsQuery(values, context);
}
Expand Down
Loading