Skip to content

Commit 524d1ea

Browse files
romseygeekromseygeek
authored
Upgrade to lucene-9.0.0-snapshot-32a0a16aff0 (#73324)
This commit upgrades elasticsearch 8.0 to use a snapshot of Lucene 9.0.0.
1 parent d381ffa commit 524d1ea

File tree

283 files changed

+1221
-1044
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

283 files changed

+1221
-1044
lines changed

‎build-tools-internal/version.properties

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
elasticsearch = 8.0.0
2-
lucene = 8.10.0-snapshot-bf2fcb53079
2+
lucene = 9.0.0-snapshot-32a0a16aff0
33

44
bundled_jdk_vendor = adoptium
55
bundled_jdk = 16.0.2+7
@@ -11,7 +11,7 @@ spatial4j = 0.7
1111
jts = 1.15.0
1212
jackson = 2.10.4
1313
snakeyaml = 1.26
14-
icu4j = 62.1
14+
icu4j = 68.2
1515
supercsv = 2.4.0
1616
# when updating log4j, please update also docs/java-api/index.asciidoc
1717
log4j = 2.11.1

‎distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88

99
package org.elasticsearch.common.settings;
1010

11+
import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
1112
import org.apache.lucene.codecs.CodecUtil;
13+
import org.apache.lucene.store.DataOutput;
1214
import org.apache.lucene.store.Directory;
1315
import org.apache.lucene.store.IOContext;
1416
import org.apache.lucene.store.IndexOutput;
@@ -345,13 +347,14 @@ private void possiblyAlterEncryptedBytes(
345347
byte[] encryptedBytes,
346348
int truncEncryptedDataLength
347349
) throws Exception {
348-
indexOutput.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
349-
indexOutput.writeInt(salt.length);
350-
indexOutput.writeBytes(salt, salt.length);
351-
indexOutput.writeInt(iv.length);
352-
indexOutput.writeBytes(iv, iv.length);
353-
indexOutput.writeInt(encryptedBytes.length - truncEncryptedDataLength);
354-
indexOutput.writeBytes(encryptedBytes, encryptedBytes.length);
350+
DataOutput out = EndiannessReverserUtil.wrapDataOutput(indexOutput);
351+
out.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length);
352+
out.writeInt(salt.length);
353+
out.writeBytes(salt, salt.length);
354+
out.writeInt(iv.length);
355+
out.writeBytes(iv, iv.length);
356+
out.writeInt(encryptedBytes.length - truncEncryptedDataLength);
357+
out.writeBytes(encryptedBytes, encryptedBytes.length);
355358
}
356359

357360
public void testUpgradeAddsSeed() throws Exception {
@@ -382,7 +385,7 @@ public void testBackcompatV1() throws Exception {
382385
Path configDir = env.configFile();
383386
try (
384387
Directory directory = newFSDirectory(configDir);
385-
IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
388+
IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
386389
) {
387390
CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
388391
output.writeByte((byte) 0); // hasPassword = false
@@ -417,9 +420,8 @@ public void testBackcompatV2() throws Exception {
417420
random().nextBytes(fileBytes);
418421
try (
419422
Directory directory = newFSDirectory(configDir);
420-
IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
423+
IndexOutput output = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT);
421424
) {
422-
423425
CodecUtil.writeHeader(output, "elasticsearch.keystore", 2);
424426
output.writeByte((byte) 0); // hasPassword = false
425427
output.writeString("PKCS12");

‎docs/Versions.asciidoc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11

22
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
33

4-
:lucene_version: 8.10.0
5-
:lucene_version_path: 8_10_0
4+
:lucene_version: 9.0.0
5+
:lucene_version_path: 9_0_0
66
:jdk: 11.0.2
77
:jdk_major: 11
88
:build_flavor: default

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicFilterFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
package org.elasticsearch.analysis.common;
99

1010
import org.apache.lucene.analysis.TokenStream;
11-
import org.apache.lucene.analysis.standard.ClassicFilter;
11+
import org.apache.lucene.analysis.classic.ClassicFilter;
1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.env.Environment;
1414
import org.elasticsearch.index.IndexSettings;

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ClassicTokenizerFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
package org.elasticsearch.analysis.common;
1010

1111
import org.apache.lucene.analysis.Tokenizer;
12-
import org.apache.lucene.analysis.standard.ClassicTokenizer;
12+
import org.apache.lucene.analysis.classic.ClassicTokenizer;
1313
import org.apache.lucene.analysis.standard.StandardAnalyzer;
1414
import org.elasticsearch.common.settings.Settings;
1515
import org.elasticsearch.env.Environment;

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727
import org.apache.lucene.analysis.cjk.CJKWidthFilter;
2828
import org.apache.lucene.analysis.ckb.SoraniAnalyzer;
2929
import org.apache.lucene.analysis.ckb.SoraniNormalizationFilter;
30+
import org.apache.lucene.analysis.classic.ClassicFilter;
31+
import org.apache.lucene.analysis.classic.ClassicTokenizer;
3032
import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
3133
import org.apache.lucene.analysis.core.DecimalDigitFilter;
3234
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -40,6 +42,7 @@
4042
import org.apache.lucene.analysis.de.GermanNormalizationFilter;
4143
import org.apache.lucene.analysis.de.GermanStemFilter;
4244
import org.apache.lucene.analysis.el.GreekAnalyzer;
45+
import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
4346
import org.apache.lucene.analysis.en.EnglishAnalyzer;
4447
import org.apache.lucene.analysis.en.KStemFilter;
4548
import org.apache.lucene.analysis.en.PorterStemFilter;
@@ -89,10 +92,7 @@
8992
import org.apache.lucene.analysis.ru.RussianAnalyzer;
9093
import org.apache.lucene.analysis.shingle.ShingleFilter;
9194
import org.apache.lucene.analysis.snowball.SnowballFilter;
92-
import org.apache.lucene.analysis.standard.ClassicFilter;
93-
import org.apache.lucene.analysis.standard.ClassicTokenizer;
9495
import org.apache.lucene.analysis.standard.StandardAnalyzer;
95-
import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
9696
import org.apache.lucene.analysis.sv.SwedishAnalyzer;
9797
import org.apache.lucene.analysis.th.ThaiAnalyzer;
9898
import org.apache.lucene.analysis.th.ThaiTokenizer;

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/MinHashTokenFilterFactory.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,18 @@ public TokenStream create(TokenStream tokenStream) {
3838

3939
private Map<String, String> convertSettings(Settings settings) {
4040
Map<String, String> settingMap = new HashMap<>();
41-
settingMap.put("hashCount", settings.get("hash_count"));
42-
settingMap.put("bucketCount", settings.get("bucket_count"));
43-
settingMap.put("hashSetSize", settings.get("hash_set_size"));
44-
settingMap.put("withRotation", settings.get("with_rotation"));
41+
if (settings.hasValue("hash_count")) {
42+
settingMap.put("hashCount", settings.get("hash_count"));
43+
}
44+
if (settings.hasValue("bucketCount")) {
45+
settingMap.put("bucketCount", settings.get("bucket_count"));
46+
}
47+
if (settings.hasValue("hashSetSize")) {
48+
settingMap.put("hashSetSize", settings.get("hash_set_size"));
49+
}
50+
if (settings.hasValue("with_rotation")) {
51+
settingMap.put("withRotation", settings.get("with_rotation"));
52+
}
4553
return settingMap;
4654
}
4755
}

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/UAX29URLEmailTokenizerFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
package org.elasticsearch.analysis.common;
1010

1111
import org.apache.lucene.analysis.Tokenizer;
12+
import org.apache.lucene.analysis.email.UAX29URLEmailTokenizer;
1213
import org.apache.lucene.analysis.standard.StandardAnalyzer;
13-
import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
1414
import org.elasticsearch.common.settings.Settings;
1515
import org.elasticsearch.env.Environment;
1616
import org.elasticsearch.index.IndexSettings;

‎modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisFactoryTests.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,18 @@ protected Map<String, Class<?>> getTokenFilters() {
8686
filters.put("latvianstem", StemmerTokenFilterFactory.class);
8787
filters.put("norwegianlightstem", StemmerTokenFilterFactory.class);
8888
filters.put("norwegianminimalstem", StemmerTokenFilterFactory.class);
89+
filters.put("norwegiannormalization", Void.class);
8990
filters.put("portuguesestem", StemmerTokenFilterFactory.class);
9091
filters.put("portugueselightstem", StemmerTokenFilterFactory.class);
9192
filters.put("portugueseminimalstem", StemmerTokenFilterFactory.class);
9293
filters.put("russianlightstem", StemmerTokenFilterFactory.class);
9394
filters.put("soranistem", StemmerTokenFilterFactory.class);
9495
filters.put("spanishlightstem", StemmerTokenFilterFactory.class);
9596
filters.put("swedishlightstem", StemmerTokenFilterFactory.class);
97+
filters.put("swedishminimalstem", Void.class);
9698
filters.put("stemmeroverride", StemmerOverrideTokenFilterFactory.class);
99+
filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
100+
filters.put("telugustem", TeluguStemFilterFactory.class);
97101
filters.put("kstem", KStemTokenFilterFactory.class);
98102
filters.put("synonym", SynonymTokenFilterFactory.class);
99103
filters.put("synonymgraph", SynonymGraphTokenFilterFactory.class);
@@ -131,8 +135,6 @@ protected Map<String, Class<?>> getTokenFilters() {
131135
filters.put("brazilianstem", BrazilianStemTokenFilterFactory.class);
132136
filters.put("czechstem", CzechStemTokenFilterFactory.class);
133137
filters.put("germanstem", GermanStemTokenFilterFactory.class);
134-
filters.put("telugunormalization", TeluguNormalizationFilterFactory.class);
135-
filters.put("telugustem", TeluguStemFilterFactory.class);
136138
// this filter is not exposed and should only be used internally
137139
filters.put("fixedshingle", Void.class);
138140
return filters;
@@ -216,7 +218,7 @@ protected Map<String, Class<?>> getPreConfiguredTokenizers() {
216218
tokenizers.put("keyword", null);
217219
tokenizers.put("lowercase", Void.class);
218220
tokenizers.put("classic", null);
219-
tokenizers.put("uax_url_email", org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory.class);
221+
tokenizers.put("uax_url_email", org.apache.lucene.analysis.email.UAX29URLEmailTokenizerFactory.class);
220222
tokenizers.put("path_hierarchy", null);
221223
tokenizers.put("letter", null);
222224
tokenizers.put("whitespace", null);

‎modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,15 +80,17 @@ public void setup() {
8080
// that ignores position length attribute
8181
expectedQueryWithUnigram= new BooleanQuery.Builder()
8282
.add(
83-
new SynonymQuery(
84-
new Term("text_shingle_unigram", "foo"),
85-
new Term("text_shingle_unigram", "foo bar")
86-
), BooleanClause.Occur.SHOULD)
83+
new SynonymQuery.Builder("text_shingle_unigram")
84+
.addTerm(new Term("text_shingle_unigram", "foo"))
85+
.addTerm(new Term("text_shingle_unigram", "foo bar"))
86+
.build(),
87+
BooleanClause.Occur.SHOULD)
8788
.add(
88-
new SynonymQuery(
89-
new Term("text_shingle_unigram", "bar"),
90-
new Term("text_shingle_unigram", "bar baz")
91-
), BooleanClause.Occur.SHOULD)
89+
new SynonymQuery.Builder("text_shingle_unigram")
90+
.addTerm(new Term("text_shingle_unigram", "bar"))
91+
.addTerm(new Term("text_shingle_unigram", "bar baz"))
92+
.build(),
93+
BooleanClause.Occur.SHOULD)
9294
.add(
9395
new TermQuery(
9496
new Term("text_shingle_unigram", "baz")

‎modules/lang-expression/licenses/lucene-expressions-8.10.0-snapshot-bf2fcb53079.jar.sha1

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎modules/lang-expression/licenses/lucene-expressions-9.0.0-snapshot-32a0a16aff0.jar.sha1

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
561301335ba14e07ec89adff0e2321cb79146a48

‎modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.apache.lucene.expressions.js.JavascriptCompiler;
1414
import org.apache.lucene.expressions.js.VariableContext;
1515
import org.apache.lucene.search.DoubleValuesSource;
16-
import org.apache.lucene.search.SortField;
1716
import org.elasticsearch.SpecialPermission;
1817
import org.elasticsearch.core.Nullable;
1918
import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -232,7 +231,7 @@ private static NumberSortScript.LeafFactory newSortScript(Expression expr, Searc
232231
for (String variable : expr.variables) {
233232
try {
234233
if (variable.equals("_score")) {
235-
bindings.add(new SortField("_score", SortField.Type.SCORE));
234+
bindings.add("_score", DoubleValuesSource.SCORES);
236235
needsScores = true;
237236
} else if (vars != null && vars.containsKey(variable)) {
238237
bindFromParams(vars, bindings, variable);
@@ -283,7 +282,7 @@ private static AggregationScript.LeafFactory newAggregationScript(Expression exp
283282
for (String variable : expr.variables) {
284283
try {
285284
if (variable.equals("_score")) {
286-
bindings.add(new SortField("_score", SortField.Type.SCORE));
285+
bindings.add("_score", DoubleValuesSource.SCORES);
287286
needsScores = true;
288287
} else if (variable.equals("_value")) {
289288
specialValue = new ReplaceableConstDoubleValueSource();
@@ -355,7 +354,7 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo
355354
for (String variable : expr.variables) {
356355
try {
357356
if (variable.equals("_score")) {
358-
bindings.add(new SortField("_score", SortField.Type.SCORE));
357+
bindings.add("_score", DoubleValuesSource.SCORES);
359358
needsScores = true;
360359
} else if (variable.equals("_value")) {
361360
specialValue = new ReplaceableConstDoubleValueSource();

‎modules/lang-expression/src/main/plugin-metadata/plugin-security.policy

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@ grant {
1818
permission org.elasticsearch.script.ClassPermission "java.lang.Math";
1919
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil";
2020
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath";
21+
permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.js.ExpressionMath";
2122
};

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
import org.apache.lucene.document.FieldType;
2121
import org.apache.lucene.index.IndexOptions;
2222
import org.apache.lucene.index.Term;
23+
import org.apache.lucene.queries.spans.FieldMaskingSpanQuery;
24+
import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper;
25+
import org.apache.lucene.queries.spans.SpanQuery;
26+
import org.apache.lucene.queries.spans.SpanTermQuery;
2327
import org.apache.lucene.search.AutomatonQuery;
2428
import org.apache.lucene.search.BooleanClause;
2529
import org.apache.lucene.search.BooleanQuery;
@@ -28,10 +32,6 @@
2832
import org.apache.lucene.search.PrefixQuery;
2933
import org.apache.lucene.search.Query;
3034
import org.apache.lucene.search.TermQuery;
31-
import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
32-
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
33-
import org.apache.lucene.search.spans.SpanQuery;
34-
import org.apache.lucene.search.spans.SpanTermQuery;
3535
import org.apache.lucene.util.automaton.Automata;
3636
import org.apache.lucene.util.automaton.Automaton;
3737
import org.apache.lucene.util.automaton.Operations;

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/query/SourceConfirmedTextQuery.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,11 @@ public int hashCode() {
183183
return 31 * Objects.hash(in, valueFetcherProvider, indexAnalyzer) + classHash();
184184
}
185185

186+
@Override
187+
public void visit(QueryVisitor visitor) {
188+
in.visit(visitor.getSubVisitor(Occur.MUST, this));
189+
}
190+
186191
@Override
187192
public Query rewrite(IndexReader reader) throws IOException {
188193
Query inRewritten = in.rewrite(reader);
@@ -244,11 +249,6 @@ public boolean isCacheable(LeafReaderContext ctx) {
244249
return false;
245250
}
246251

247-
@Override
248-
public void extractTerms(Set<Term> termSet) {
249-
termSet.addAll(terms);
250-
}
251-
252252
@Override
253253
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
254254
RuntimePhraseScorer scorer = scorer(context);

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
import org.apache.lucene.index.IndexableField;
1616
import org.apache.lucene.index.IndexableFieldType;
1717
import org.apache.lucene.index.Term;
18+
import org.apache.lucene.queries.spans.FieldMaskingSpanQuery;
19+
import org.apache.lucene.queries.spans.SpanNearQuery;
20+
import org.apache.lucene.queries.spans.SpanTermQuery;
1821
import org.apache.lucene.queryparser.classic.ParseException;
1922
import org.apache.lucene.search.BooleanClause;
2023
import org.apache.lucene.search.BooleanQuery;
@@ -26,9 +29,6 @@
2629
import org.apache.lucene.search.Query;
2730
import org.apache.lucene.search.SynonymQuery;
2831
import org.apache.lucene.search.TermQuery;
29-
import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
30-
import org.apache.lucene.search.spans.SpanNearQuery;
31-
import org.apache.lucene.search.spans.SpanTermQuery;
3232
import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
3333
import org.elasticsearch.common.xcontent.XContentBuilder;
3434
import org.elasticsearch.index.IndexSettings;

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/query/SourceConfirmedTextQueryTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
import org.apache.lucene.index.IndexWriter;
1717
import org.apache.lucene.index.LeafReaderContext;
1818
import org.apache.lucene.index.Term;
19+
import org.apache.lucene.queries.spans.SpanNearQuery;
20+
import org.apache.lucene.queries.spans.SpanQuery;
21+
import org.apache.lucene.queries.spans.SpanTermQuery;
1922
import org.apache.lucene.search.BooleanClause.Occur;
2023
import org.apache.lucene.search.BooleanQuery;
2124
import org.apache.lucene.search.CheckHits;
@@ -27,9 +30,6 @@
2730
import org.apache.lucene.search.Query;
2831
import org.apache.lucene.search.ScoreDoc;
2932
import org.apache.lucene.search.TermQuery;
30-
import org.apache.lucene.search.spans.SpanNearQuery;
31-
import org.apache.lucene.search.spans.SpanQuery;
32-
import org.apache.lucene.search.spans.SpanTermQuery;
3333
import org.apache.lucene.store.Directory;
3434
import org.elasticsearch.common.CheckedIntFunction;
3535
import org.elasticsearch.common.lucene.Lucene;

‎modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.apache.lucene.search.IndexSearcher;
1414
import org.apache.lucene.search.MatchNoDocsQuery;
1515
import org.apache.lucene.search.Query;
16+
import org.apache.lucene.search.QueryVisitor;
1617
import org.apache.lucene.search.join.JoinUtil;
1718
import org.apache.lucene.search.join.ScoreMode;
1819
import org.apache.lucene.search.similarities.Similarity;
@@ -411,6 +412,11 @@ public String toString(String s) {
411412
return "LateParsingQuery {joinField=" + joinField + "}";
412413
}
413414

415+
@Override
416+
public void visit(QueryVisitor visitor) {
417+
visitor.visitLeaf(this);
418+
}
419+
414420
public int getMinChildren() {
415421
return minChildren;
416422
}

0 commit comments

Comments
 (0)