Skip to content

Commit 0a516d9

Browse files
asimmahmood1asimmahmood1
committed
Add skip_list parameter to Numeric Field Mappers
- Add skip_list parameter to numeric field mappers (default false) - Update CHANGELOG.md with the new feature - Add comprehensive unit tests for skip_list functionality - Include test data files for validation
1 parent 82b57a8 commit 0a516d9

File tree

15 files changed

+9823
-39
lines changed

15 files changed

+9823
-39
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4444
- Make GRPC transport extensible to allow plugins to register and expose their own GRPC services ([#18516](https://github.com/opensearch-project/OpenSearch/pull/18516))
4545
- Added approximation support for range queries with now in date field ([#18511](https://github.com/opensearch-project/OpenSearch/pull/18511))
4646
- Upgrade to protobufs 0.6.0 and clean up deprecated TermQueryProtoUtils code ([#18880](https://github.com/opensearch-project/OpenSearch/pull/18880))
47+
- Add skip_list parameter to Numeric Field Mappers (default false) ([#18889](https://github.com/opensearch-project/OpenSearch/pull/18889))
4748

4849
### Changed
4950
- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))

‎ecommerce-field_mappings.json‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
{
2+
"mappings" : {
3+
"properties" : {
4+
"category" : {
5+
"type" : "text",
6+
"fields" : {
7+
"keyword" : {
8+
"type" : "keyword"
9+
}
10+
}
11+
},
12+
"currency" : {
13+
"type" : "keyword"
14+
},
15+
"customer_birth_date" : {
16+
"type" : "date"
17+
},
18+
"customer_first_name" : {
19+
"type": "text",
20+
"fields" : {
21+
"keyword" : {
22+
"type" : "keyword",
23+
"ignore_above" : 256
24+
}
25+
}
26+
},
27+
"customer_full_name" : {
28+
"type" : "text",
29+
"fields" : {
30+
"keyword" : {
31+
"type" : "keyword",
32+
"ignore_above" : 256
33+
}
34+
}
35+
},
36+
"customer_gender" : {
37+
"type" : "keyword"
38+
},
39+
"customer_id" : {
40+
"type" : "keyword"
41+
},
42+
"customer_last_name" : {
43+
"type" : "text",
44+
"fields" : {
45+
"keyword" : {
46+
"type" : "keyword",
47+
"ignore_above" : 256
48+
}
49+
}
50+
},
51+
"customer_phone" : {
52+
"type" : "keyword"
53+
},
54+
"day_of_week" : {
55+
"type" : "keyword"
56+
},
57+
"day_of_week_i" : {
58+
"type" : "integer"
59+
},
60+
"email" : {
61+
"type" : "keyword"
62+
},
63+
"event" : {
64+
"properties" : {
65+
"dataset" : {
66+
"type" : "keyword"
67+
}
68+
}
69+
},
70+
"geoip" : {
71+
"properties" : {
72+
"city_name" : {
73+
"type" : "keyword"
74+
},
75+
"continent_name" : {
76+
"type" : "keyword"
77+
},
78+
"country_iso_code" : {
79+
"type" : "keyword"
80+
},
81+
"location" : {
82+
"type" : "geo_point"
83+
},
84+
"region_name" : {
85+
"type" : "keyword"
86+
}
87+
}
88+
},
89+
"manufacturer" : {
90+
"type" : "text",
91+
"fields" : {
92+
"keyword" : {
93+
"type" : "keyword"
94+
}
95+
}
96+
},
97+
"order_date" : {
98+
"type" : "date"
99+
},
100+
"order_id" : {
101+
"type" : "keyword"
102+
},
103+
"products" : {
104+
"properties" : {
105+
"_id" : {
106+
"type" : "text",
107+
"fields" : {
108+
"keyword" : {
109+
"type" : "keyword",
110+
"ignore_above" : 256
111+
}
112+
}
113+
},
114+
"base_price" : {
115+
"type" : "half_float", "skiplist": true
116+
},
117+
"base_unit_price" : {
118+
"type" : "half_float", "skiplist": true
119+
},
120+
"category" : {
121+
"type" : "text",
122+
"fields" : {
123+
"keyword" : {
124+
"type" : "keyword"
125+
}
126+
}
127+
},
128+
"created_on" : {
129+
"type" : "date"
130+
},
131+
"discount_amount" : {
132+
"type" : "half_float"
133+
},
134+
"discount_percentage" : {
135+
"type" : "half_float"
136+
},
137+
"manufacturer" : {
138+
"type" : "text",
139+
"fields" : {
140+
"keyword" : {
141+
"type" : "keyword"
142+
}
143+
}
144+
},
145+
"min_price" : {
146+
"type" : "half_float"
147+
},
148+
"price" : {
149+
"type" : "half_float", "skiplist": true
150+
},
151+
"product_id" : {
152+
"type" : "long"
153+
},
154+
"product_name" : {
155+
"type" : "text",
156+
"fields" : {
157+
"keyword" : {
158+
"type" : "keyword"
159+
}
160+
},
161+
"analyzer" : "english"
162+
},
163+
"quantity" : {
164+
"type" : "integer", "skiplist": true
165+
},
166+
"sku" : {
167+
"type" : "keyword"
168+
},
169+
"tax_amount" : {
170+
"type" : "half_float"
171+
},
172+
"taxful_price" : {
173+
"type" : "half_float"
174+
},
175+
"taxless_price" : {
176+
"type" : "half_float"
177+
},
178+
"unit_discount_amount" : {
179+
"type" : "half_float"
180+
}
181+
}
182+
},
183+
"sku" : {
184+
"type" : "keyword"
185+
},
186+
"taxful_total_price" : {
187+
"type" : "half_float"
188+
},
189+
"taxless_total_price" : {
190+
"type" : "half_float"
191+
},
192+
"total_quantity" : {
193+
"type" : "integer"
194+
},
195+
"total_unique_products" : {
196+
"type" : "integer"
197+
},
198+
"type" : {
199+
"type" : "keyword"
200+
},
201+
"user" : {
202+
"type" : "keyword"
203+
}
204+
}
205+
}
206+
}

‎ecommerce.ndjson‎

Lines changed: 9350 additions & 0 deletions
Large diffs are not rendered by default.

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,14 @@ protected void parseCreateField(ParseContext context) throws IOException {
468468
}
469469
long scaledValue = Math.round(doubleValue * scalingFactor);
470470

471-
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(fieldType().name(), scaledValue, indexed, hasDocValues, stored);
471+
List<Field> fields = NumberFieldMapper.NumberType.LONG.createFields(
472+
fieldType().name(),
473+
scaledValue,
474+
indexed,
475+
hasDocValues,
476+
false,
477+
stored
478+
);
472479
context.doc().addAll(fields);
473480

474481
if (hasDocValues == false && (indexed || stored)) {

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/TokenCountFieldMapper.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ protected void parseCreateField(ParseContext context) throws IOException {
173173
tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements);
174174
}
175175

176-
context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, store));
176+
context.doc()
177+
.addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, false, store));
177178
}
178179

179180
/**

‎modules/percolator/src/test/java/org/opensearch/percolator/CandidateQueryTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ public void testDuel() throws Exception {
321321
document.add(new TextField(entry.getKey(), value, Field.Store.NO));
322322
}
323323
for (Integer intValue : intValues) {
324-
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false);
324+
List<Field> numberFields = NumberFieldMapper.NumberType.INTEGER.createFields("int_field", intValue, true, true, false, false);
325325
for (Field numberField : numberFields) {
326326
document.add(numberField);
327327
}
@@ -449,6 +449,7 @@ public void testDuel2() throws Exception {
449449
between(range[0], range[1]),
450450
true,
451451
true,
452+
false,
452453
false
453454
);
454455
for (Field numberField : numberFields) {

‎plugins/mapper-size/src/main/java/org/opensearch/index/mapper/size/SizeFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public void postParse(ParseContext context) throws IOException {
9999
return;
100100
}
101101
final int value = context.sourceToParse().source().length();
102-
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, true));
102+
context.doc().addAll(NumberType.INTEGER.createFields(name(), value, true, true, false, true));
103103
}
104104

105105
@Override

0 commit comments

Comments
 (0)