Skip to content

Commit 5d3894d

Browse files
joegallojoegallo
authored
[8.12] Ingest correctly handle upsert operations and drop processors together ( #104585) (#104884)
1 parent 4f94a2a commit 5d3894d

File tree

5 files changed

+289
-109
lines changed

5 files changed

+289
-109
lines changed

‎docs/changelog/104585.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 104585
2+
summary: Ingest correctly handle upsert operations and drop processors together
3+
area: Ingest Node
4+
type: bug
5+
issues:
6+
- 36746

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/220_drop_processor.yml

Lines changed: 170 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,185 @@
11
---
22
teardown:
3-
- do:
4-
ingest.delete_pipeline:
5-
id: "my_pipeline"
6-
ignore: 404
3+
- do:
4+
indices.delete:
5+
index: "test"
6+
ignore_unavailable: true
7+
- do:
8+
ingest.delete_pipeline:
9+
id: "my_pipeline"
10+
ignore: 404
711

812
---
913
"Test Drop Processor":
10-
- do:
11-
ingest.put_pipeline:
12-
id: "my_pipeline"
13-
body: >
14-
{
15-
"description" : "pipeline with drop",
16-
"processors" : [
17-
{
18-
"drop" : {
19-
"if": "ctx.foo == 'bar'"
14+
- do:
15+
ingest.put_pipeline:
16+
id: "my_pipeline"
17+
body: >
18+
{
19+
"description" : "pipeline with drop",
20+
"processors" : [
21+
{
22+
"drop" : {
23+
"if": "ctx.foo == 'bar'"
24+
}
2025
}
21-
}
22-
]
26+
]
27+
}
28+
- match: { acknowledged: true }
29+
30+
- do:
31+
index:
32+
index: test
33+
id: "1"
34+
pipeline: "my_pipeline"
35+
body: {
36+
foo: "bar"
2337
}
24-
- match: { acknowledged: true }
25-
26-
- do:
27-
index:
28-
index: test
29-
id: "1"
30-
pipeline: "my_pipeline"
31-
body: {
32-
foo: "bar"
33-
}
34-
35-
- do:
36-
index:
37-
index: test
38-
id: "2"
39-
pipeline: "my_pipeline"
40-
body: {
41-
foo: "blub"
42-
}
43-
44-
- do:
45-
catch: missing
46-
get:
47-
index: test
48-
id: "1"
49-
- match: { found: false }
50-
51-
- do:
52-
get:
53-
index: test
54-
id: "2"
55-
- match: { _source.foo: "blub" }
38+
39+
- do:
40+
index:
41+
index: test
42+
id: "2"
43+
pipeline: "my_pipeline"
44+
body: {
45+
foo: "blub"
46+
}
47+
48+
- do:
49+
catch: missing
50+
get:
51+
index: test
52+
id: "1"
53+
- match: { found: false }
54+
55+
- do:
56+
get:
57+
index: test
58+
id: "2"
59+
- match: { _source.foo: "blub" }
5660

5761
---
5862
"Test Drop Processor On Failure":
59-
- do:
60-
ingest.put_pipeline:
61-
id: "my_pipeline_with_failure"
62-
body: >
63-
{
64-
"description" : "pipeline with on failure drop",
65-
"processors": [
63+
- do:
64+
ingest.put_pipeline:
65+
id: "my_pipeline_with_failure"
66+
body: >
67+
{
68+
"description" : "pipeline with on failure drop",
69+
"processors": [
70+
{
71+
"fail": {
72+
"message": "failed",
73+
"on_failure": [
74+
{
75+
"drop": {}
76+
}
77+
]
78+
}
79+
}
80+
]
81+
}
82+
- match: { acknowledged: true }
83+
84+
- do:
85+
index:
86+
index: test
87+
id: "3"
88+
pipeline: "my_pipeline_with_failure"
89+
body: {
90+
foo: "bar"
91+
}
92+
93+
- do:
94+
catch: missing
95+
get:
96+
index: test
97+
id: "3"
98+
99+
---
100+
"Test Drop Processor with Upsert (_bulk)":
101+
- skip:
102+
version: ' - 8.12.0'
103+
reason: 'https://github.com/elastic/elasticsearch/issues/36746 fixed in 8.12.1'
104+
- do:
105+
ingest.put_pipeline:
106+
id: "my_pipeline"
107+
body: >
108+
{
109+
"processors": [
66110
{
67-
"fail": {
68-
"message": "failed",
69-
"on_failure": [
70-
{
71-
"drop": {}
72-
}
73-
]
111+
"drop": {
74112
}
75113
}
76114
]
77-
}
78-
- match: { acknowledged: true }
79-
80-
- do:
81-
index:
82-
index: test
83-
id: "3"
84-
pipeline: "my_pipeline_with_failure"
85-
body: {
86-
foo: "bar"
87-
}
88-
89-
- do:
90-
catch: missing
91-
get:
92-
index: test
93-
id: "3"
115+
}
116+
- match: { acknowledged: true }
117+
118+
- do:
119+
bulk:
120+
refresh: true
121+
pipeline: "my_pipeline"
122+
body:
123+
- update:
124+
_index: test
125+
_id: 4
126+
- '{"upsert":{"some":"fields"},"script":"ctx"}'
127+
- match: { errors: false }
128+
- match: { items.0.update._index: test }
129+
- match: { items.0.update._id: "4" }
130+
- match: { items.0.update._version: -3 }
131+
- match: { items.0.update.result: noop }
132+
- match: { items.0.update.status: 200 }
133+
134+
- do:
135+
catch: missing
136+
get:
137+
index: test
138+
id: "4"
139+
140+
---
141+
"Test Drop Processor with Upsert (_update)":
142+
- skip:
143+
version: ' - 8.12.0'
144+
reason: 'https://github.com/elastic/elasticsearch/issues/36746 fixed in 8.12.1'
145+
- do:
146+
ingest.put_pipeline:
147+
id: "my_pipeline"
148+
body: >
149+
{
150+
"processors": [
151+
{
152+
"drop": {
153+
}
154+
}
155+
]
156+
}
157+
- match: { acknowledged: true }
158+
159+
- do:
160+
indices.create:
161+
index: test
162+
body:
163+
settings:
164+
index:
165+
default_pipeline: "my_pipeline"
166+
167+
- do:
168+
update:
169+
index: test
170+
id: "5"
171+
body:
172+
script:
173+
source: "ctx._source.foo = 'bar'"
174+
upsert:
175+
foo: "bar"
176+
177+
- match: { _index: test }
178+
- match: { _id: "5" }
179+
- match: { result: noop }
94180

181+
- do:
182+
catch: missing
183+
get:
184+
index: test
185+
id: "5"

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/60_fail.yml

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
---
22
teardown:
3+
- do:
4+
indices.delete:
5+
index: "test"
6+
ignore_unavailable: true
37
- do:
48
ingest.delete_pipeline:
59
id: "my_pipeline"
@@ -10,7 +14,7 @@ teardown:
1014
- do:
1115
ingest.put_pipeline:
1216
id: "my_pipeline"
13-
body: >
17+
body: >
1418
{
1519
"description": "_description",
1620
"processors": [
@@ -36,7 +40,7 @@ teardown:
3640
- do:
3741
ingest.put_pipeline:
3842
id: "my_pipeline"
39-
body: >
43+
body: >
4044
{
4145
"description": "_description",
4246
"processors": [
@@ -69,3 +73,87 @@ teardown:
6973
index: test
7074
id: "1"
7175
- match: { _source.error_message: "fail_processor_ran" }
76+
77+
---
78+
"Test Fail Processor with Upsert (bulk)":
79+
- skip:
80+
version: ' - 8.12.0'
81+
reason: 'https://github.com/elastic/elasticsearch/issues/36746 fixed in 8.12.1'
82+
- do:
83+
ingest.put_pipeline:
84+
id: "my_pipeline"
85+
body: >
86+
{
87+
"processors": [
88+
{
89+
"fail": {
90+
"message": "error-message"
91+
}
92+
}
93+
]
94+
}
95+
- match: { acknowledged: true }
96+
97+
- do:
98+
bulk:
99+
refresh: true
100+
pipeline: "my_pipeline"
101+
body:
102+
- update:
103+
_index: test
104+
_id: 3
105+
- '{"upsert":{"some":"fields"},"script":"ctx"}'
106+
- match: { errors: true }
107+
- match: { items.0.update._index: test }
108+
- match: { items.0.update._id: "3" }
109+
- match: { items.0.update.status: 500 }
110+
- match: { items.0.update.error.type: fail_processor_exception }
111+
- match: { items.0.update.error.reason: /error-message/ }
112+
113+
- do:
114+
catch: missing
115+
get:
116+
index: test
117+
id: "3"
118+
119+
---
120+
"Test Fail Processor with Upsert (_update)":
121+
- do:
122+
ingest.put_pipeline:
123+
id: "my_pipeline"
124+
body: >
125+
{
126+
"processors": [
127+
{
128+
"fail": {
129+
"message": "error-message"
130+
}
131+
}
132+
]
133+
}
134+
- match: { acknowledged: true }
135+
136+
- do:
137+
indices.create:
138+
index: test
139+
body:
140+
settings:
141+
index:
142+
default_pipeline: "my_pipeline"
143+
144+
- do:
145+
update:
146+
index: test
147+
id: "4"
148+
body:
149+
script:
150+
source: "ctx._source.foo = 'bar'"
151+
upsert:
152+
foo: "bar"
153+
catch: /error-message/
154+
155+
- do:
156+
catch: missing
157+
get:
158+
index: test
159+
id: "4"

0 commit comments

Comments
 (0)