16
16
#
17
17
18
18
import argparse
19
+ import os
20
+ import time
19
21
from confluent_kafka import Consumer , KafkaError , KafkaException
20
22
from verifiable_client import VerifiableClient
21
23
@@ -95,10 +97,10 @@ def on_revoke(self, consumer, partitions):
95
97
# Send final consumed records prior to rebalancing to make sure
96
98
# latest consumed is in par with what is going to be committed.
97
99
self .send_records_consumed (immediate = True )
100
+ self .do_commit (immediate = True , async = False )
98
101
self .assignment = list ()
99
102
self .assignment_dict = dict ()
100
103
self .send_assignment ('revoked' , partitions )
101
- self .do_commit (immediate = True )
102
104
103
105
def on_commit (self , err , partitions ):
104
106
""" Offsets Committed callback """
@@ -125,6 +127,10 @@ def on_commit(self, err, partitions):
125
127
pd ['error' ] = str (p .error )
126
128
d ['offsets' ].append (pd )
127
129
130
+ if len (self .assignment ) == 0 :
131
+ self .dbg ('Not sending offsets_committed: No current assignment: would be: %s' % d )
132
+ return
133
+
128
134
self .send (d )
129
135
130
136
def do_commit (self , immediate = False , async = None ):
@@ -149,15 +155,33 @@ def do_commit(self, immediate=False, async=None):
149
155
(self .consumed_msgs - self .consumed_msgs_at_last_commit ,
150
156
async_mode ))
151
157
152
- try :
153
- self .consumer .commit (async = async_mode )
154
- except KafkaException as e :
155
- if e .args [0 ].code () == KafkaError ._WAIT_COORD :
156
- self .dbg ('Ignoring commit failure, still waiting for coordinator' )
157
- elif e .args [0 ].code () == KafkaError ._NO_OFFSET :
158
- self .dbg ('No offsets to commit' )
159
- else :
160
- raise
158
+ retries = 3
159
+ while True :
160
+ try :
161
+ self .dbg ('Commit' )
162
+ offsets = self .consumer .commit (async = async_mode )
163
+ self .dbg ('Commit done: offsets %s' % offsets )
164
+
165
+ if not async_mode :
166
+ self .on_commit (None , offsets )
167
+
168
+ break
169
+
170
+ except KafkaException as e :
171
+ if e .args [0 ].code () == KafkaError ._NO_OFFSET :
172
+ self .dbg ('No offsets to commit' )
173
+ break
174
+ elif e .args [0 ].code () in (KafkaError .REQUEST_TIMED_OUT ,
175
+ KafkaError .NOT_COORDINATOR_FOR_GROUP ,
176
+ KafkaError ._WAIT_COORD ):
177
+ self .dbg ('Commit failed: %s (%d retries)' % (str (e ), retries ))
178
+ if retries <= 0 :
179
+ raise
180
+ retries -= 1
181
+ time .sleep (1 )
182
+ continue
183
+ else :
184
+ raise
161
185
162
186
self .consumed_msgs_at_last_commit = self .consumed_msgs
163
187
@@ -168,7 +192,7 @@ def msg_consume(self, msg):
168
192
# ignore EOF
169
193
pass
170
194
else :
171
- self .err ('Consume failed: %s' % msg .error (), term = True )
195
+ self .err ('Consume failed: %s' % msg .error (), term = False )
172
196
return
173
197
174
198
if False :
@@ -192,6 +216,7 @@ def msg_consume(self, msg):
192
216
193
217
self .consumed_msgs += 1
194
218
219
+ self .consumer .store_offsets (message = msg )
195
220
self .send_records_consumed (immediate = False )
196
221
self .do_commit (immediate = False )
197
222
@@ -229,7 +254,11 @@ def to_dict(self):
229
254
args = vars (parser .parse_args ())
230
255
231
256
conf = {'broker.version.fallback' : '0.9.0' ,
232
- 'default.topic.config' : dict ()}
257
+ 'default.topic.config' : dict (),
258
+ # Do explicit manual offset stores to avoid race conditions
259
+ # where a message is consumed from librdkafka but not yet handled
260
+ # by the Python code that keeps track of last consumed offset.
261
+ 'enable.auto.offset.store' : False }
233
262
234
263
VerifiableClient .set_config (conf , args )
235
264
@@ -239,6 +268,7 @@ def to_dict(self):
239
268
vc .use_auto_commit = args ['enable.auto.commit' ]
240
269
vc .max_msgs = args ['max_messages' ]
241
270
271
+ vc .dbg ('Pid %d' % os .getpid ())
242
272
vc .dbg ('Using config: %s' % conf )
243
273
244
274
vc .dbg ('Subscribing to %s' % args ['topic' ])
@@ -261,6 +291,8 @@ def to_dict(self):
261
291
vc .msg_consume (msg )
262
292
263
293
except KeyboardInterrupt :
294
+ vc .dbg ('KeyboardInterrupt' )
295
+ vc .run = False
264
296
pass
265
297
266
298
vc .dbg ('Closing consumer' )
0 commit comments