pipelines: improve logging

ref:inspirehep/inspirehep#2202
inspirehep · Oct 11, 2021 · e7a2ea7 · e7a2ea7
1 parent 273f264
commit e7a2ea7
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/hepcrawl/pipelines.py b/hepcrawl/pipelines.py
@@ -228,6 +228,10 @@ def close_spider(self, spider):
             )
 
             res = self.celery.send_task(task_endpoint, kwargs=kwargs)
-            logger.info('Sent celery task %s', res)
+            celery_task_info_payload = {
+                'celery_task_id': res.id,
+                'scrapy_job_id': os.environ.get('SCRAPY_JOB')
+            }
+            logger.info('Sent celery task %s', pprint.pformat(celery_task_info_payload))
 
         self._cleanup(spider)