Merge pull request #331 from MJedr/desy-dfferent-queue

add queue to the submit results payload
inspirehep · May 23, 2023 · 2fbdc83 · 2fbdc83
2 parents 3ed0466 + a6c9ea1
commit 2fbdc83
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 2 deletions.
diff --git a/docker-compose.test.py2.yml b/docker-compose.test.py2.yml
@@ -216,7 +216,7 @@ services:
       - 80:80
 
   rabbitmq:
-    image: rabbitmq:3-management
+    image: rabbitmq:3.9.11-management
     restart: "always"
     healthcheck:
       timeout: 5s

diff --git a/hepcrawl/pipelines.py b/hepcrawl/pipelines.py
@@ -141,6 +141,7 @@ def _prepare_payload(self, spider):
                 results_data=[result],
                 errors=[],
                 log_file=None,
+                spider_name=spider.name
             ) for result in self.results_data
         ]
         if spider.state.get('errors'):

diff --git a/hepcrawl/testlib/tasks.py b/hepcrawl/testlib/tasks.py
@@ -28,7 +28,7 @@ class Config(object):
 
 
 @app.task
-def submit_results(job_id, errors, results_uri, log_file=None, results_data=None):
+def submit_results(job_id, errors, results_uri, spider_name, log_file=None, results_data=None):
     """Receive the submission of the results of a crawl job."""
 
     def _extract_results_data(results_path):
@@ -54,4 +54,5 @@ def _extract_results_data(results_path):
         'errors': errors,
         'results_url': results_uri,
         'results_data': results_data,
+        'spider_name': spider_name
     }
diff --git a/tests/unit/responses/arxiv/sample_arxiv_record10_parsed.json b/tests/unit/responses/arxiv/sample_arxiv_record10_parsed.json
@@ -3,6 +3,7 @@
         "errors": [],
         "results_uri": "scrapy_feed_uri",
         "log_file": "None",
+        "spider_name": "arXiv",
         "results_data": [{
             "_collections": [
                 "Literature"