strip /api/files in desy spider

inspirehep · Mar 15, 2023 · c17f0a5 · c17f0a5
1 parent 575b189
commit c17f0a5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/hepcrawl/spiders/desy_spider.py b/hepcrawl/spiders/desy_spider.py
@@ -169,7 +169,7 @@ def _is_local_path(cls, url):
     def _get_full_uri(self, file_name, schema='https'):
         self.move_file_to_processed(file_name)
         url = self.s3_url_for_file(file_name, bucket=self.s3_output_bucket)
-        return url
+        return url.lstrip('/api/files/')
 
     def parse(self, response):
         """Parse a ``Desy`` jsonl file into a :class:`hepcrawl.utils.ParsedItem`.

diff --git a/tests/functional/desy/test_desy.py b/tests/functional/desy/test_desy.py
@@ -326,6 +326,7 @@ def test_desy_crawl_twice(expected_results, settings, cleanup):
     # preproces s3 urls
     for rec in gotten_records:
         for document in rec.get('documents', []):
+            assert '/api/files' not in document
             if settings['CRAWLER_ARGUMENTS']['s3_server'] in document['url']:
                 assert "&Expires=" in document['url']
                 document['url'] = document['url'].split('&Expires=')[0]