several: improves several things

* Fixes bf:Organisation records with conference=true.
* Corrects exception handling (raise of Exceptions).
* Improves working with PersistentIdentifiers.
* Makes CSV diff cli less memory intensive.
* Adds REDIS, Elastic search and time stamp monitoring.
* Updates dependencies.

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

rero_mef/marctojson/do_idref_agent.py

@@ -367,7 +367,8 @@ def trans_idref_authorized_access_point(self):
         if self.marc.get_fields('210'):
             tag = '210'
             agent = 'bf:Organisation'
-            self.json_dict['conference'] = self.marc['210'].indicators[0] == 1
+            self.json_dict['conference'] = \
+                self.marc['210'].indicators[0] == '1'
             subfields = {'a': ', ', 'b': '. ', 'c': ', ', 'd': ' ; ',
                          'e': ' ; ', 'f': ' ; ', 'x': ' - -'}
             tag_grouping = [

rero_mef/utils.py

@@ -56,7 +56,8 @@
 from invenio_records_rest.utils import obj_or_import_string
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 from pymarc.marcxml import parse_xml_to_array
-from sickle import Sickle
+from sickle import Sickle, oaiexceptions
+from sickle.iterator import OAIItemIterator
 from sickle.oaiexceptions import NoRecordsMatch
 
 
@@ -147,6 +148,58 @@ def oai_set_last_run(name, date, verbose=False):
     return None
 
 
+class MyOAIItemIterator(OAIItemIterator):
+    """OAI item iterator with accessToken."""
+
+    def next_resumption_token_and_items(self):
+        """Get next resumtion token and items."""
+        self.resumption_token = self._get_resumption_token()
+        self._items = self.oai_response.xml.iterfind(
+            './/' + self.sickle.oai_namespace + self.element)
+
+    def _next_response(self):
+        """Get the next response from the OAI server."""
+        params = self.params
+        access_token = params.get('accessToken')
+        if self.resumption_token:
+            params = {
+                'resumptionToken': self.resumption_token.token,
+                'verb': self.verb
+            }
+        if access_token:
+            params['accessToken'] = access_token
+
+        self.oai_response = self.sickle.harvest(**params)
+        error = self.oai_response.xml.find(
+            './/' + self.sickle.oai_namespace + 'error')
+        if error is not None:
+            code = error.attrib.get('code', 'UNKNOWN')
+            description = error.text or ''
+            try:
+                raise getattr(
+                    oaiexceptions, code[0].upper() + code[1:])(description)
+            except AttributeError:
+                raise oaiexceptions.OAIError(description)
+        if self.resumption_token:
+            # Test we got a complete response ('resumptionToken' in xml)
+            resumption_token_element = self.oai_response.xml.find(
+                './/' + self.sickle.oai_namespace + 'resumptionToken')
+
+            if resumption_token_element is None:
+                msg = ('ERROR HARVESTING incomplete response:'
+                       '{cursor} {token}').format(
+                    cursor=self.resumption_token.cursor,
+                    token=self.resumption_token.token
+                )
+                current_app.logger.error(msg)
+                sleep(60)
+            else:
+                self.next_resumption_token_and_items()
+        else:
+            # first time
+            self.next_resumption_token_and_items()
+
+
 def oai_process_records_from_dates(name, sickle, oai_item_iterator,
                                    transformation, record_cls, max_retries=0,
                                    access_token=None, days_spann=30,

tests/agents/test_idref_contributions_person.py

@@ -494,6 +494,38 @@ def test_idref_authorized_access_point():
             'Brontë, Charlotte, écrivain, biographe, 1816-1855',
         'bf:Agent': 'bf:Person'
     }
+    """Test Authorized access point representing a organisation 210 $ab9"""
+    xml_part_to_add = """
+        <datafield ind1=" " ind2=" " tag="210">
+            <subfield code="9">0</subfield>
+            <subfield code="a">Stift Alter Dom Sankt Pauli</subfield>
+            <subfield code="c">Münster in Westfalen, Allemagne</subfield>
+        </datafield>
+     """
+    trans = trans_prep('idref', xml_part_to_add)
+    trans.trans_idref_authorized_access_point()
+    assert trans.json == {
+        'authorized_access_point': 'Stift Alter Dom Sankt Pauli ( Münster in '
+                                   'Westfalen, Allemagne )',
+        'bf:Agent': 'bf:Organisation',
+        'conference': False,
+    }
+    """Test Authorized access point representing a organisation 210 $ab9"""
+    xml_part_to_add = """
+        <datafield ind1="1" ind2=" " tag="210">
+            <subfield code="9">0</subfield>
+            <subfield code="a">Stift Alter Dom Sankt Pauli</subfield>
+            <subfield code="c">Münster in Westfalen, Allemagne</subfield>
+        </datafield>
+     """
+    trans = trans_prep('idref', xml_part_to_add)
+    trans.trans_idref_authorized_access_point()
+    assert trans.json == {
+        'authorized_access_point': 'Stift Alter Dom Sankt Pauli ( Münster in '
+                                   'Westfalen, Allemagne )',
+        'bf:Agent': 'bf:Organisation',
+        'conference': True,
+    }
 
 
 def test_idref_authorized_access_point_missing():