feat: config oai server client

Signed-off-by: Peter Weber <Peter.Weber@rero.ch>
Co-authored-by: Bertrand Zuchuat <Bertrand.Zuchuat@rero.ch>

MANIFEST.in

@@ -26,7 +26,7 @@ include scripts/server
 include scripts/setup
 include scripts/update
 prune docs/_build
-recursive-include rero_ebooks *.po *.pot *.mo
+# recursive-include rero_ebooks *.po *.pot *.mo
 recursive-include docker *.cfg *.conf *.crt *.ini *.key *.pem *.sh
 
 # added by check_manifest.py
@@ -49,6 +49,7 @@ recursive-include tests *.py
 
 # added by check_manifest.py
 recursive-include data *.xml
+recursive-include data *.yml
 recursive-include rero_ebooks *.json
 
 # added by check_manifest.py

data/oaisources.yml

@@ -0,0 +1,18 @@
+#
+# Copyright (C) 2018 RERO.
+#
+# RERO Ebooks is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+#
+# OAI-PMH connection settings
+
+VS:
+    baseurl: https://mediatheque-valais.cantookstation.eu/oai.xml
+    metadataprefix: marc21
+    comment: 'cantook'
+    setspecs: ''
+NJ:
+    baseurl: https://bm.ebibliomedia.ch/oai.xml
+    metadataprefix: marc21
+    comment: 'cantook'
+    setspecs: ''

docker-compose.yml

@@ -23,3 +23,10 @@ services:
     extends:
       file: docker-services.yml
       service: es
+        # Monitoring
+  flower:
+    extends:
+      file: docker-services.yml
+      service: flower
+    links:
+      - mq

rero_ebooks/__init__.py

@@ -9,6 +9,7 @@
 
 from __future__ import absolute_import, print_function
 
+from .ext import ReroEBooks
 from .version import __version__
 
-__all__ = ('__version__', )
+__all__ = ('__version__', 'ReroEBooks')

rero_ebooks/cli.py

@@ -30,10 +30,13 @@
 import sys
 
 import click
+import yaml
 from flask.cli import with_appcontext
+from invenio_oaiharvester.cli import oaiharvester
 from invenio_records.cli import records
 
-from rero_ebooks.api import Ebook
+from .api import Ebook
+from .utils import add_oai_source
 
 
 @records.command()
@@ -54,3 +57,50 @@ def create_or_update(source, verbose, vendor):
             record, vendor=vendor, dbcommit=True, reindex=True
         )
         click.echo('record uuid: ' + str(record.id) + '| ' + status)
+
+
+@oaiharvester.command('addsource')
+@click.argument('name')
+@click.argument('baseurl')
+@click.option('-m', '--metadataprefix', default='marc21',
+              help='The prefix for the metadata')
+@click.option('-s', '--setspecs', default='',
+              help='The ‘set’ criteria for the harvesting')
+@click.option('-c', '--comment', default='',
+              help='Comment')
+@with_appcontext
+def add_oai_source_config(name, baseurl, metadataprefix, setspecs, comment):
+    """Add OAIHarvestConfig."""
+    click.echo('Add OAIHarvestConfig: {0} '.format(name), nl=False)
+    add_oai_source(
+        name=name,
+        baseurl=baseurl,
+        metadataprefix=metadataprefix,
+        setspecs=setspecs,
+        comment=comment
+    )
+    click.secho('Ok', fg='green')
+
+
+@oaiharvester.command('initconfig')
+@click.argument('configfile', type=click.File('rb'))
+@with_appcontext
+def init_oai_harvest_config(configfile):
+    """Init OAIHarvestConfig."""
+    configs = yaml.load(configfile)
+    for name, values in sorted(configs.items()):
+        baseurl = values['baseurl']
+        metadataprefix = values.get('metadataprefix', 'marc21')
+        setspecs = values.get('setspecs', '')
+        comment = values.get('comment', '')
+        click.echo(
+            'Add OAIHarvestConfig: {0} {1} '.format(name, baseurl), nl=False
+        )
+        add_oai_source(
+            name=name,
+            baseurl=baseurl,
+            metadataprefix=metadataprefix,
+            setspecs=setspecs,
+            comment=comment
+        )
+        click.secho('Ok', fg='green')

rero_ebooks/ext.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2018 RERO.
+#
+# RERO Ebooks is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""RERO DOC Invenio application."""
+
+
+from invenio_oaiharvester.signals import oaiharvest_finished
+
+from . import config
+from .receivers import publish_harvested_records
+
+
+class ReroEBooks(object):
+    """ReroEBooks App extension."""
+
+    def __init__(self, app=None):
+        """Extension initialization."""
+        if app:
+            self.init_app(app)
+        self.register_signals(app)
+
+    def init_app(self, app):
+        """Flask application initialization."""
+        self.init_config(app)
+        app.extensions['reroebooks-app'] = self
+
+    def init_config(self, app):
+        """Initialize configuration."""
+        for k in dir(config):
+            if k.startswith('REROEBOOKS_APP_'):
+                app.config.setdefault(k, getattr(config, k))
+
+    @staticmethod
+    def register_signals(app):
+        """Register Zenodo Deposit signals."""
+        oaiharvest_finished.connect(publish_harvested_records,
+                                    weak=False)

rero_ebooks/receivers.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2018 RERO.
+#
+# RERO Ebooks is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Signals connections for RERO ebooks."""
+
+from dojson.contrib.marc21.utils import create_record
+
+from rero_ebooks.tasks import create_records
+
+from .dojson.marc21 import marc21
+
+
+def publish_harvested_records(sender=None, records=[], *args, **kwargs):
+    """Create, index the harvested records."""
+    # name = kwargs['name']
+    converted_records = []
+    for record in records:
+        rec = create_record(record.xml)
+        rec = marc21.do(rec)
+        converted_records.append(rec)
+    verbose = kwargs.get('verbose', False)
+    create_records(converted_records, verbose=verbose)

rero_ebooks/tasks.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2018 RERO.
+#
+# RERO Ebooks is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Celery tasks to create records."""
+
+from __future__ import absolute_import, print_function
+
+import click
+from celery import shared_task
+from invenio_oaiharvester.api import list_records
+from invenio_oaiharvester.signals import oaiharvest_finished
+
+from .api import Ebook
+
+
+@shared_task(ignore_result=True)
+def create_records(records, verbose=False):
+    """Records creation and indexing."""
+    for record in records:
+        rec, status = Ebook.create_or_update(
+            record,
+            # TODO vendor config
+            vendor='cantook',
+            dbcommit=True,
+            reindex=True
+        )
+        if verbose:
+            click.echo('record uuid: ' + str(rec.id) + ' | ' + status)
+        # TODO bulk update and reindexing
+
+
+@shared_task(ignore_result=True)
+def harvest(source, verbose=False):
+    """Async source harvesting."""
+    records = None
+    request, records = list_records(
+        name=source
+    )
+    if records:
+        oaiharvest_finished.send(
+            request,
+            records=records,
+            name=source,
+            verbose=verbose
+        )

rero_ebooks/utils.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2018 RERO.
+#
+# RERO Ebooks is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Utilities."""
+
+
+from flask import current_app
+from invenio_db import db
+from invenio_oaiharvester.models import OAIHarvestConfig
+
+
+def add_oai_source(name, baseurl, metadataprefix='marc21',
+                   setspecs='', comment=''):
+    """Add OAIHarvestConfig."""
+    with current_app.app_context():
+        source = OAIHarvestConfig(
+            name=name,
+            baseurl=baseurl,
+            metadataprefix=metadataprefix,
+            setspecs=setspecs,
+            comment=comment
+        )
+        source.save()
+        db.session.commit()

scripts/setup

@@ -15,6 +15,7 @@ invenio db init create
 invenio index destroy --force --yes-i-know
 invenio index init --force
 invenio index queue init purge
+invenio oaiharvester initconfig data/oaisources.yml
 
 # Create records (in progress...)
 # ===============================

setup.py

@@ -59,7 +59,8 @@
     'invenio-oaiserver>=1.0.0,<1.1.0',
     'invenio-pidstore>=1.0.0,<1.1.0',
     'invenio-records>=1.0.0,<1.1.0',
-    'invenio-oaiharvester>=1.0.0a4'
+    'invenio-oaiharvester>=1.0.0a4',
+    'PyYAML>=3.13'
 ]
 
 packages = find_packages()
@@ -89,6 +90,9 @@
         'console_scripts': [
             'rero-ebooks = invenio_app.cli:cli',
         ],
+        'invenio_base.apps': [
+            'rero_ebooks = rero_ebooks:ReroEBooks',
+        ],
         'invenio_config.module': [
             'rero_ebooks = rero_ebooks.config',
         ],
@@ -108,6 +112,7 @@
             'cantookmarc21 = rero_ebooks.dojson.marc21:marc21',
         ],
         'flask.commands': [
+            'oaiharvester = rero_ebooks.cli:oaiharvester'
             'records = rero_ebooks.cli:records',
         ],
         'rero_ebooks.marc21': [
@@ -126,6 +131,10 @@
             'bd80x83x = rero_ebooks.dojson.marc21.fields.bd80x83x',
             'bd84188x = rero_ebooks.dojson.marc21.fields.bd84188x'
         ],
+        'invenio_celery.tasks': [
+            'rero_ebooks = rero_ebooks.tasks'
+        ]
+
     },
     extras_require=extras_require,
     install_requires=install_requires,