Working through scripts to add new publicaitons from DOIs

This proposal section works through the process of adding and uploading records to Neotoma through publication data stored online in CrossRef.

Author: SimonGoring

Proposals/publications/README.md

@@ -0,0 +1,40 @@
+# Resolving Publications in Neotoma
+
+This project is intended to help add DOIs to existing publications, support meta-data enrichment from manual citation editing, and support the addition of records to Neotoma from external sources. This workflow links the [Neotoma API](https://api.neotomadb.org) to the [CrossRef Works API](https://api.crossref.org/works), helping to validate and augment information in Neotoma.
+
+## Updating Existing Neotoma Records
+
+### Finding new DOIs
+
+For existing publications that do not include DOIs we can scan the Neotoma Publications database from the commandline:
+
+```python
+uv run src/find_potential_dois.py --limit 100 --skip 100 --output ./data/offset100.csv
+```
+
+This will return a CSV file (saved in the `--output` directory) with the Neotoma `publicationid`, current `citation`, the `doi` stored in Neotoma (generally empty) and then columns for the `newdoi`, obtained from a CrossRef search, as well as the `bibtex` citation.
+
+This output file can be manually edited, and, subsequently used as the source for data upload.
+
+![A screenshot of the edited csv file, with the "good" DOIs remaining.](./assets/editedoffset100.png)
+
+### Submitting New DOIs for Existing Records
+
+If we have a csv file with existing publication IDs and "new" DOIs, we can then upload the data from the commandline:
+
+```py
+uv run src/post_doi_bibtex.py --input ./data/offset100.csv --doi newdoi --commit True
+```
+
+The `--commit` flag allows us to test the run, to ensure that we don't accidentally upload data incorrectly. If it is set to `True` we will see a complete run:
+
+![Running the script with commit set to True](./assets/script_output.png)
+
+Otherwise the upload will end with the statement:
+
+```
+The --commit flag was set to False, rolling back operation.
+```
+
+## Inserting New Publicaitons from DOIs
+

Proposals/publications/src/find_potential_dois.py

@@ -12,13 +12,15 @@
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--output', '-O', help="A valid output filename.", type= str, default = 'output.csv')
-parser.add_argument('--limit', '-l', help="How many Neotoma publicaitons to process?", type= int, default= 100)
+parser.add_argument('--limit', '-L', help="How many Neotoma publications to process?", type= int, default= 100)
+parser.add_argument('--skip', '-S', help="How many Neotoma publications to skip?", type= int, default= 0)
 
 args = parser.parse_args()
 
 csv.field_size_limit(sys.maxsize)
 
-db_data = [j for j in [i.get('publication') for i in call_publications(limit = args.limit)] if j.get('doi') is None]
+ndb_pubs = call_publications(limit = args.limit, offset = args.skip)
+db_data = [j for j in [i.get('publication') for i in ndb_pubs] if j.get('doi') is None]
 
 # For each Neotoma publication record without a DOI:
 for i in db_data:
@@ -56,7 +58,7 @@
             i['newdoi'] = outcome.get('DOI')
             i['json'] = json.dumps(outcome)
             bibtex = return_bibtex(outcome.get('DOI'))
-            i['bibtex'] = i.get('bibtex', '') + bibtex
+            i['bibtex'] = i.get('bibtex', '') + (bibtex or '')
         else:
             print('No new match.')
 

Proposals/publications/src/insert_new_from_doi.py

@@ -1 +1,44 @@
+from publications import return_bibtex, parse_raw, add_citation
+import argparse
+import psycopg2
+import json
+import dotenv
+import os
 
+parser = argparse.ArgumentParser()
+parser.add_argument('--input', '-I', help="A valid output filename.", type= str, default = 'output.csv')
+parser.add_argument('--doi', '-d', help="Which column contains the DOI for upload?", type=str)
+parser.add_argument('--commit', '-c', help="Should we commit the data to the database?", type=bool, default= False)
+
+args = parser.parse_args()
+
+dotenv.load_dotenv()
+
+dbauth = json.loads(os.getenv('DBAUTH'))
+
+conn = psycopg2.connect(**dbauth, connect_timeout=5)
+
+QUERY = """
+    INSERT INTO ndb.publications(citation, doi, bibtex)
+    VALUES(%(citation)s, %(doi)s, %(bibtex)s);"""
+
+contents = parse_raw(args.input)
+
+for i in [i for i in contents if i is not None]:
+    bibtex = return_bibtex(i)
+    citation = add_citation(bibtex)
+    if bibtex is not None and citation is not None:
+        with conn.cursor() as cur:
+                cur.execute(QUERY, {'doi': i,
+                                    'bibtex': bibtex,
+                                    'citation': citation})
+                if args.commit:
+                    print(f'Committing the following citation:\n{citation}.')
+                    conn.commit()
+                cur.close()
+
+if args.commit is True:
+     conn.commit()
+else:
+     conn.rollback()
+     print("The --commit flag was set to False, rolling back operation.")

Proposals/publications/src/post_doi_bibtex.py

@@ -8,8 +8,15 @@
 import os
 import json
 import csv
-import sys
 from publications import return_bibtex
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--input', '-I', help="A valid output filename.", type= str, default = 'output.csv')
+parser.add_argument('--doi', '-d', help="Which column contains the DOI for upload?", type=str, default = 'newdoi')
+parser.add_argument('--commit', '-c', help="Should we commit the data to the database?", type=bool, default= False)
+
+args = parser.parse_args()
 
 dotenv.load_dotenv()
 
@@ -23,19 +30,23 @@
            DO UPDATE SET    doi = EXCLUDED.doi,
                          bibtex = EXCLUDED.bibtex;"""
 
-with open(sys.argv[1], 'r') as newdois:
+with open(args.input, 'r') as newdois:
     reader = csv.DictReader(newdois)
-    if sys.argv[2] not in reader.fieldnames:
-        raise KeyError(f'The value {sys.argv[2]} is not a column heading in {sys.argv[1]}')
+    if args.doi not in reader.fieldnames:
+        raise KeyError(f'The value {args.doi} is not a column heading in {args.input}')
     for i in reader:
-        bibtex = return_bibtex(i.get(sys.argv[2]))
+        bibtex = return_bibtex(i.get(args.doi))
         if bibtex is not None:
             with conn.cursor() as cur:
-                cur.execute(QUERY, {'doi': i.get(sys.argv[2]),
+                cur.execute(QUERY, {'doi': i.get(args.doi),
                                     'bibtex': bibtex,
                                     'publicationid': int(i.get('publicationid'))})
-                conn.commit()
+                if args.commit:
+                    conn.commit()
                 cur.close()
             print(f'Added bibtex and DOI for publication {i.get('publicationid')}.')
         else:
-            print(f'Could not resolve bibtex/ DOI for publication {i.get('publicationid')}: {i.get(sys.argv[2])}.')
+            print(f'Could not resolve bibtex/DOI for publication {i.get('publicationid')}: {i.get(args.doi)}.')
+    if not args.commit:
+        print("The --commit flag was set to False, rolling back operation.")
+        conn.rollback()

Proposals/publications/src/publications/__init__.py

@@ -4,3 +4,4 @@
 from .clean_doi import clean_doi as clean_doi
 from .parse_raw import parse_raw as parse_raw
 from .return_bibtex import return_bibtex as return_bibtex
+from .add_citation import add_citation as add_citation

Proposals/publications/src/publications/add_citation.py

@@ -2,13 +2,13 @@
 from pybtex.database import parse_string
 
 def add_citation(bibtex:str) -> str:
-    """_Return an APA citation from Bibtex_
+    """_Return an APA citation from Bibtex._
 
     Args:
-        bibtex (str): _A properly formatted Bibtex string_
+        bibtex (str): _A properly formatted Bibtex string._
 
     Returns:
-        str: _An APA formatted citation string generated from the bibtex field._
+        str: _An APA formatted citation string generated from the Bibtex field._
     >>> bibtex = "@article{Davis_1979, title={Wetland Succession, Fire and the Pollen Record: A Midwestern Example}, volume={102}, ISSN={0003-0031}, url={http://dx.doi.org/10.2307/2425069}, DOI={10.2307/2425069}, number={1}, journal={American Midland Naturalist}, publisher={JSTOR}, author={Davis, Anthony M.}, year={1979}, month=jul, pages={86} }"
     >>> add_citation(bibtex)
     'Davis, A. M. (1979 , July). Wetland succession, fire and the pollen record: a midwestern example. American Midland Naturalist, 102(1), 86. URL: http://dx.doi.org/10.2307/2425069, doi:10.2307/2425069'
@@ -22,8 +22,11 @@ def add_citation(bibtex:str) -> str:
     """
     aa = parse_string(bibtex, "bibtex")
     APA = find_plugin('pybtex.style.formatting', 'apa')()
-    formattedBib = APA.format_bibliography(aa)
-    if len(formattedBib.entries) == 0:
-        raise ValueError(f"The passed Bibtex string:\n\n{bibtex}\n\nis not valid Bibtex.")
-    else:
-        return [entry.text.render_as('text') for entry in formattedBib][0]
+    try:
+        formattedBib = APA.format_bibliography(aa)
+        if len(formattedBib.entries) == 0:
+            raise ValueError(f"The passed Bibtex string:\n\n{bibtex}\n\nis not valid Bibtex.")
+        else:
+            return [entry.text.render_as('text') for entry in formattedBib][0]
+    except Exception as e:
+        

Proposals/publications/src/publications/call_publications.py

@@ -2,13 +2,16 @@
 from requests.exceptions import ReadTimeout
 from json import loads
 
-def call_publications(limit:int = 100):
+def call_publications(limit:int = 100, offset:int = 0):
     """_Get Publications from Neotoma_
     Returns:
         _dict_: _A dictionary of Neotoma Publications_
     """    
     try:
-        result = get(f"https://api.neotomadb.org/v2.0/data/publications?limit={limit}", timeout = 10)
+        result = get("https://api.neotomadb.org/v2.0/data/publications",
+                     params = {"limit": limit,
+                               "offset": offset},
+                    timeout = 10)
     except ReadTimeout as e:
         print(e)
         return None