Merge branch 'feature/api_refactor' into feature/api_refactor_search

config/database_versions.py

@@ -1,5 +1,6 @@
 # Used for automatic database upgrades and version checks
 DATABASE_VERSIONS = [
+    '8.8.0',
     '8.6.0',
     '8.5.0',
     '8.4.0',

config/default.py

@@ -50,7 +50,7 @@
 # e.g. PROXIES = {
 #    'http': 'http://someurl.org:8080',
 #    'https': 'http://someurl.org:8080'}
-PROXIES = None
+PROXIES: dict[str, str] = {}
 
 # Table options
 TABLE_ROWS = {10: '10', 25: '25', 50: '50', 100: '100'}

install/3_data_web.sql

@@ -15,7 +15,7 @@ INSERT INTO web.user (username, password, active, email, group_id) VALUES (
   (SELECT id FROM web.group WHERE name = 'admin'));
 
 INSERT INTO web.settings (name, value) VALUES
-  ('database_version', '8.6.0'),
+  ('database_version', '8.8.0'),
   ('api_public', ''),
   ('default_language', 'en'),
   ('table_rows', '25'),

install/upgrade/8.8.0.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+-- Raise database version
+UPDATE web.settings SET value = '8.8.0' WHERE name = 'database_version';
+
+-- #2357: Fix possible wrong direction for reference links to files
+UPDATE model.link
+SET (domain_id, range_id) = (range_id, domain_id)
+WHERE id IN (
+	SELECT l.id
+	FROM model.link l
+	JOIN model.entity d ON l.domain_id = d.id AND d.openatlas_class_name = 'file'
+	JOIN model.entity r ON l.range_id = r.id AND r.openatlas_class_name IN ('bibliography', 'edition', 'external_reference')
+	WHERE l.property_code = 'P67');
+
+END;

install/upgrade/upgrade.md

@@ -17,6 +17,9 @@ then run the database upgrade script, then restart Apache:
     sudo python3 install/upgrade/database_upgrade.py
     sudo service apache2 restart
 
+### 8.7.0 to 8.8.0
+8.8.0.sql is needed but will be taken care of by the database upgrade script.
+
 ### 8.6.x to 8.7.0
 No database update is required but an additional Python package is needed:
 

openatlas/database/imports.py

@@ -58,6 +58,16 @@ def check_origin_ids(project_id: int, origin_ids: list[str]) -> list[str]:
     return [row['origin_id'] for row in g.cursor.fetchall()]
 
 
+def get_id_from_origin_id(project_id: int, origin_id: str) -> list[str]:
+    g.cursor.execute(
+        """
+        SELECT entity_id FROM import.entity
+        WHERE project_id = %(project_id)s AND origin_id = %(ids)s;
+        """,
+        {'project_id': project_id, 'ids': origin_id})
+    return g.cursor.fetchone()
+
+
 def check_duplicates(class_: str, names: list[str]) -> list[str]:
     g.cursor.execute(
         """

openatlas/forms/process.py

@@ -130,9 +130,14 @@ def process_origin(manager: Any) -> None:
     elif manager.origin.class_.name == 'source' \
             and manager.entity.class_.name != 'source_translation':
         manager.add_link('P67', manager.origin, inverse=True)
-    elif manager.origin.class_.name == 'file' \
-            and manager.entity.class_.name != 'creation':
-        manager.add_link('P67', manager.origin, inverse=True)
+    elif manager.origin.class_.name == 'file':
+        if manager.entity.class_.view == 'reference':
+            manager.add_link(
+                'P67',
+                manager.origin,
+                return_link_id=True)
+        elif manager.entity.class_.name != 'creation':
+            manager.add_link('P67', manager.origin, inverse=True)
 
 
 def process_dates(manager: Any) -> dict[str, Any]:

openatlas/models/imports.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from collections import defaultdict
 from typing import Any, Optional
 
@@ -61,6 +62,10 @@ def get_origin_ids(project: Project, origin_ids: list[str]) -> list[str]:
     return db.check_origin_ids(project.id, origin_ids)
 
 
+def get_id_from_origin_id(project: Project, origin_id: str) -> list[str]:
+    return db.get_id_from_origin_id(project.id, origin_id)
+
+
 def check_duplicates(class_: str, names: list[str]) -> list[str]:
     return db.check_duplicates(class_, names)
 
@@ -70,7 +75,7 @@ def check_type_id(type_id: str, class_: str) -> bool:
         return False
     if not g.types[int(type_id)].root:
         return False
-    root_type = g.types[g.types[int(type_id)].root[-1]]
+    root_type = g.types[g.types[int(type_id)].root[0]]
     if class_ not in root_type.classes:
         return False
     if root_type.name in ['Administrative unit', 'Historical place']:
@@ -81,9 +86,9 @@ def check_type_id(type_id: str, class_: str) -> bool:
 def check_single_type_duplicates(type_ids: list[str]) -> list[str]:
     single_types = defaultdict(list)
     for type_id in type_ids:
-        if not g.types[int(type_id)].multiple:
+        if not g.types[g.types[int(type_id)].root[0]].multiple:
             single_types[
-                g.types[g.types[int(type_id)].root[-1]].name].append(type_id)
+                g.types[g.types[int(type_id)].root[0]].name].append(type_id)
     single_type_ids = []
     for value in single_types.values():
         if len(value) > 1:
@@ -109,7 +114,7 @@ def import_data_(project: Project, class_: str, data: list[Any]) -> None:
             insert_alias(entity, row)
         insert_dates(entity, row)
         link_types(entity, row, class_)
-        link_references(entity, row, class_)
+        link_references(entity, row, class_, project)
         if class_ in g.view_class_mapping['place'] \
                 + g.view_class_mapping['artifact']:
             insert_gis(entity, row, project)
@@ -159,16 +164,27 @@ def link_types(entity: Entity, row: dict[str, Any], class_: str) -> None:
                 entity.link('P2', g.types[int(value_type[0])], value_type[1])
 
 
-def link_references(entity: Entity, row: dict[str, Any], class_: str) -> None:
-    if data := row.get('references'):
-        for references in str(data).split():
+def link_references(
+        entity: Entity,
+        row: dict[str, Any],
+        class_: str,
+        project: Project) -> None:
+    if ref_ids := row.get('reference_ids'):
+        for references in clean_reference_pages(str(ref_ids)):
             reference = references.split(';')
             if len(reference) <= 2 and reference[0].isdigit():
                 try:
                     ref_entity = ApiEntity.get_by_id(int(reference[0]))
                 except EntityDoesNotExistError:
                     continue
-                page = reference[1] if len(reference) > 1 else None
+                page = reference[1] or None
+                ref_entity.link('P67', entity, page)
+    if origin_ref_ids := row.get('origin_reference_ids'):
+        for references in clean_reference_pages(str(origin_ref_ids)):
+            reference = references.split(';')
+            if ref_id := get_id_from_origin_id(project, reference[0]):
+                ref_entity = ApiEntity.get_by_id(int(ref_id[0]))
+                page = reference[1] or None
                 ref_entity.link('P67', entity, page)
     match_types = get_match_types()
     systems = list(set(i for i in row if i.startswith('reference_system_')))
@@ -189,14 +205,14 @@ def link_references(entity: Entity, row: dict[str, Any], class_: str) -> None:
 def insert_gis(entity: Entity, row: dict[str, Any], project: Project) -> None:
     location = Entity.insert('object_location', f"Location of {row['name']}")
     entity.link('P53', location)
-    if data := row.get('administrative_unit'):
+    if data := row.get('administrative_unit_id'):
         if ((str(data).isdigit() and int(data) in g.types) and
-                g.types[g.types[int(data)].root[-1]].name in [
+                g.types[g.types[int(data)].root[0]].name in [
                     'Administrative unit']):
             location.link('P89', g.types[int(data)])
-    if data := row.get('historical_place'):
+    if data := row.get('historical_place_id'):
         if ((str(data).isdigit() and int(data) in g.types) and
-                g.types[g.types[int(data)].root[-1]].name in [
+                g.types[g.types[int(data)].root[0]].name in [
                     'Historical place']):
             location.link('P89', g.types[int(data)])
     if coordinates := row.get('wkt'):
@@ -214,3 +230,8 @@ def insert_gis(entity: Entity, row: dict[str, Any], project: Project) -> None:
                     Gis.insert_wkt(entity, location, project, poly)
             else:
                 Gis.insert_wkt(entity, location, project, wkt_)
+
+
+def clean_reference_pages(value: str) -> list[str]:
+    matches =  re.findall(r'([\w\-]*;[^;]*?(?=[\w\-]*;|$))', value)
+    return [match.strip() for match in matches]

openatlas/static/example.csv

@@ -1,4 +1,4 @@
-id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,references,reference_system_wikidata,reference_system_geonames,administrative_unit,historical_place
-place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,117293;IV,Q152419;close_match,2761369;exact_match,87,221630
-place_2,London,,,,,,,,,"POINT (-0.1290 51.5053)",,,,,,,
-place_3,Rom,,,,,,,,,"LINESTRING (12.458533781141528 41.922205268362234, 12.53062334955289 41.917606998887024, 12.52169797441624 41.888476931243254)",,,,,,,
+id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_ids,reference_system_wikidata,reference_system_geonames,administrative_unit_id,historical_place_id,origin_reference_ids
+place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,"117293;IV, 23-45 23235;45 34",Q152419;close_match,2761369;exact_match,87,221630,literature_1;34-45
+place_2,London,,,,,,,,,POINT (-0.1290 51.5053),,,,,,,,
+place_3,Rom,,,,,,,,,"LINESTRING (12.458533781141528 41.922205268362234, 12.53062334955289 41.917606998887024, 12.52169797441624 41.888476931243254)",,,,,,,,

openatlas/static/example_place_hierarchy.csv

@@ -1,4 +1,4 @@
-id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_system_wikidata,reference_system_geonames,administrative_unit,historical_place,openatlas_class,parent_id,openatlas_parent_id
+id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_system_wikidata,reference_system_geonames,administrative_unit_id,historical_place_id,openatlas_class,parent_id,openatlas_parent_id
 graveyard_1,Wiener Zentralfriedhof,Vienna Central Cemetery;Totenstadt,Important cemetery of Vienna,1874-10-30,1874-11-01,Catholic sanctification.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.432684607765054 48.1594203105231,16.427191443702554 48.15363730600759,16.43277043845353 48.144016607097214,16.435087867042398 48.141897520349765,16.43903607871232 48.14138205358886,16.452168174049234 48.1413247791848,16.455944724342203 48.15031607808294,16.441868491432047 48.15438168967352,16.432684607765054 48.1594203105231))",80 184895,128787;-13.56,Q240744;exact_match,3322690;exact_match,87,221630,place,,
 feature_1,Grave 1,,,,,,,,,,,,,,,,feature,graveyard_1,
 strati_1,Filling 1,,,,,,,,,"GEOMETRYCOLLECTION (POINT (10 10), LINESTRING (0 0, 10 10))",,,,,,,stratigraphic_unit,feature_1,

openatlas/static/manual/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 1f19e91a489895326c4c29410c95ac74
+config: 3cd6e480b825a2d6c550f9663f92431f
 tags: 645f666f9bcd5a90fca523b33c5a78b7

openatlas/static/manual/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '8.7.0',
+    VERSION: '8.8.0',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

openatlas/static/manual/admin/api.html

@@ -5,7 +5,7 @@
   <meta charset="utf-8" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-  <title>API &mdash; OpenAtlas 8.7.0 documentation</title>
+  <title>API &mdash; OpenAtlas 8.8.0 documentation</title>
       <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
       <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
       <link rel="stylesheet" href="../_static/style.css" type="text/css" />
@@ -35,7 +35,7 @@
               <img src="../_static/logo.png" class="logo" alt="Logo"/>
           </a>
               <div class="version">
-                8.7.0
+                8.8.0
               </div>
 <div role="search">
   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -113,13 +113,28 @@ <h1>API<a class="headerlink" href="#api" title="Permalink to this heading"></
 <div class="toctree-wrapper compound">
 </div>
 <p>Description: <a class="reference internal" href="../technical/api.html"><span class="doc">API</span></a></p>
-<ul class="simple">
-<li><p><strong>Public</strong> (default=off). If you turn it on the API and linked files with a license can be accessed even without logging in. This might be useful if you want the data open anyway to allow other systems to use it.</p></li>
+<ul>
+<li><p><strong>Public</strong> (default=off)</p>
+<blockquote>
+<div><ul>
+<li><p>If turned off the API and files with a license can still be accessed by</p>
+<blockquote>
+<div><ul class="simple">
+<li><p>A browser used b a logged in user</p></li>
+<li><p>If the IP of the request-computer is on the <strong>IP Whitelist</strong></p></li>
 </ul>
-<p><strong>Note</strong>: The API and files linked to a license can still be requested from:</p>
-<ul class="simple">
-<li><p>a browser, where a user is logged in</p></li>
-<li><p>if the IP of the request-computer is on the <strong>IP Whitelist</strong></p></li>
+</div></blockquote>
+</li>
+<li><dl class="simple">
+<dt>If turned on the API and linked files with a</dt><dd><p>license can be accessed without being logged in. This might be useful
+if you want to allow other systems to use your data without
+restrictions.</p>
+</dd>
+</dl>
+</li>
+</ul>
+</div></blockquote>
+</li>
 </ul>
 </section>