diff --git a/config/database_versions.py b/config/database_versions.py index 8a85480d4..3e14ab526 100644 --- a/config/database_versions.py +++ b/config/database_versions.py @@ -1,5 +1,6 @@ # Used for automatic database upgrades and version checks DATABASE_VERSIONS = [ + '8.8.0', '8.6.0', '8.5.0', '8.4.0', diff --git a/config/default.py b/config/default.py index f9f466f14..80f20a62f 100644 --- a/config/default.py +++ b/config/default.py @@ -50,7 +50,7 @@ # e.g. PROXIES = { # 'http': 'http://someurl.org:8080', # 'https': 'http://someurl.org:8080'} -PROXIES = None +PROXIES: dict[str, str] = {} # Table options TABLE_ROWS = {10: '10', 25: '25', 50: '50', 100: '100'} diff --git a/install/3_data_web.sql b/install/3_data_web.sql index 1c34a0c68..b82ce742d 100644 --- a/install/3_data_web.sql +++ b/install/3_data_web.sql @@ -15,7 +15,7 @@ INSERT INTO web.user (username, password, active, email, group_id) VALUES ( (SELECT id FROM web.group WHERE name = 'admin')); INSERT INTO web.settings (name, value) VALUES - ('database_version', '8.6.0'), + ('database_version', '8.8.0'), ('api_public', ''), ('default_language', 'en'), ('table_rows', '25'), diff --git a/install/upgrade/8.8.0.sql b/install/upgrade/8.8.0.sql new file mode 100644 index 000000000..965b0a2a3 --- /dev/null +++ b/install/upgrade/8.8.0.sql @@ -0,0 +1,16 @@ +BEGIN; + +-- Raise database version +UPDATE web.settings SET value = '8.8.0' WHERE name = 'database_version'; + +-- #2357: Fix possible wrong direction for reference links to files +UPDATE model.link +SET (domain_id, range_id) = (range_id, domain_id) +WHERE id IN ( + SELECT l.id + FROM model.link l + JOIN model.entity d ON l.domain_id = d.id AND d.openatlas_class_name = 'file' + JOIN model.entity r ON l.range_id = r.id AND r.openatlas_class_name IN ('bibliography', 'edition', 'external_reference') + WHERE l.property_code = 'P67'); + +END; diff --git a/install/upgrade/upgrade.md b/install/upgrade/upgrade.md index c3eb73f19..aa5bd0cd0 100644 --- a/install/upgrade/upgrade.md +++ b/install/upgrade/upgrade.md @@ -17,6 +17,9 @@ then run the database upgrade script, then restart Apache: sudo python3 install/upgrade/database_upgrade.py sudo service apache2 restart +### 8.7.0 to 8.8.0 +8.8.0.sql is needed but will be taken care of by the database upgrade script. + ### 8.6.x to 8.7.0 No database update is required but an additional Python package is needed: diff --git a/openatlas/database/imports.py b/openatlas/database/imports.py index c7e790820..6c066a07a 100644 --- a/openatlas/database/imports.py +++ b/openatlas/database/imports.py @@ -58,6 +58,16 @@ def check_origin_ids(project_id: int, origin_ids: list[str]) -> list[str]: return [row['origin_id'] for row in g.cursor.fetchall()] +def get_id_from_origin_id(project_id: int, origin_id: str) -> list[str]: + g.cursor.execute( + """ + SELECT entity_id FROM import.entity + WHERE project_id = %(project_id)s AND origin_id = %(ids)s; + """, + {'project_id': project_id, 'ids': origin_id}) + return g.cursor.fetchone() + + def check_duplicates(class_: str, names: list[str]) -> list[str]: g.cursor.execute( """ diff --git a/openatlas/forms/process.py b/openatlas/forms/process.py index 9190d36ee..06121bcfe 100644 --- a/openatlas/forms/process.py +++ b/openatlas/forms/process.py @@ -130,9 +130,14 @@ def process_origin(manager: Any) -> None: elif manager.origin.class_.name == 'source' \ and manager.entity.class_.name != 'source_translation': manager.add_link('P67', manager.origin, inverse=True) - elif manager.origin.class_.name == 'file' \ - and manager.entity.class_.name != 'creation': - manager.add_link('P67', manager.origin, inverse=True) + elif manager.origin.class_.name == 'file': + if manager.entity.class_.view == 'reference': + manager.add_link( + 'P67', + manager.origin, + return_link_id=True) + elif manager.entity.class_.name != 'creation': + manager.add_link('P67', manager.origin, inverse=True) def process_dates(manager: Any) -> dict[str, Any]: diff --git a/openatlas/models/imports.py b/openatlas/models/imports.py index b92dbeaa9..db501a8d3 100644 --- a/openatlas/models/imports.py +++ b/openatlas/models/imports.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from collections import defaultdict from typing import Any, Optional @@ -61,6 +62,10 @@ def get_origin_ids(project: Project, origin_ids: list[str]) -> list[str]: return db.check_origin_ids(project.id, origin_ids) +def get_id_from_origin_id(project: Project, origin_id: str) -> list[str]: + return db.get_id_from_origin_id(project.id, origin_id) + + def check_duplicates(class_: str, names: list[str]) -> list[str]: return db.check_duplicates(class_, names) @@ -70,7 +75,7 @@ def check_type_id(type_id: str, class_: str) -> bool: return False if not g.types[int(type_id)].root: return False - root_type = g.types[g.types[int(type_id)].root[-1]] + root_type = g.types[g.types[int(type_id)].root[0]] if class_ not in root_type.classes: return False if root_type.name in ['Administrative unit', 'Historical place']: @@ -81,9 +86,9 @@ def check_type_id(type_id: str, class_: str) -> bool: def check_single_type_duplicates(type_ids: list[str]) -> list[str]: single_types = defaultdict(list) for type_id in type_ids: - if not g.types[int(type_id)].multiple: + if not g.types[g.types[int(type_id)].root[0]].multiple: single_types[ - g.types[g.types[int(type_id)].root[-1]].name].append(type_id) + g.types[g.types[int(type_id)].root[0]].name].append(type_id) single_type_ids = [] for value in single_types.values(): if len(value) > 1: @@ -109,7 +114,7 @@ def import_data_(project: Project, class_: str, data: list[Any]) -> None: insert_alias(entity, row) insert_dates(entity, row) link_types(entity, row, class_) - link_references(entity, row, class_) + link_references(entity, row, class_, project) if class_ in g.view_class_mapping['place'] \ + g.view_class_mapping['artifact']: insert_gis(entity, row, project) @@ -159,16 +164,27 @@ def link_types(entity: Entity, row: dict[str, Any], class_: str) -> None: entity.link('P2', g.types[int(value_type[0])], value_type[1]) -def link_references(entity: Entity, row: dict[str, Any], class_: str) -> None: - if data := row.get('references'): - for references in str(data).split(): +def link_references( + entity: Entity, + row: dict[str, Any], + class_: str, + project: Project) -> None: + if ref_ids := row.get('reference_ids'): + for references in clean_reference_pages(str(ref_ids)): reference = references.split(';') if len(reference) <= 2 and reference[0].isdigit(): try: ref_entity = ApiEntity.get_by_id(int(reference[0])) except EntityDoesNotExistError: continue - page = reference[1] if len(reference) > 1 else None + page = reference[1] or None + ref_entity.link('P67', entity, page) + if origin_ref_ids := row.get('origin_reference_ids'): + for references in clean_reference_pages(str(origin_ref_ids)): + reference = references.split(';') + if ref_id := get_id_from_origin_id(project, reference[0]): + ref_entity = ApiEntity.get_by_id(int(ref_id[0])) + page = reference[1] or None ref_entity.link('P67', entity, page) match_types = get_match_types() systems = list(set(i for i in row if i.startswith('reference_system_'))) @@ -189,14 +205,14 @@ def link_references(entity: Entity, row: dict[str, Any], class_: str) -> None: def insert_gis(entity: Entity, row: dict[str, Any], project: Project) -> None: location = Entity.insert('object_location', f"Location of {row['name']}") entity.link('P53', location) - if data := row.get('administrative_unit'): + if data := row.get('administrative_unit_id'): if ((str(data).isdigit() and int(data) in g.types) and - g.types[g.types[int(data)].root[-1]].name in [ + g.types[g.types[int(data)].root[0]].name in [ 'Administrative unit']): location.link('P89', g.types[int(data)]) - if data := row.get('historical_place'): + if data := row.get('historical_place_id'): if ((str(data).isdigit() and int(data) in g.types) and - g.types[g.types[int(data)].root[-1]].name in [ + g.types[g.types[int(data)].root[0]].name in [ 'Historical place']): location.link('P89', g.types[int(data)]) if coordinates := row.get('wkt'): @@ -214,3 +230,8 @@ def insert_gis(entity: Entity, row: dict[str, Any], project: Project) -> None: Gis.insert_wkt(entity, location, project, poly) else: Gis.insert_wkt(entity, location, project, wkt_) + + +def clean_reference_pages(value: str) -> list[str]: + matches = re.findall(r'([\w\-]*;[^;]*?(?=[\w\-]*;|$))', value) + return [match.strip() for match in matches] diff --git a/openatlas/static/example.csv b/openatlas/static/example.csv index 8e478fb28..ef4d42a8c 100644 --- a/openatlas/static/example.csv +++ b/openatlas/static/example.csv @@ -1,4 +1,4 @@ -id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,references,reference_system_wikidata,reference_system_geonames,administrative_unit,historical_place -place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,117293;IV,Q152419;close_match,2761369;exact_match,87,221630 -place_2,London,,,,,,,,,"POINT (-0.1290 51.5053)",,,,,,, -place_3,Rom,,,,,,,,,"LINESTRING (12.458533781141528 41.922205268362234, 12.53062334955289 41.917606998887024, 12.52169797441624 41.888476931243254)",,,,,,, +id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_ids,reference_system_wikidata,reference_system_geonames,administrative_unit_id,historical_place_id,origin_reference_ids +place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,"117293;IV, 23-45 23235;45 34",Q152419;close_match,2761369;exact_match,87,221630,literature_1;34-45 +place_2,London,,,,,,,,,POINT (-0.1290 51.5053),,,,,,,, +place_3,Rom,,,,,,,,,"LINESTRING (12.458533781141528 41.922205268362234, 12.53062334955289 41.917606998887024, 12.52169797441624 41.888476931243254)",,,,,,,, diff --git a/openatlas/static/example_place_hierarchy.csv b/openatlas/static/example_place_hierarchy.csv index 3806d6a96..9eb26d304 100644 --- a/openatlas/static/example_place_hierarchy.csv +++ b/openatlas/static/example_place_hierarchy.csv @@ -1,4 +1,4 @@ -id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_system_wikidata,reference_system_geonames,administrative_unit,historical_place,openatlas_class,parent_id,openatlas_parent_id +id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,reference_system_wikidata,reference_system_geonames,administrative_unit_id,historical_place_id,openatlas_class,parent_id,openatlas_parent_id graveyard_1,Wiener Zentralfriedhof,Vienna Central Cemetery;Totenstadt,Important cemetery of Vienna,1874-10-30,1874-11-01,Catholic sanctification.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.432684607765054 48.1594203105231,16.427191443702554 48.15363730600759,16.43277043845353 48.144016607097214,16.435087867042398 48.141897520349765,16.43903607871232 48.14138205358886,16.452168174049234 48.1413247791848,16.455944724342203 48.15031607808294,16.441868491432047 48.15438168967352,16.432684607765054 48.1594203105231))",80 184895,128787;-13.56,Q240744;exact_match,3322690;exact_match,87,221630,place,, feature_1,Grave 1,,,,,,,,,,,,,,,,feature,graveyard_1, strati_1,Filling 1,,,,,,,,,"GEOMETRYCOLLECTION (POINT (10 10), LINESTRING (0 0, 10 10))",,,,,,,stratigraphic_unit,feature_1, diff --git a/openatlas/static/manual/.buildinfo b/openatlas/static/manual/.buildinfo index 57e117942..280be436d 100644 --- a/openatlas/static/manual/.buildinfo +++ b/openatlas/static/manual/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 1f19e91a489895326c4c29410c95ac74 +config: 3cd6e480b825a2d6c550f9663f92431f tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/openatlas/static/manual/.doctrees/admin/api.doctree b/openatlas/static/manual/.doctrees/admin/api.doctree index 8156e26d4..e3fc44d88 100644 Binary files a/openatlas/static/manual/.doctrees/admin/api.doctree and b/openatlas/static/manual/.doctrees/admin/api.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/arche.doctree b/openatlas/static/manual/.doctrees/admin/arche.doctree index 0a8fccbd7..7e6ea5afc 100644 Binary files a/openatlas/static/manual/.doctrees/admin/arche.doctree and b/openatlas/static/manual/.doctrees/admin/arche.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/content.doctree b/openatlas/static/manual/.doctrees/admin/content.doctree index 349db79d4..f1c72cba0 100644 Binary files a/openatlas/static/manual/.doctrees/admin/content.doctree and b/openatlas/static/manual/.doctrees/admin/content.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/data_integrity_checks.doctree b/openatlas/static/manual/.doctrees/admin/data_integrity_checks.doctree index f0883bf59..ca406f80b 100644 Binary files a/openatlas/static/manual/.doctrees/admin/data_integrity_checks.doctree and b/openatlas/static/manual/.doctrees/admin/data_integrity_checks.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/execute_sql.doctree b/openatlas/static/manual/.doctrees/admin/execute_sql.doctree index 6f1496066..602f65ace 100644 Binary files a/openatlas/static/manual/.doctrees/admin/execute_sql.doctree and b/openatlas/static/manual/.doctrees/admin/execute_sql.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/export.doctree b/openatlas/static/manual/.doctrees/admin/export.doctree index af42300c2..fc4753057 100644 Binary files a/openatlas/static/manual/.doctrees/admin/export.doctree and b/openatlas/static/manual/.doctrees/admin/export.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/frontend.doctree b/openatlas/static/manual/.doctrees/admin/frontend.doctree index 4864d9c37..30e0fc7b7 100644 Binary files a/openatlas/static/manual/.doctrees/admin/frontend.doctree and b/openatlas/static/manual/.doctrees/admin/frontend.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/general.doctree b/openatlas/static/manual/.doctrees/admin/general.doctree index 93b3ebd0c..f3a4222d2 100644 Binary files a/openatlas/static/manual/.doctrees/admin/general.doctree and b/openatlas/static/manual/.doctrees/admin/general.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/iiif.doctree b/openatlas/static/manual/.doctrees/admin/iiif.doctree index 941d7e3f8..317d034fb 100644 Binary files a/openatlas/static/manual/.doctrees/admin/iiif.doctree and b/openatlas/static/manual/.doctrees/admin/iiif.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/import.doctree b/openatlas/static/manual/.doctrees/admin/import.doctree index 27b91742b..62f83560e 100644 Binary files a/openatlas/static/manual/.doctrees/admin/import.doctree and b/openatlas/static/manual/.doctrees/admin/import.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/mail.doctree b/openatlas/static/manual/.doctrees/admin/mail.doctree index d8728d023..d9dc0dd0e 100644 Binary files a/openatlas/static/manual/.doctrees/admin/mail.doctree and b/openatlas/static/manual/.doctrees/admin/mail.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/map.doctree b/openatlas/static/manual/.doctrees/admin/map.doctree index 3b51ef2e0..1292c1911 100644 Binary files a/openatlas/static/manual/.doctrees/admin/map.doctree and b/openatlas/static/manual/.doctrees/admin/map.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/modules.doctree b/openatlas/static/manual/.doctrees/admin/modules.doctree index 080b9e65c..1f069a619 100644 Binary files a/openatlas/static/manual/.doctrees/admin/modules.doctree and b/openatlas/static/manual/.doctrees/admin/modules.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/user.doctree b/openatlas/static/manual/.doctrees/admin/user.doctree index a350fe32b..b720d8c6b 100644 Binary files a/openatlas/static/manual/.doctrees/admin/user.doctree and b/openatlas/static/manual/.doctrees/admin/user.doctree differ diff --git a/openatlas/static/manual/.doctrees/admin/vocabs.doctree b/openatlas/static/manual/.doctrees/admin/vocabs.doctree index ea6795d16..96b020a74 100644 Binary files a/openatlas/static/manual/.doctrees/admin/vocabs.doctree and b/openatlas/static/manual/.doctrees/admin/vocabs.doctree differ diff --git a/openatlas/static/manual/.doctrees/environment.pickle b/openatlas/static/manual/.doctrees/environment.pickle index f64a9b312..c05f32488 100644 Binary files a/openatlas/static/manual/.doctrees/environment.pickle and b/openatlas/static/manual/.doctrees/environment.pickle differ diff --git a/openatlas/static/manual/.doctrees/technical/api.doctree b/openatlas/static/manual/.doctrees/technical/api.doctree index 496ab1ff7..de5a8f75d 100644 Binary files a/openatlas/static/manual/.doctrees/technical/api.doctree and b/openatlas/static/manual/.doctrees/technical/api.doctree differ diff --git a/openatlas/static/manual/_static/documentation_options.js b/openatlas/static/manual/_static/documentation_options.js index 37201a40f..732ad76f8 100644 --- a/openatlas/static/manual/_static/documentation_options.js +++ b/openatlas/static/manual/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '8.7.0', + VERSION: '8.8.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/openatlas/static/manual/admin/api.html b/openatlas/static/manual/admin/api.html index bee02eba4..10ae5fa36 100644 --- a/openatlas/static/manual/admin/api.html +++ b/openatlas/static/manual/admin/api.html @@ -5,7 +5,7 @@ - API — OpenAtlas 8.7.0 documentation + API — OpenAtlas 8.8.0 documentation @@ -35,7 +35,7 @@
- 8.7.0 + 8.8.0
@@ -113,13 +113,28 @@

API

Description: API

-