From cd898a59859355b6ef1a7de564f8aafff67ba1b1 Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Mon, 5 Jun 2023 12:20:11 +0200 Subject: [PATCH 1/7] feat(db): add indexes for atlas view --- .../migrations/26d6515219fe_add_indexes.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 backend/gn_module_zh/migrations/26d6515219fe_add_indexes.py diff --git a/backend/gn_module_zh/migrations/26d6515219fe_add_indexes.py b/backend/gn_module_zh/migrations/26d6515219fe_add_indexes.py new file mode 100644 index 00000000..a3728a02 --- /dev/null +++ b/backend/gn_module_zh/migrations/26d6515219fe_add_indexes.py @@ -0,0 +1,103 @@ +"""add indexes + +Revision ID: 26d6515219fe +Revises: 22b14fc3abe0 +Create Date: 2023-06-05 12:07:39.416188 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "26d6515219fe" +down_revision = "22b14fc3abe0" +branch_labels = None +depends_on = None + +SCHEMA = "pr_zh" + + +def upgrade(): + op.create_index("index_t_zh_id_sdage", table_name="t_zh", columns=["id_sdage"], schema=SCHEMA) + op.create_index( + "index_t_zh_id_thread", table_name="t_zh", columns=["id_thread"], schema=SCHEMA + ) + op.create_index( + "index_t_zh_id_diag_bio", table_name="t_zh", columns=["id_diag_bio"], schema=SCHEMA + ) + op.create_index( + "index_t_zh_id_diag_hydro", table_name="t_zh", columns=["id_diag_hydro"], schema=SCHEMA + ) + op.create_index( + "index_t_zh_id_lim_list", table_name="t_zh", columns=["id_lim_list"], schema=SCHEMA + ) + op.create_index("index_t_zh_id_zh_uuid", table_name="t_zh", columns=["zh_uuid"], schema=SCHEMA) + op.create_index( + "index_cor_zh_rb_id_zh", table_name="cor_zh_rb", columns=["id_zh"], schema=SCHEMA + ) + op.create_index( + "index_cor_lim_list_id_lim_list", + table_name="cor_lim_list", + columns=["id_lim_list"], + schema=SCHEMA, + ) + op.create_index( + "index_cor_zh_area_id_zh", table_name="cor_zh_area", columns=["id_zh"], schema=SCHEMA + ) + op.create_index( + "index_cor_zh_area_id_area", table_name="cor_zh_area", columns=["id_area"], schema=SCHEMA + ) + + +def downgrade(): + op.drop_index( + "index_t_zh_id_sdage", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_t_zh_id_thread", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_t_zh_id_diag_bio", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_t_zh_id_diag_hydro", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_t_zh_id_lim_list", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_t_zh_id_zh_uuid", + table_name="t_zh", + schema=SCHEMA, + ) + op.drop_index( + "index_cor_zh_rb_id_zh", + table_name="cor_zh_rb", + schema=SCHEMA, + ) + op.drop_index( + "index_cor_lim_list_id_lim_list", + table_name="cor_lim_list", + schema=SCHEMA, + ) + op.drop_index( + "index_cor_zh_area_id_zh", + table_name="cor_zh_area", + schema=SCHEMA, + ) + op.drop_index( + "index_cor_zh_area_id_area", + table_name="cor_zh_area", + schema=SCHEMA, + ) From cd05ca537efa174a13e3df236ab605f097ee0e60 Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Mon, 5 Jun 2023 13:14:19 +0200 Subject: [PATCH 2/7] feat(db): add atlas view --- .../migrations/643743e807f6_add_atlas_view.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py diff --git a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py new file mode 100644 index 00000000..baedfd6e --- /dev/null +++ b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py @@ -0,0 +1,94 @@ +"""add atlas view + +Revision ID: 643743e807f6 +Revises: 26d6515219fe +Create Date: 2023-06-05 12:20:36.897280 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "643743e807f6" +down_revision = "26d6515219fe" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + CREATE OR REPLACE FUNCTION pr_zh.slugify("value" TEXT) + RETURNS TEXT AS $$ + -- removes accents (diacritic signs) from a given string -- + WITH "unaccented" AS ( + SELECT unaccent("value") AS "value" + ), + -- lowercases the string + "lowercase" AS ( + SELECT lower("value") AS "value" + FROM "unaccented" + ), + -- replaces anything that's not a letter, number, hyphen('-'), or underscore('_') with a hyphen('-') + "hyphenated" AS ( + SELECT regexp_replace("value", '[^a-z0-9\\-_]+', '-', 'gi') AS "value" + FROM "lowercase" + ), + -- trims hyphens('-') if they exist on the head or tail of the string + "trimmed" AS ( + SELECT regexp_replace(regexp_replace("value", '\\-+$', ''), '^\\-', '') AS "value" + FROM "hyphenated" + ) + SELECT "value" FROM "trimmed"; + $$ LANGUAGE SQL STRICT IMMUTABLE; + """ + ) + op.execute( + """ + CREATE OR REPLACE VIEW pr_zh.atlas_app + AS SELECT tzh.id_zh AS id, + tzh.main_name AS nom, + ( SELECT pr_zh.slugify(tzh.main_name::text) AS slugify) AS slug, + tzh.code, + tzh.create_date AS date, + tzh.geom AS polygon_4326, + ( SELECT st_area(st_transform(st_setsrid(tzh.geom, 4326), 2154)) AS st_area) AS superficie, + bo.nom_organisme AS operateur, + sdage.cd_nomenclature AS type_code, + sdage.mnemonique AS type, + thread.mnemonique AS menaces, + diag_bio.mnemonique AS diagnostic_bio, + diag_hydro.mnemonique AS diagnostic_hydro, + ( SELECT array_agg(DISTINCT tn.mnemonique) AS array_agg) AS criteres_delim, + ( SELECT array_agg(DISTINCT la.area_name) AS array_agg) AS communes, + ( SELECT array_agg(DISTINCT trb.name) AS array_agg) AS bassin_versant, + ( SELECT COALESCE(json_agg(t.*), '[]'::json) AS "coalesce" + FROM ( SELECT t_medias.title_fr AS label, + t_medias.media_path AS url + FROM gn_commons.t_medias + WHERE t_medias.uuid_attached_row = tzh.zh_uuid) t) AS images + FROM pr_zh.t_zh tzh + LEFT JOIN pr_zh.cor_lim_list cll ON tzh.id_lim_list = cll.id_lim_list + LEFT JOIN ref_nomenclatures.t_nomenclatures tn ON cll.id_lim = tn.id_nomenclature + LEFT JOIN pr_zh.cor_zh_area cza ON tzh.id_zh = cza.id_zh + LEFT JOIN ref_geo.l_areas la ON cza.id_area = la.id_area + LEFT JOIN pr_zh.cor_zh_rb czr ON tzh.id_zh = czr.id_zh + LEFT JOIN pr_zh.t_river_basin trb ON czr.id_rb = trb.id_rb + LEFT JOIN gn_commons.t_medias med ON med.uuid_attached_row = tzh.zh_uuid + LEFT JOIN utilisateurs.t_roles tr ON tr.id_role = tzh.create_author + LEFT JOIN utilisateurs.bib_organismes bo ON bo.id_organisme = tr.id_organisme + LEFT JOIN ref_nomenclatures.t_nomenclatures sdage ON sdage.id_nomenclature = tzh.id_sdage + LEFT JOIN ref_nomenclatures.t_nomenclatures thread ON thread.id_nomenclature = tzh.id_thread + LEFT JOIN ref_nomenclatures.t_nomenclatures diag_bio ON diag_bio.id_nomenclature = tzh.id_diag_bio + LEFT JOIN ref_nomenclatures.t_nomenclatures diag_hydro ON diag_hydro.id_nomenclature = tzh.id_diag_hydro + WHERE cza.cover IS NOT NULL + GROUP BY tzh.id_zh, bo.nom_organisme, sdage.cd_nomenclature, sdage.mnemonique, thread.mnemonique, diag_bio.mnemonique, diag_hydro.mnemonique + ORDER BY tzh.id_zh; + """ + ) + + +def downgrade(): + op.execute("DROP VIEW pr_zh.atlas_app") + op.execute("DROP FUNCTION pr_zh.slugify") From 0eff9c1b1dc7a52a3c32bd230c19941b7788ca17 Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Mon, 5 Jun 2023 13:33:26 +0200 Subject: [PATCH 3/7] feat(db): add drop old materialized view --- backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py index baedfd6e..7f3355c3 100644 --- a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py +++ b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py @@ -44,6 +44,7 @@ def upgrade(): $$ LANGUAGE SQL STRICT IMMUTABLE; """ ) + op.execute("DROP MATERIALIZED VIEW IF EXISTS pr_zh.atlas_app") op.execute( """ CREATE OR REPLACE VIEW pr_zh.atlas_app From 6d390510bdf47e88b0c34ef199222d4526f250e4 Mon Sep 17 00:00:00 2001 From: Camille Monchicourt Date: Tue, 6 Jun 2023 09:58:37 +0200 Subject: [PATCH 4/7] Changelog 1.1.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aff8177..e15f9f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.1.1 (2023-06-06) + +- Remplacement de la vue matérialisée `pr_zh.atlas_app` par une vue (utilisée par la route `/api/zones_humides/pbf/complete`) pour corriger et simplifier la mise à jour des données de l'[atlas des zones humides](https://github.com/PnX-SI/GeoNature-ZH-atlas) (#24) + ## 1.1.0 - Taillefer (2023-06-02) Nécessite la version 2.12.0 (ou plus) de GeoNature. From 70a8af29417c5c498a5f19ce5c71d35587a8db33 Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Tue, 6 Jun 2023 14:04:28 +0200 Subject: [PATCH 5/7] fix(backend): return null if no content in pbf --- backend/gn_module_zh/blueprint.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/gn_module_zh/blueprint.py b/backend/gn_module_zh/blueprint.py index ea2417de..7d109ad3 100644 --- a/backend/gn_module_zh/blueprint.py +++ b/backend/gn_module_zh/blueprint.py @@ -366,8 +366,7 @@ def get_pbf_complete(): """ query = DB.session.execute(sql) row = query.first() - if row["pbf"]: - return Response(bytes(row["pbf"]), mimetype="application/protobuf") + return Response(bytes(row["pbf"]) if row["pbf"] else bytes(), mimetype="application/protobuf") @blueprint.route("/geojson", methods=["GET"]) From 0a9e848a252c827790897b9050344a26790b4d5f Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Tue, 6 Jun 2023 14:04:56 +0200 Subject: [PATCH 6/7] chore(db): add comment to explain revision --- .../gn_module_zh/migrations/643743e807f6_add_atlas_view.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py index 7f3355c3..53c47c38 100644 --- a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py +++ b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py @@ -15,8 +15,13 @@ branch_labels = None depends_on = None +# This revision performs 2 things: +# - replaces the materialized view by a view +# - treats better the downgrade part by removing the slugify function +# used only for the view (thus this function need to be re-created) def upgrade(): + # Recreate function here if it is dropped by the op.execute( """ CREATE OR REPLACE FUNCTION pr_zh.slugify("value" TEXT) From 7ce8ce7503a1f8f550ba3d7752c271d1c15e6565 Mon Sep 17 00:00:00 2001 From: Maxime Vergez Date: Tue, 6 Jun 2023 14:07:58 +0200 Subject: [PATCH 7/7] build: update version & black --- VERSION | 2 +- backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 1cc5f657..8cfbc905 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.0 \ No newline at end of file +1.1.1 \ No newline at end of file diff --git a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py index 53c47c38..e5f3555c 100644 --- a/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py +++ b/backend/gn_module_zh/migrations/643743e807f6_add_atlas_view.py @@ -20,8 +20,9 @@ # - treats better the downgrade part by removing the slugify function # used only for the view (thus this function need to be re-created) + def upgrade(): - # Recreate function here if it is dropped by the + # Recreate function here if it is dropped by the op.execute( """ CREATE OR REPLACE FUNCTION pr_zh.slugify("value" TEXT)