Skip to content

Commit 6f190bd

Browse files
bolinocroustibatbolinocroustibat
authored
feat: include dataset information in 410 error message for deleted resources (#58)
Related to datagouv/data.gouv.fr#1862 and [this discussion](https://mattermost.incubateur.net/betagouv/pl/z9fdmompbibeznu536s685cc1r). Needs datagouv/hydra#341
1 parent 8fefd1a commit 6f190bd

File tree

4 files changed

+21
-13
lines changed

4 files changed

+21
-13
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
- Return HTTP 410 Gone response for deleted resources [#56](https://github.com/datagouv/api-tabular/pull/56)
66
- Fix CI due to end-of-lifed ubuntu distro [#62](https://github.com/datagouv/api-tabular/pull/62)
77
- Fix swagger structure according to guidelines [#60](https://github.com/datagouv/api-tabular/pull/60)
8+
- Include dataset information in 410 error message for deleted resources [#58](https://github.com/datagouv/api-tabular/pull/58)
89

910
## 0.2.5 (2025-07-21)
1011

‎api_tabular/query.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@
88

99

1010
async def get_resource(session: ClientSession, resource_id: str, columns: list) -> dict:
11-
# Always include deleted_at for deletion checking, but don't duplicate it
11+
# Always include deleted_at and dataset_id for deletion checking, but don't duplicate them
1212
if "deleted_at" not in columns:
1313
columns.append("deleted_at")
14+
if "dataset_id" not in columns:
15+
columns.append("dataset_id")
1416
q = f"select={','.join(columns)}&resource_id=eq.{resource_id}&order=created_at.desc"
1517
url = f"{config.PGREST_ENDPOINT}/tables_index?{q}"
1618
async with session.get(url) as res:
@@ -21,9 +23,13 @@ async def get_resource(session: ClientSession, resource_id: str, columns: list)
2123
raise web.HTTPNotFound()
2224
if record[0].get("deleted_at") is not None:
2325
deleted_at: str = record[0]["deleted_at"]
24-
raise web.HTTPGone(
25-
text=f"Resource {resource_id} has been permanently deleted on {deleted_at} by its producer. Contact the resource producer to get more information."
26-
)
26+
dataset_id: str | None = record[0].get("dataset_id")
27+
message = f"Resource {resource_id} has been permanently deleted on {deleted_at} by its producer."
28+
if dataset_id:
29+
message += f" You can find more information about this resource at https://www.data.gouv.fr/datasets/{dataset_id}"
30+
else:
31+
message += " Contact the resource producer to get more information."
32+
raise web.HTTPGone(text=message)
2733
return record[0]
2834

2935

‎db/initdb/0-init.sql‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,14 @@ CREATE TABLE IF NOT EXISTS "csvapi".tables_index (
77
resource_id UUID,
88
url VARCHAR,
99
created_at TIMESTAMP DEFAULT NOW(),
10-
deleted_at TIMESTAMPTZ
10+
deleted_at TIMESTAMPTZ,
11+
dataset_id VARCHAR(24)
1112
);
1213

1314
ALTER TABLE tables_index
1415
ALTER COLUMN created_at TYPE TIMESTAMPTZ;
1516

16-
COPY tables_index(id, parsing_table, csv_detective, resource_id, url, created_at, deleted_at) FROM '/tmp/tables_index.csv' DELIMITER ',' CSV HEADER;
17+
COPY tables_index(id, parsing_table, csv_detective, resource_id, url, created_at, deleted_at, dataset_id) FROM '/tmp/tables_index.csv' DELIMITER ',' CSV HEADER;
1718

1819
CREATE TABLE IF NOT EXISTS "csvapi".resources_exceptions (
1920
id serial PRIMARY KEY,

‎db/tables_index.csv‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
id,parsing_table,csv_detective,resource_id,url,created_at,deleted_at
2-
11922,eb7a008177131590c2f1a2ca0,"{""encoding"": ""ASCII"", ""separator"": "","", ""header_row_idx"": 0, ""header"": [""id"", ""score"", ""decompte"", ""is_true"", ""birth"", ""liste""], ""total_lines"": 1000, ""nb_duplicates"": 0, ""heading_columns"": 0, ""trailing_columns"": 0, ""categorical"": [""is_true"", ""liste""], ""columns_fields"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.0}, ""score"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 1.0}, ""decompte"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 0.914}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""columns_labels"": {""id"": {""python_type"": ""string"", ""format"": ""mongo_object_id"", ""score"": 1.0}, ""score"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""decompte"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""is_true"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""birth"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""liste"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""formats"": {""uuid"": [""id""], ""float"": [""score""], ""int"": [""decompte""], ""booleen"": [""is_true""], ""date"": [""birth""], ""json"": [""liste""]}, ""profile"": {""id"": {""tops"": [{""count"": ""8c7a6452-9295-4db2-b692-34104574fded"", ""value"": 1}, {""count"": ""d757b3bd-2f1d-4dbf-9b64-0a8f2c130573"", ""value"": 1}, {""count"": ""000df1fb-d54d-4c26-843b-7f1d91077622"", ""value"": 1}, {""count"": ""328c6162-6da5-4fdf-a660-62725f8d8130"", ""value"": 1}, {""count"": ""734eb7df-87db-41e2-9b8f-d1ac3b0781f5"", ""value"": 1}, {""count"": ""a6a8447b-9414-488f-823e-27e14ef8008a"", ""value"": 1}, {""count"": ""39958910-06dd-4827-abcd-e980dc3b1ad8"", ""value"": 1}, {""count"": ""ad6b6a54-6661-43ee-9563-8ef6b11a7824"", ""value"": 1}, {""count"": ""e0c37343-9370-40a1-b93f-9aeaf9748a84"", ""value"": 1}, {""count"": ""147f1bb6-e68a-41cc-9f10-4399406cb67b"", ""value"": 1}], ""nb_distinct"": 1000, ""nb_missing_values"": 0}, ""score"": {""min"": 0.002, ""max"": 1.0, ""mean"": 0.511048, ""std"": 0.2913067306247958, ""tops"": [{""count"": 0.149, ""value"": 5}, {""count"": 0.772, ""value"": 5}, {""count"": 0.409, ""value"": 5}, {""count"": 0.685, ""value"": 5}, {""count"": 0.779, ""value"": 4}, {""count"": 0.701, ""value"": 4}, {""count"": 0.751, ""value"": 4}, {""count"": 0.667, ""value"": 4}, {""count"": 0.7, ""value"": 4}, {""count"": 0.925, ""value"": 4}], ""nb_distinct"": 624, ""nb_missing_values"": 0}, ""decompte"": {""min"": 1.0, ""max"": 99.0, ""mean"": 48.255, ""std"": 28.172285186911136, ""tops"": [{""count"": 61.0, ""value"": 16}, {""count"": 25.0, ""value"": 16}, {""count"": 29.0, ""value"": 15}, {""count"": 72.0, ""value"": 14}, {""count"": 76.0, ""value"": 14}, {""count"": 30.0, ""value"": 14}, {""count"": 21.0, ""value"": 14}, {""count"": 23.0, ""value"": 14}, {""count"": 17.0, ""value"": 14}, {""count"": 60.0, ""value"": 14}], ""nb_distinct"": 99, ""nb_missing_values"": 0}, ""is_true"": {""tops"": [{""count"": ""True"", ""value"": 504}, {""count"": ""False"", ""value"": 496}], ""nb_distinct"": 2, ""nb_missing_values"": 0}, ""birth"": {""tops"": [{""count"": ""1936-07-04"", ""value"": 2}, {""count"": ""1989-06-25"", ""value"": 2}, {""count"": ""1915-01-07"", ""value"": 2}, {""count"": ""1940-07-03"", ""value"": 2}, {""count"": ""1929-09-18"", ""value"": 2}, {""count"": ""1925-07-14"", ""value"": 2}, {""count"": ""1956-04-18"", ""value"": 2}, {""count"": ""1983-01-13"", ""value"": 2}, {""count"": ""1916-08-01"", ""value"": 2}, {""count"": ""1997-02-05"", ""value"": 2}], ""nb_distinct"": 974, ""nb_missing_values"": 0}, ""liste"": {""tops"": [{""count"": ""[0]"", ""value"": 356}, {""count"": ""[0, 1, 2]"", ""value"": 342}, {""count"": ""[0, 1]"", ""value"": 302}], ""nb_distinct"": 3, ""nb_missing_values"": 0}}}",aaaaaaaa-1111-bbbb-2222-cccccccccccc,https://data.gouv.fr/datasets/example/resources/fake.csv,2023-04-22 00:54:22.043492+02,
3-
11923,a6311c164ebfb165ddc828ded,"{}",dddddddd-3333-eeee-4444-ffffffffffff,https://data.gouv.fr/datasets/example/resources/bad_labels.csv,2024-11-18 00:54:22.043492+02,
4-
11924,s34fff81a3a7292c64a77e5cz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-5555-bbbb-6666-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes.csv,2025-04-28 11:54:22.043492+02,
5-
11925,aa2zoa2zfb243p45azj33ap1o,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",dddddddd-7777-eeee-8888-ffffffffffff,https://data.gouv.fr/datasets/example/resources/aggregation_allowed.csv,2025-07-11 12:05:22.043492+02,
6-
11926,p34zej8pnq446k2ejfz2m3dqz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-9999-bbbb-1010-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes_aggregation_allowed.csv,2025-07-11 13:10:22.043492+02,
7-
11927,deleted_table123456789,"{""total_lines"": 5, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""name"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}}",deadbeef-dead-beef-dead-beefdeadbeef,https://data.gouv.fr/datasets/example/resources/deleted.csv,2023-01-01 00:00:00+00,2023-12-01 00:00:00+00
1+
id,parsing_table,csv_detective,resource_id,url,created_at,deleted_at,dataset_id
2+
11922,eb7a008177131590c2f1a2ca0,"{""encoding"": ""ASCII"", ""separator"": "","", ""header_row_idx"": 0, ""header"": [""id"", ""score"", ""decompte"", ""is_true"", ""birth"", ""liste""], ""total_lines"": 1000, ""nb_duplicates"": 0, ""heading_columns"": 0, ""trailing_columns"": 0, ""categorical"": [""is_true"", ""liste""], ""columns_fields"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.0}, ""score"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 1.0}, ""decompte"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 0.914}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""columns_labels"": {""id"": {""python_type"": ""string"", ""format"": ""mongo_object_id"", ""score"": 1.0}, ""score"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""decompte"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""is_true"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""birth"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""liste"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""formats"": {""uuid"": [""id""], ""float"": [""score""], ""int"": [""decompte""], ""booleen"": [""is_true""], ""date"": [""birth""], ""json"": [""liste""]}, ""profile"": {""id"": {""tops"": [{""count"": ""8c7a6452-9295-4db2-b692-34104574fded"", ""value"": 1}, {""count"": ""d757b3bd-2f1d-4dbf-9b64-0a8f2c130573"", ""value"": 1}, {""count"": ""000df1fb-d54d-4c26-843b-7f1d91077622"", ""value"": 1}, {""count"": ""328c6162-6da5-4fdf-a660-62725f8d8130"", ""value"": 1}, {""count"": ""734eb7df-87db-41e2-9b8f-d1ac3b0781f5"", ""value"": 1}, {""count"": ""a6a8447b-9414-488f-823e-27e14ef8008a"", ""value"": 1}, {""count"": ""39958910-06dd-4827-abcd-e980dc3b1ad8"", ""value"": 1}, {""count"": ""ad6b6a54-6661-43ee-9563-8ef6b11a7824"", ""value"": 1}, {""count"": ""e0c37343-9370-40a1-b93f-9aeaf9748a84"", ""value"": 1}, {""count"": ""147f1bb6-e68a-41cc-9f10-4399406cb67b"", ""value"": 1}], ""nb_distinct"": 1000, ""nb_missing_values"": 0}, ""score"": {""min"": 0.002, ""max"": 1.0, ""mean"": 0.511048, ""std"": 0.2913067306247958, ""tops"": [{""count"": 0.149, ""value"": 5}, {""count"": 0.772, ""value"": 5}, {""count"": 0.409, ""value"": 5}, {""count"": 0.685, ""value"": 5}, {""count"": 0.779, ""value"": 4}, {""count"": 0.701, ""value"": 4}, {""count"": 0.751, ""value"": 4}, {""count"": 0.667, ""value"": 4}, {""count"": 0.7, ""value"": 4}, {""count"": 0.925, ""value"": 4}], ""nb_distinct"": 624, ""nb_missing_values"": 0}, ""decompte"": {""min"": 1.0, ""max"": 99.0, ""mean"": 48.255, ""std"": 28.172285186911136, ""tops"": [{""count"": 61.0, ""value"": 16}, {""count"": 25.0, ""value"": 16}, {""count"": 29.0, ""value"": 15}, {""count"": 72.0, ""value"": 14}, {""count"": 76.0, ""value"": 14}, {""count"": 30.0, ""value"": 14}, {""count"": 21.0, ""value"": 14}, {""count"": 23.0, ""value"": 14}, {""count"": 17.0, ""value"": 14}, {""count"": 60.0, ""value"": 14}], ""nb_distinct"": 99, ""nb_missing_values"": 0}, ""is_true"": {""tops"": [{""count"": ""True"", ""value"": 504}, {""count"": ""False"", ""value"": 496}], ""nb_distinct"": 2, ""nb_missing_values"": 0}, ""birth"": {""tops"": [{""count"": ""1936-07-04"", ""value"": 2}, {""count"": ""1989-06-25"", ""value"": 2}, {""count"": ""1915-01-07"", ""value"": 2}, {""count"": ""1940-07-03"", ""value"": 2}, {""count"": ""1929-09-18"", ""value"": 2}, {""count"": ""1925-07-14"", ""value"": 2}, {""count"": ""1956-04-18"", ""value"": 2}, {""count"": ""1983-01-13"", ""value"": 2}, {""count"": ""1916-08-01"", ""value"": 2}, {""count"": ""1997-02-05"", ""value"": 2}], ""nb_distinct"": 974, ""nb_missing_values"": 0}, ""liste"": {""tops"": [{""count"": ""[0]"", ""value"": 356}, {""count"": ""[0, 1, 2]"", ""value"": 342}, {""count"": ""[0, 1]"", ""value"": 302}], ""nb_distinct"": 3, ""nb_missing_values"": 0}}}",aaaaaaaa-1111-bbbb-2222-cccccccccccc,https://data.gouv.fr/datasets/example/resources/fake.csv,2023-04-22 00:54:22.043492+02,,5d6e8f7c9d1234567890aaaa
3+
11923,a6311c164ebfb165ddc828ded,"{}",dddddddd-3333-eeee-4444-ffffffffffff,https://data.gouv.fr/datasets/example/resources/bad_labels.csv,2024-11-18 00:54:22.043492+02,,5d6e8f7c9d1234567890bbbb
4+
11924,s34fff81a3a7292c64a77e5cz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-5555-bbbb-6666-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes.csv,2025-04-28 11:54:22.043492+02,,5d6e8f7c9d1234567890cccc
5+
11925,aa2zoa2zfb243p45azj33ap1o,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",dddddddd-7777-eeee-8888-ffffffffffff,https://data.gouv.fr/datasets/example/resources/aggregation_allowed.csv,2025-07-11 12:05:22.043492+02,,5d6e8f7c9d1234567890dddd
6+
11926,p34zej8pnq446k2ejfz2m3dqz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-9999-bbbb-1010-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes_aggregation_allowed.csv,2025-07-11 13:10:22.043492+02,,5d6e8f7c9d1234567890eeee
7+
11927,deleted_table123456789,"{""total_lines"": 5, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""name"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}}",deadbeef-dead-beef-dead-beefdeadbeef,https://data.gouv.fr/datasets/example/resources/deleted.csv,2023-01-01 00:00:00+00,2023-12-01 00:00:00+00,5d6e8f7c9d1234567890ffff

0 commit comments

Comments
 (0)