Skip to content

feat: include dataset information in 410 error message for deleted resources #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 8, 2025
Merged

feat: include dataset information in 410 error message for deleted resources #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
169b4c7
feat: add info about dataset_id in the 410 error message when resourc…
bolinocroustibat Oct 1, 2025
ceca811
docs: update changelog
bolinocroustibat Oct 1, 2025
aaa4470
Merge branch 'main' into dataset-id-in-410-error-message
bolinocroustibat Oct 8, 2025
1be2acc
tests: fix tests by modifying the test data CSV file
bolinocroustibat Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
- Return HTTP 410 Gone response for deleted resources [#56](https://github.com/datagouv/api-tabular/pull/56)
- Fix CI due to end-of-lifed ubuntu distro [#62](https://github.com/datagouv/api-tabular/pull/62)
- Fix swagger structure according to guidelines [#60](https://github.com/datagouv/api-tabular/pull/60)
- Include dataset information in 410 error message for deleted resources [#58](https://github.com/datagouv/api-tabular/pull/58)

## 0.2.5 (2025-07-21)

Expand Down
14 changes: 10 additions & 4 deletions api_tabular/query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@


async def get_resource(session: ClientSession, resource_id: str, columns: list) -> dict:
# Always include deleted_at for deletion checking, but don't duplicate it
# Always include deleted_at and dataset_id for deletion checking, but don't duplicate them
if "deleted_at" not in columns:
columns.append("deleted_at")
if "dataset_id" not in columns:
columns.append("dataset_id")
q = f"select={','.join(columns)}&resource_id=eq.{resource_id}&order=created_at.desc"
url = f"{config.PGREST_ENDPOINT}/tables_index?{q}"
async with session.get(url) as res:
Expand All @@ -21,9 +23,13 @@ async def get_resource(session: ClientSession, resource_id: str, columns: list)
raise web.HTTPNotFound()
if record[0].get("deleted_at") is not None:
deleted_at: str = record[0]["deleted_at"]
raise web.HTTPGone(
text=f"Resource {resource_id} has been permanently deleted on {deleted_at} by its producer. Contact the resource producer to get more information."
)
dataset_id: str | None = record[0].get("dataset_id")
message = f"Resource {resource_id} has been permanently deleted on {deleted_at} by its producer."
if dataset_id:
message += f" You can find more information about this resource at https://www.data.gouv.fr/datasets/{dataset_id}"
else:
message += " Contact the resource producer to get more information."
raise web.HTTPGone(text=message)
return record[0]


Expand Down
5 changes: 3 additions & 2 deletions db/initdb/0-init.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ CREATE TABLE IF NOT EXISTS "csvapi".tables_index (
resource_id UUID,
url VARCHAR,
created_at TIMESTAMP DEFAULT NOW(),
deleted_at TIMESTAMPTZ
deleted_at TIMESTAMPTZ,
dataset_id VARCHAR(24)
);

ALTER TABLE tables_index
ALTER COLUMN created_at TYPE TIMESTAMPTZ;

COPY tables_index(id, parsing_table, csv_detective, resource_id, url, created_at, deleted_at) FROM '/tmp/tables_index.csv' DELIMITER ',' CSV HEADER;
COPY tables_index(id, parsing_table, csv_detective, resource_id, url, created_at, deleted_at, dataset_id) FROM '/tmp/tables_index.csv' DELIMITER ',' CSV HEADER;

CREATE TABLE IF NOT EXISTS "csvapi".resources_exceptions (
id serial PRIMARY KEY,
Expand Down
14 changes: 7 additions & 7 deletions db/tables_index.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
id,parsing_table,csv_detective,resource_id,url,created_at,deleted_at
11922,eb7a008177131590c2f1a2ca0,"{""encoding"": ""ASCII"", ""separator"": "","", ""header_row_idx"": 0, ""header"": [""id"", ""score"", ""decompte"", ""is_true"", ""birth"", ""liste""], ""total_lines"": 1000, ""nb_duplicates"": 0, ""heading_columns"": 0, ""trailing_columns"": 0, ""categorical"": [""is_true"", ""liste""], ""columns_fields"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.0}, ""score"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 1.0}, ""decompte"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 0.914}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""columns_labels"": {""id"": {""python_type"": ""string"", ""format"": ""mongo_object_id"", ""score"": 1.0}, ""score"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""decompte"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""is_true"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""birth"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""liste"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""formats"": {""uuid"": [""id""], ""float"": [""score""], ""int"": [""decompte""], ""booleen"": [""is_true""], ""date"": [""birth""], ""json"": [""liste""]}, ""profile"": {""id"": {""tops"": [{""count"": ""8c7a6452-9295-4db2-b692-34104574fded"", ""value"": 1}, {""count"": ""d757b3bd-2f1d-4dbf-9b64-0a8f2c130573"", ""value"": 1}, {""count"": ""000df1fb-d54d-4c26-843b-7f1d91077622"", ""value"": 1}, {""count"": ""328c6162-6da5-4fdf-a660-62725f8d8130"", ""value"": 1}, {""count"": ""734eb7df-87db-41e2-9b8f-d1ac3b0781f5"", ""value"": 1}, {""count"": ""a6a8447b-9414-488f-823e-27e14ef8008a"", ""value"": 1}, {""count"": ""39958910-06dd-4827-abcd-e980dc3b1ad8"", ""value"": 1}, {""count"": ""ad6b6a54-6661-43ee-9563-8ef6b11a7824"", ""value"": 1}, {""count"": ""e0c37343-9370-40a1-b93f-9aeaf9748a84"", ""value"": 1}, {""count"": ""147f1bb6-e68a-41cc-9f10-4399406cb67b"", ""value"": 1}], ""nb_distinct"": 1000, ""nb_missing_values"": 0}, ""score"": {""min"": 0.002, ""max"": 1.0, ""mean"": 0.511048, ""std"": 0.2913067306247958, ""tops"": [{""count"": 0.149, ""value"": 5}, {""count"": 0.772, ""value"": 5}, {""count"": 0.409, ""value"": 5}, {""count"": 0.685, ""value"": 5}, {""count"": 0.779, ""value"": 4}, {""count"": 0.701, ""value"": 4}, {""count"": 0.751, ""value"": 4}, {""count"": 0.667, ""value"": 4}, {""count"": 0.7, ""value"": 4}, {""count"": 0.925, ""value"": 4}], ""nb_distinct"": 624, ""nb_missing_values"": 0}, ""decompte"": {""min"": 1.0, ""max"": 99.0, ""mean"": 48.255, ""std"": 28.172285186911136, ""tops"": [{""count"": 61.0, ""value"": 16}, {""count"": 25.0, ""value"": 16}, {""count"": 29.0, ""value"": 15}, {""count"": 72.0, ""value"": 14}, {""count"": 76.0, ""value"": 14}, {""count"": 30.0, ""value"": 14}, {""count"": 21.0, ""value"": 14}, {""count"": 23.0, ""value"": 14}, {""count"": 17.0, ""value"": 14}, {""count"": 60.0, ""value"": 14}], ""nb_distinct"": 99, ""nb_missing_values"": 0}, ""is_true"": {""tops"": [{""count"": ""True"", ""value"": 504}, {""count"": ""False"", ""value"": 496}], ""nb_distinct"": 2, ""nb_missing_values"": 0}, ""birth"": {""tops"": [{""count"": ""1936-07-04"", ""value"": 2}, {""count"": ""1989-06-25"", ""value"": 2}, {""count"": ""1915-01-07"", ""value"": 2}, {""count"": ""1940-07-03"", ""value"": 2}, {""count"": ""1929-09-18"", ""value"": 2}, {""count"": ""1925-07-14"", ""value"": 2}, {""count"": ""1956-04-18"", ""value"": 2}, {""count"": ""1983-01-13"", ""value"": 2}, {""count"": ""1916-08-01"", ""value"": 2}, {""count"": ""1997-02-05"", ""value"": 2}], ""nb_distinct"": 974, ""nb_missing_values"": 0}, ""liste"": {""tops"": [{""count"": ""[0]"", ""value"": 356}, {""count"": ""[0, 1, 2]"", ""value"": 342}, {""count"": ""[0, 1]"", ""value"": 302}], ""nb_distinct"": 3, ""nb_missing_values"": 0}}}",aaaaaaaa-1111-bbbb-2222-cccccccccccc,https://data.gouv.fr/datasets/example/resources/fake.csv,2023-04-22 00:54:22.043492+02,
11923,a6311c164ebfb165ddc828ded,"{}",dddddddd-3333-eeee-4444-ffffffffffff,https://data.gouv.fr/datasets/example/resources/bad_labels.csv,2024-11-18 00:54:22.043492+02,
11924,s34fff81a3a7292c64a77e5cz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-5555-bbbb-6666-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes.csv,2025-04-28 11:54:22.043492+02,
11925,aa2zoa2zfb243p45azj33ap1o,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",dddddddd-7777-eeee-8888-ffffffffffff,https://data.gouv.fr/datasets/example/resources/aggregation_allowed.csv,2025-07-11 12:05:22.043492+02,
11926,p34zej8pnq446k2ejfz2m3dqz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-9999-bbbb-1010-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes_aggregation_allowed.csv,2025-07-11 13:10:22.043492+02,
11927,deleted_table123456789,"{""total_lines"": 5, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""name"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}}",deadbeef-dead-beef-dead-beefdeadbeef,https://data.gouv.fr/datasets/example/resources/deleted.csv,2023-01-01 00:00:00+00,2023-12-01 00:00:00+00
id,parsing_table,csv_detective,resource_id,url,created_at,deleted_at,dataset_id
11922,eb7a008177131590c2f1a2ca0,"{""encoding"": ""ASCII"", ""separator"": "","", ""header_row_idx"": 0, ""header"": [""id"", ""score"", ""decompte"", ""is_true"", ""birth"", ""liste""], ""total_lines"": 1000, ""nb_duplicates"": 0, ""heading_columns"": 0, ""trailing_columns"": 0, ""categorical"": [""is_true"", ""liste""], ""columns_fields"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.0}, ""score"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 1.0}, ""decompte"": {""python_type"": ""float"", ""format"": ""latitude_wgs"", ""score"": 0.914}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""columns_labels"": {""id"": {""python_type"": ""string"", ""format"": ""mongo_object_id"", ""score"": 1.0}, ""score"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""decompte"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""is_true"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""birth"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}, ""liste"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}, ""formats"": {""uuid"": [""id""], ""float"": [""score""], ""int"": [""decompte""], ""booleen"": [""is_true""], ""date"": [""birth""], ""json"": [""liste""]}, ""profile"": {""id"": {""tops"": [{""count"": ""8c7a6452-9295-4db2-b692-34104574fded"", ""value"": 1}, {""count"": ""d757b3bd-2f1d-4dbf-9b64-0a8f2c130573"", ""value"": 1}, {""count"": ""000df1fb-d54d-4c26-843b-7f1d91077622"", ""value"": 1}, {""count"": ""328c6162-6da5-4fdf-a660-62725f8d8130"", ""value"": 1}, {""count"": ""734eb7df-87db-41e2-9b8f-d1ac3b0781f5"", ""value"": 1}, {""count"": ""a6a8447b-9414-488f-823e-27e14ef8008a"", ""value"": 1}, {""count"": ""39958910-06dd-4827-abcd-e980dc3b1ad8"", ""value"": 1}, {""count"": ""ad6b6a54-6661-43ee-9563-8ef6b11a7824"", ""value"": 1}, {""count"": ""e0c37343-9370-40a1-b93f-9aeaf9748a84"", ""value"": 1}, {""count"": ""147f1bb6-e68a-41cc-9f10-4399406cb67b"", ""value"": 1}], ""nb_distinct"": 1000, ""nb_missing_values"": 0}, ""score"": {""min"": 0.002, ""max"": 1.0, ""mean"": 0.511048, ""std"": 0.2913067306247958, ""tops"": [{""count"": 0.149, ""value"": 5}, {""count"": 0.772, ""value"": 5}, {""count"": 0.409, ""value"": 5}, {""count"": 0.685, ""value"": 5}, {""count"": 0.779, ""value"": 4}, {""count"": 0.701, ""value"": 4}, {""count"": 0.751, ""value"": 4}, {""count"": 0.667, ""value"": 4}, {""count"": 0.7, ""value"": 4}, {""count"": 0.925, ""value"": 4}], ""nb_distinct"": 624, ""nb_missing_values"": 0}, ""decompte"": {""min"": 1.0, ""max"": 99.0, ""mean"": 48.255, ""std"": 28.172285186911136, ""tops"": [{""count"": 61.0, ""value"": 16}, {""count"": 25.0, ""value"": 16}, {""count"": 29.0, ""value"": 15}, {""count"": 72.0, ""value"": 14}, {""count"": 76.0, ""value"": 14}, {""count"": 30.0, ""value"": 14}, {""count"": 21.0, ""value"": 14}, {""count"": 23.0, ""value"": 14}, {""count"": 17.0, ""value"": 14}, {""count"": 60.0, ""value"": 14}], ""nb_distinct"": 99, ""nb_missing_values"": 0}, ""is_true"": {""tops"": [{""count"": ""True"", ""value"": 504}, {""count"": ""False"", ""value"": 496}], ""nb_distinct"": 2, ""nb_missing_values"": 0}, ""birth"": {""tops"": [{""count"": ""1936-07-04"", ""value"": 2}, {""count"": ""1989-06-25"", ""value"": 2}, {""count"": ""1915-01-07"", ""value"": 2}, {""count"": ""1940-07-03"", ""value"": 2}, {""count"": ""1929-09-18"", ""value"": 2}, {""count"": ""1925-07-14"", ""value"": 2}, {""count"": ""1956-04-18"", ""value"": 2}, {""count"": ""1983-01-13"", ""value"": 2}, {""count"": ""1916-08-01"", ""value"": 2}, {""count"": ""1997-02-05"", ""value"": 2}], ""nb_distinct"": 974, ""nb_missing_values"": 0}, ""liste"": {""tops"": [{""count"": ""[0]"", ""value"": 356}, {""count"": ""[0, 1, 2]"", ""value"": 342}, {""count"": ""[0, 1]"", ""value"": 302}], ""nb_distinct"": 3, ""nb_missing_values"": 0}}}",aaaaaaaa-1111-bbbb-2222-cccccccccccc,https://data.gouv.fr/datasets/example/resources/fake.csv,2023-04-22 00:54:22.043492+02,,5d6e8f7c9d1234567890aaaa
11923,a6311c164ebfb165ddc828ded,"{}",dddddddd-3333-eeee-4444-ffffffffffff,https://data.gouv.fr/datasets/example/resources/bad_labels.csv,2024-11-18 00:54:22.043492+02,,5d6e8f7c9d1234567890bbbb
11924,s34fff81a3a7292c64a77e5cz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-5555-bbbb-6666-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes.csv,2025-04-28 11:54:22.043492+02,,5d6e8f7c9d1234567890cccc
11925,aa2zoa2zfb243p45azj33ap1o,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",dddddddd-7777-eeee-8888-ffffffffffff,https://data.gouv.fr/datasets/example/resources/aggregation_allowed.csv,2025-07-11 12:05:22.043492+02,,5d6e8f7c9d1234567890dddd
11926,p34zej8pnq446k2ejfz2m3dqz,"{""total_lines"": 10, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""score"": {""python_type"": ""float"", ""format"": ""float"", ""score"": 1.0}, ""decompte"": {""python_type"": ""int"", ""format"": ""int"", ""score"": 1.0}, ""is_true"": {""python_type"": ""bool"", ""format"": ""booleen"", ""score"": 1.0}, ""birth"": {""python_type"": ""date"", ""format"": ""date"", ""score"": 1.0}, ""liste"": {""python_type"": ""json"", ""format"": ""json"", ""score"": 1.0}}}",aaaaaaaa-9999-bbbb-1010-cccccccccccc,https://data.gouv.fr/datasets/example/resources/with_indexes_aggregation_allowed.csv,2025-07-11 13:10:22.043492+02,,5d6e8f7c9d1234567890eeee
11927,deleted_table123456789,"{""total_lines"": 5, ""columns"": {""id"": {""python_type"": ""string"", ""format"": ""uuid"", ""score"": 1.5}, ""name"": {""python_type"": ""string"", ""format"": ""string"", ""score"": 1.0}}}",deadbeef-dead-beef-dead-beefdeadbeef,https://data.gouv.fr/datasets/example/resources/deleted.csv,2023-01-01 00:00:00+00,2023-12-01 00:00:00+00,5d6e8f7c9d1234567890ffff