Create CLI to manage, check, and patch data

wblondel · Jan 15, 2021 · 8c4b8d1 · 8c4b8d1
1 parent 08c4895
commit 8c4b8d1
Show file tree

Hide file tree

Showing 25 changed files with 591 additions and 115 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,15 @@
+# Source CSV and ZIP files
+data/source/csv/*
+!data/source/csv/.gitkeep
+data/source/zip/*
+!data/source/zip/.gitkeep
+
+# Corrected CSV and GZ files
+data/corrected/csv/*
+!data/corrected/csv/.gitkeep
+data/corrected/gz/*
+!data/corrected/gz/.gitkeep
+
+tests
+__pycache__
+.idea
diff --git a/Pipfile b/Pipfile
@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+click = "~=7.1"
+more-itertools = "~=8.6"
+regex = "~=2020.11.13"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/data-prep/.gitignore b/data-prep/.gitignore
diff --git a/data-prep/00-1-source-files-download.py b/data-prep/00-1-source-files-download.py
diff --git a/data-prep/00-2-source-files-extract.py b/data-prep/00-2-source-files-extract.py
diff --git a/data-prep/01-1-source-to-corrected.py b/data-prep/01-1-source-to-corrected.py
diff --git a/data-prep/01-2-apply-patches-to-corrected b/data-prep/01-2-apply-patches-to-corrected
diff --git a/data-prep/01-3-corrected-compress.py b/data-prep/01-3-corrected-compress.py
diff --git a/data-prep/01-4-create-patches b/data-prep/01-4-create-patches
diff --git a/data-prep/00-source/csv/.gitkeep → data/corrected/csv/.gitkeep b/data-prep/00-source/csv/.gitkeep → data/corrected/csv/.gitkeep
diff --git a/data-prep/00-source/zip/.gitkeep → data/corrected/gz/.gitkeep b/data-prep/00-source/zip/.gitkeep → data/corrected/gz/.gitkeep
diff --git a/data-prep/01-corrected/csv/.gitkeep → data/patches/.gitkeep b/data-prep/01-corrected/csv/.gitkeep → data/patches/.gitkeep
diff --git a/data-prep/01-corrected/gz/.gitkeep → data/source/csv/.gitkeep b/data-prep/01-corrected/gz/.gitkeep → data/source/csv/.gitkeep
diff --git a/data-prep/patches/.gitkeep → data/source/zip/.gitkeep b/data-prep/patches/.gitkeep → data/source/zip/.gitkeep
diff --git a/...cratch/01-2-apply-patches-to-corrected.py → ...cratch/01-2-apply-patches-to-corrected.py b/...cratch/01-2-apply-patches-to-corrected.py → ...cratch/01-2-apply-patches-to-corrected.py
@@ -1,4 +1,4 @@
-# TOO SLOW
+# REALLY TOO SLOW
 
 from pathlib import Path
 import subprocess

diff --git a/.../.scratch/01-4-create-patches-git-diff.py → .../.scratch/01-4-create-patches-git-diff.py b/.../.scratch/01-4-create-patches-git-diff.py → .../.scratch/01-4-create-patches-git-diff.py
diff --git a/...1-4-create-patches-udiff-difflib-bytes.py → ...1-4-create-patches-udiff-difflib-bytes.py b/...1-4-create-patches-udiff-difflib-bytes.py → ...1-4-create-patches-udiff-difflib-bytes.py
diff --git a/.../01-4-create-patches-udiff-difflib-ut8.py → .../01-4-create-patches-udiff-difflib-ut8.py b/.../01-4-create-patches-udiff-difflib-ut8.py → .../01-4-create-patches-udiff-difflib-ut8.py
diff --git a/insee-deaths-manager/cli.py b/insee-deaths-manager/cli.py
@@ -0,0 +1,17 @@
+import click
+
+from cli_check import cli_check
+from cli_manage import cli_manage
+from cli_patch import cli_patch
+
+
+@click.group()
+def cli():
+    pass
+
+
+if __name__ == '__main__':
+    cli.add_command(cli_check)
+    cli.add_command(cli_manage)
+    cli.add_command(cli_patch)
+    cli()