Improve CI workflows, add autoformatters / linters

.github/workflows/tox-pytest.yml

@@ -62,7 +62,7 @@ jobs:
         env:
           API_KEY_EIA: ${{ secrets.API_KEY_EIA }}
         run: |
-          conda run -n pudl-rmi tox
+          conda run -n pudl-rmi tox -- --five-year-coverage
 
       - name: Log post-test PUDL workspace contents
         run: find ~/pudl-work/

.pre-commit-config.yaml

@@ -6,10 +6,14 @@ repos:
       - id: python-check-blanket-noqa # Prohibit overly broad QA exclusions.
       - id: python-no-eval # Never use eval() it's dangerous.
       - id: python-no-log-warn # logger.warning(), not old .warn()
+      - id: rst-backticks # Find single rather than double backticks
+      - id: rst-directive-colons # Missing double-colons after directives
+      - id: rst-inline-touching-normal # Inline code should never touch normal text
+      - id: python-use-type-annotations # Use annotations not old-style type comments
 
   # Other file formatting, plus common Git mistakes & text file standardization:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
       - id: check-added-large-files # Don't accidentally commit giant files.
       - id: check-merge-conflict # Watch for lingering merge markers.
@@ -21,19 +25,44 @@ repos:
       - id: trailing-whitespace # Remove trailing whitespace.
       - id: name-tests-test # Follow PyTest naming convention.
 
-  # Format the code
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  ########################################################################################
+  # Formatters: hooks that re-write Python and RST files
+  ########################################################################################
+  # Convert relative imports to absolute imports
+  - repo: https://github.com/MarcoGorelli/absolufy-imports
+    rev: v0.3.1
     hooks:
-      - id: black
-        language_version: python3.10
+      - id: absolufy-imports
 
   # Make sure import statements are sorted uniformly.
   - repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
       - id: isort
 
+  # Remove f-string prefix when there's nothing in the string to format.
+  - repo: https://github.com/dannysepler/rm_unneeded_f_str
+    rev: v0.1.0
+    hooks:
+      - id: rm-unneeded-f-str
+
+  # Deterministic python formatting:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+        language_version: python3.10
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.7.1
+    hooks:
+      - id: prettier
+        types_or: [yaml]
+
+  ########################################################################################
+  # Linters: hooks that check but don't alter Python & RST files
+  ########################################################################################
+
   # Check for PEP8 non-compliance, code complexity, style, errors, etc:
   - repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
@@ -52,6 +81,21 @@ repos:
           - flake8-rst-docstrings
           - flake8-use-fstring
 
+  # Check for errors in restructuredtext (.rst) files under the doc hierarchy
+  - repo: https://github.com/PyCQA/doc8
+    rev: 0.11.2
+    hooks:
+      - id: doc8
+        args: ["--config", "tox.ini"]
+
+  # Lint any RST files and embedded code blocks for syntax / formatting errors
+  - repo: https://github.com/myint/rstcheck
+    rev: v6.0.0.post1
+    hooks:
+      - id: rstcheck
+        additional_dependencies: [sphinx]
+        args: ["--config", "tox.ini"]
+
   # Our own pre-commit hooks, which don't come from the pre-commit project
   - repo: local
     hooks:
@@ -64,7 +108,9 @@ repos:
         always_run: true
         entry: pytest --doctest-modules src/pudl_rmi test/unit
 
+########################################################################################
 # Configuration for pre-commit.ci
+########################################################################################
 ci:
   autofix_commit_msg: |
     [pre-commit.ci] auto fixes from pre-commit.com hooks

inputs/expected_errors/expected_errors_capex_total_plants_ferc1_to_eia_vs_deprish_to_ferc1_five_year.csv

@@ -0,0 +1,91 @@
+report_year,data_source,utility_id_pudl,plant_id_eia
+2015,FERC,293,3298
+2015,PUC,18,8
+2015,PUC,18,10
+2015,PUC,31,113
+2015,PUC,31,116
+2015,PUC,31,117
+2015,PUC,31,118
+2015,PUC,89,1733
+2015,PUC,89,1740
+2015,PUC,89,1743
+2015,PUC,89,6034
+2015,PUC,301,51
+2015,PUC,318,1239
+2015,PUC,318,6065
+2015,PUC,318,7296
+2015,PUC,318,56456
+2016,FERC,91,638
+2016,FERC,91,8049
+2016,FERC,103,3456
+2016,FERC,246,3648
+2016,FERC,365,4072
+2016,FERC,365,4078
+2016,PUC,97,3250
+2016,PUC,97,6014
+2016,PUC,130,643
+2016,PUC,144,6166
+2016,PUC,349,3809
+2017,FERC,169,1363
+2017,FERC,243,6098
+2017,FERC,330,2451
+2017,FERC,330,4941
+2017,FERC,330,55343
+2017,PUC,29,3775
+2017,PUC,123,702
+2017,PUC,123,6052
+2017,PUC,123,6124
+2017,PUC,161,6068
+2017,PUC,178,2176
+2017,PUC,178,6089
+2018,FERC,29,3935
+2018,FERC,89,6034
+2018,FERC,97,6014
+2018,FERC,97,58697
+2018,FERC,130,6073
+2018,FERC,144,6166
+2018,FERC,150,1073
+2018,FERC,150,6254
+2018,FERC,162,3948
+2018,FERC,263,3048
+2018,FERC,263,6106
+2018,FERC,274,2451
+2018,FERC,275,2963
+2018,FERC,301,6138
+2018,FERC,301,7902
+2018,FERC,359,1248
+2018,FERC,364,4050
+2018,PUC,31,116
+2018,PUC,31,117
+2018,PUC,31,118
+2018,PUC,31,2442
+2018,PUC,31,55455
+2018,PUC,92,55110
+2018,PUC,97,3250
+2018,PUC,97,6014
+2018,PUC,121,612
+2018,PUC,121,6043
+2018,PUC,171,8023
+2018,PUC,190,2049
+2018,PUC,272,470
+2018,PUC,272,525
+2018,PUC,334,2104
+2019,FERC,7,6166
+2019,FERC,89,1733
+2019,FERC,89,1743
+2019,FERC,89,6034
+2019,FERC,109,6641
+2019,FERC,150,1073
+2019,FERC,150,6254
+2019,FERC,160,1241
+2019,FERC,243,6098
+2019,FERC,274,2442
+2019,FERC,274,2451
+2019,FERC,274,55343
+2019,FERC,361,3948
+2019,FERC,364,4050
+2019,PUC,224,1904
+2019,PUC,224,6090
+2019,PUC,224,7237
+2020,PUC,97,6014
+2020,PUC,246,6101

inputs/expected_errors/expected_errors_capex_total_utilities_ferc1_to_eia_vs_deprish_to_ferc1_five_year.csv

@@ -0,0 +1,96 @@
+report_year,data_source,utility_id_pudl
+2015,FERC,222
+2015,FERC,292
+2015,FERC,293
+2015,PUC,18
+2015,PUC,31
+2015,PUC,63
+2015,PUC,89
+2015,PUC,140
+2015,PUC,287
+2015,PUC,301
+2015,PUC,318
+2016,FERC,91
+2016,FERC,92
+2016,FERC,103
+2016,FERC,106
+2016,FERC,109
+2016,FERC,113
+2016,FERC,246
+2016,FERC,284
+2016,FERC,313
+2016,FERC,339
+2016,FERC,365
+2016,PUC,97
+2016,PUC,130
+2016,PUC,144
+2016,PUC,162
+2016,PUC,204
+2016,PUC,349
+2017,FERC,103
+2017,FERC,163
+2017,FERC,169
+2017,FERC,243
+2017,FERC,330
+2017,PUC,29
+2017,PUC,79
+2017,PUC,123
+2017,PUC,161
+2017,PUC,178
+2017,PUC,226
+2017,PUC,244
+2017,PUC,275
+2018,FERC,29
+2018,FERC,35
+2018,FERC,42
+2018,FERC,89
+2018,FERC,90
+2018,FERC,97
+2018,FERC,130
+2018,FERC,144
+2018,FERC,146
+2018,FERC,150
+2018,FERC,160
+2018,FERC,162
+2018,FERC,185
+2018,FERC,190
+2018,FERC,263
+2018,FERC,274
+2018,FERC,275
+2018,FERC,298
+2018,FERC,301
+2018,FERC,359
+2018,FERC,364
+2018,PUC,19
+2018,PUC,23
+2018,PUC,31
+2018,PUC,90
+2018,PUC,92
+2018,PUC,97
+2018,PUC,106
+2018,PUC,121
+2018,PUC,171
+2018,PUC,190
+2018,PUC,272
+2018,PUC,295
+2018,PUC,302
+2018,PUC,334
+2019,FERC,7
+2019,FERC,35
+2019,FERC,89
+2019,FERC,109
+2019,FERC,150
+2019,FERC,160
+2019,FERC,243
+2019,FERC,274
+2019,FERC,284
+2019,FERC,359
+2019,FERC,361
+2019,FERC,364
+2019,PUC,222
+2019,PUC,224
+2020,FERC,185
+2020,PUC,90
+2020,PUC,97
+2020,PUC,185
+2020,PUC,246

inputs/expected_errors/expected_errors_plant_balance_w_common_plants_deprish_to_eia_vs_deprish_to_ferc1_five_year.csv

@@ -0,0 +1,22 @@
+report_year,data_source,utility_id_pudl,plant_id_eia
+2015,MODERNIZED,81,1710
+2015,PUC,89,6034
+2016,FERC,365,4078
+2016,Hybrid,365,4078
+2016,MODERNIZED,81,1710
+2016,PUC,144,6166
+2016,PUC - KS,159,1241
+2016,PUC - MO,159,1241
+2017,MODERNIZED,81,1710
+2018,FERC,89,6034
+2018,FERC,144,6166
+2018,FERC,162,3948
+2018,FERC,263,3048
+2018,FERC,263,6106
+2018,MODERNIZED,81,1710
+2018,PUC,90,3264
+2018,PUC,121,207
+2018,PUC,171,8023
+2018,PUC,302,2446
+2019,FERC,89,6034
+2019,FERC,284,55985

inputs/expected_errors/expected_errors_plant_balance_w_common_plants_deprish_vs_deprish_to_eia_five_year.csv

@@ -0,0 +1,45 @@
+report_year,data_source,utility_id_pudl,plant_id_eia
+2015,FERC,222,997
+2015,FERC,222,6085
+2015,PUC,18,2
+2015,PUC,31,116
+2015,PUC,63,96
+2015,PUC,63,6293
+2016,FERC,106,168
+2016,FERC,106,169
+2016,FERC,106,170
+2016,FERC,106,6009
+2016,FERC,106,8055
+2016,FERC,106,55467
+2016,FERC,109,2051
+2016,FERC,109,6641
+2016,FERC,113,1393
+2016,FERC,113,3457
+2016,FERC,113,3459
+2016,FERC,246,3659
+2016,FERC,246,3661
+2016,FERC,313,645
+2016,PUC,130,643
+2016,PUC,144,6166
+2016,PUC,349,6168
+2017,FERC,330,2451
+2017,FERC,330,4941
+2017,FERC,330,8223
+2017,PUC,123,649
+2017,PUC,123,6124
+2017,PUC,178,2176
+2017,PUC,178,6089
+2018,FERC,150,1077
+2018,FERC,160,1240
+2018,FERC,160,1241
+2018,FERC,185,56809
+2018,FERC,275,2965
+2018,FERC,364,7203
+2018,PUC,90,3264
+2018,PUC,97,6014
+2018,PUC,121,612
+2018,PUC,302,2446
+2019,FERC,109,55218
+2019,FERC,284,55985
+2019,PUC,222,6085
+2020,PUC,185,7985