Merge pull request #35 from regulatorystudies/test_update_sig_rules

Test update sig rules

data/cumulative_es_rules/update_cumulative_es_rules.py

@@ -15,7 +15,7 @@
             'Obama':[2009,2017],
             'Trump':[2017,2021],
             'Biden':[2021,]}
-print(f"The current dataset covers the {list(admin_year.keys())} administrations.\n"
+print(f"The current dataset covers the {", ".join(list(admin_year.keys()))} administrations.\n"
       f"If there is a new administration, revise the admin_year dictionary and re-run the code.")
 
 #%% Import the current dataset
@@ -46,7 +46,7 @@ def update_admin(admin,update_start_date,update_end_date):
     df_fr = pd.concat([df_fr, pd.DataFrame(data={'publication_date': ['1/21/2021']})], ignore_index=True)
 
     # Change data type
-    df_fr['publication_date'] = pd.to_datetime(df_fr['publication_date']).dt.date
+    df_fr['publication_date'] = pd.to_datetime(df_fr['publication_date'], format="mixed").dt.date
     df_fr['publication_year'] = pd.to_datetime(df_fr['publication_date']).dt.year
     df_fr['publication_month'] = pd.to_datetime(df_fr['publication_date']).dt.month
     df_fr['econ_significant'] = pd.to_numeric(df_fr['econ_significant'], errors='coerce')

data/es_rules/update_es_rules.py

@@ -3,7 +3,7 @@
 import pandas as pd
 import os
 import sys
-from datetime import datetime
+from datetime import date
 
 #%% Import the current dataset
 dir_path=os.path.dirname(os.path.realpath(__file__))
@@ -20,7 +20,7 @@
 last_year_with_data=df['Presidential Year (February 1 - January 31)'].iloc[-1]
 
 # Years to be updated
-current_year=datetime.now().year
+current_year = date.today().year
 
 if last_year_with_data<current_year-1:
     first_year_to_update=last_year_with_data+1
@@ -33,15 +33,15 @@
 #%% FR tracking data
 df_fr = pd.read_csv(f'{dir_path}/../fr_tracking/fr_tracking.csv', encoding="ISO-8859-1")
 
-df_fr['publication_date']=df_fr['publication_date'].astype('datetime64[ns]')
+df_fr['publication_date'] = pd.to_datetime(df_fr['publication_date'], format="mixed").dt.date
 df_fr['econ_significant'] = pd.to_numeric(df_fr['econ_significant'], errors='coerce')
 df_fr['3(f)(1) significant']=pd.to_numeric(df_fr['3(f)(1) significant'], errors='coerce')
 
 #%% Get user input pn presidential party
 def input_party(year):
     party_option = ['democratic','d','republican','r']
     while True:
-        party=input(f'Is presidential year {year} Democratic (D) or Republican (R)?').lower()
+        party=input(f'Is presidential year {year} Democratic (d) or Republican (r)? >>> ').lower()
         if party in party_option:
             output='Democratic' if (party in ['democratic','d']) else 'Republican'
             return output
@@ -53,22 +53,22 @@ def input_party(year):
 update_data=[]
 for year in range(first_year_to_update,last_year_to_update+1):
     if year==2023:
-        count1=df_fr[(df_fr['publication_date']>=datetime(year,2,1)) & \
-                     (df_fr['publication_date']<datetime(year,4,6))]['econ_significant'].sum()
-        count2=df_fr[(df_fr['publication_date']>=datetime(year,4,6)) & \
-                     (df_fr['publication_date']<=datetime(year+1,1,31))]['3(f)(1) significant'].sum()
+        count1=df_fr[(df_fr['publication_date'] >= date(year,2,1)) & \
+                     (df_fr['publication_date'] < date(year,4,6))]['econ_significant'].sum()
+        count2=df_fr[(df_fr['publication_date'] >= date(year,4,6)) & \
+                     (df_fr['publication_date'] <= date(year+1,1,31))]['3(f)(1) significant'].sum()
         count=count1+count2
     else:
         col='econ_significant' if year<2023 else '3(f)(1) significant'
-        count=df_fr[(df_fr['publication_date']>=datetime(year,2,1)) & \
-                     (df_fr['publication_date']<=datetime(year+1,1,31))][col].sum()
+        count=df_fr[(df_fr['publication_date']>=date(year,2,1)) & \
+                     (df_fr['publication_date']<=date(year+1,1,31))][col].sum()
 
     party=input_party(year)
     update_data.append([year,party,count])
     print(f'Presidential year {year} has been updated.')
 
 #%% Append new data
-df=pd.concat([df, pd.DataFrame(update_data, columns=df.columns[0:3])], ignore_index=True)
+df = pd.concat([df, pd.DataFrame(update_data, columns=df.columns[0:3])], ignore_index=True)
 
 #%% Export
 df.to_csv(file_path, index=False)

data/sig_rules/update_sig_rules.py

@@ -3,7 +3,7 @@
 import pandas as pd
 import os
 import sys
-from datetime import datetime
+from datetime import date
 
 #%% Import the current dataset
 dir_path=os.path.dirname(os.path.realpath(__file__))
@@ -20,7 +20,7 @@
 last_year_with_data=df['Presidential Year (February 1 - January 31)'].iloc[-1]
 
 # Years to be updated
-current_year=datetime.now().year
+current_year = date.today().year
 
 if last_year_with_data<current_year-1:
     first_year_to_update=last_year_with_data+1
@@ -33,14 +33,14 @@
 #%% FR tracking data
 df_fr = pd.read_csv(f'{dir_path}/../fr_tracking/fr_tracking.csv', encoding="ISO-8859-1")
 
-df_fr['publication_date']=df_fr['publication_date'].astype('datetime64[ns]')
+df_fr['publication_date'] = pd.to_datetime(df_fr['publication_date'], format="mixed").dt.date
 df_fr['significant'] = pd.to_numeric(df_fr['significant'], errors='coerce')
 
 #%% Get user input pn presidential party
 def input_party(year):
     party_option = ['democratic','d','republican','r']
     while True:
-        party=input(f'Is presidential year {year} Democratic (D) or Republican (R)?').lower()
+        party=input(f'Is presidential year {year} Democratic (d) or Republican (r)? >>> ').lower()
         if party in party_option:
             output='Democratic' if (party in ['democratic','d']) else 'Republican'
             return output
@@ -51,8 +51,8 @@ def input_party(year):
 #%% Count annual economically/section 3(f)(1) significant rules
 update_data=[]
 for year in range(first_year_to_update,last_year_to_update+1):
-    count=df_fr[(df_fr['publication_date']>=datetime(year,2,1)) & \
-                (df_fr['publication_date']<=datetime(year+1,1,31))]['significant'].sum()
+    count=df_fr[(df_fr['publication_date'] >= date(year,2,1)) & \
+                (df_fr['publication_date'] <= date(year+1,1,31))]['significant'].sum()
     party=input_party(year)
     update_data.append([year,party,count])
     print(f'Presidential year {year} has been updated.')