feat: added more logging to script.py to troubleshoot problems

.github/workflows/scrape.yaml

@@ -53,9 +53,8 @@ jobs:
         uses: EndBug/add-and-commit@v9
         with:
           message: "Commit updated data 📈"
-          add: "./data/*.json ./data/*.csv"
+          add: "./data/*.json"
           cwd: "."
-          force: true
           default_author: github_actions
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

script.py

@@ -65,6 +65,23 @@ def scrape_data_point():
         dem.save()
         loguru.logger.info("Saved daily event monitor")
 
+    def print_tree(directory, ignore_dirs=[".git", "__pycache__"]):
+        loguru.logger.info(f"Printing tree of files/dirs at {directory}")
+        for root, dirs, files in os.walk(directory):
+            dirs[:] = [d for d in dirs if d not in ignore_dirs]
+            level = root.replace(directory, "").count(os.sep)
+            indent = " " * 4 * (level)
+            loguru.logger.info(f"{indent}+--{os.path.basename(root)}/")
+            sub_indent = " " * 4 * (level + 1)
+            for file in files:
+                loguru.logger.info(f"{sub_indent}+--{file}")
+
+    print_tree(os.getcwd())
+
+    loguru.logger.info("Printing contents of data file {}".format(dem.file_path))
+    with open(dem.file_path, "r") as f:
+        loguru.logger.info(f.read())
+
     # Finish
     loguru.logger.info("Scrape complete")
     loguru.logger.info("Exiting")