Merge pull request #71 from rjsears/v0.991b

V0.991b

Author: rjsears

README.md

@@ -1,7 +1,7 @@
  <h2 align="center">
   <a name="chia_drive_logo" href="https://github.com/rjsears/chia_plot_manager"><img src="https://github.com/rjsears/chia_plot_manager/blob/main/images/chia_plot_manager_new.png" alt="Chia Plot Manager"></a><br>
 
-  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.991a - August 28th, 2023)
+  Chia Plot, Drive Manager, Coin Monitor & Auto Drive (V0.991b - Sept. 4th, 2023)
   </h2>
   <p align="center">
 Multi-Server Chia Plot, Drive Management Solution & Coin Monitor
@@ -582,7 +582,7 @@ Dual Power supplies
 <h2>Chia Famer Node</h2>
 
 ```
-Ubuntu 20.04.3 LTS VM running on Proxmox
+Ubuntu 22.04.3 LTS VM running on Proxmox
 ```
 
 My extra drive chassis are empty right now, but as you can see from my drive layout and mounting
@@ -591,6 +591,9 @@ the strategy above, it is super easy to add more drives.
 <br><hr>
 
 ### <a name="changelog"></a>Changelog
+<b>V0.991b 2023-09-04</b>
+   - Relocated  & reconfigured Farmer & Harvester Health checks so they could be installed/used standalone.
+
 <b>V0.991a 2023-08-28</b>
    - Updated farmer_health_check.py to include notifications, better checks, and better logging.
    - Updated install script for additional packages

extras/harvester_health_check.py

@@ -0,0 +1,196 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+"""
+Simple script that checks to make sure our chia_harvester is running and if not
+attempts to (re)start it. Sends notifications when necessary via email to text.
+
+This ONLY checks to see if 'chia_harvester' is a zombie process, or if it is not
+running at all. All other psutil p.info(status) just pass through.
+
+Set the email address and hostname below and load in your chia user crontab.
+"""
+VERSION = "V0.991b (2023-09-04)"
+
+import subprocess
+import apt
+import os
+import configparser
+import logging
+from system_logging import setup_logging
+from system_logging import read_logging_config
+from datetime import datetime
+import time
+from system_info import notify_email_address, notify_sms_address, from_address, host
+import psutil
+
+# Set config
+config = configparser.ConfigParser()
+
+# Do some housework
+today = datetime.today().strftime('%A').lower()
+current_military_time = datetime.now().strftime('%H:%M:%S')
+current_timestamp = int(time.time())
+
+
+# Setup Module logging. Main logging is configured in system_logging.py
+setup_logging()
+level = read_logging_config('harvester_health_config', 'system_logging', 'log_level')
+level = logging._checkLevel(level)
+log = logging.getLogger(__name__)
+log.setLevel(level)
+
+# Setup to read and write to our config file.
+# If we are expecting a boolean back pass True/1 for bool,
+# otherwise False/0
+def read_config_data(file, section, item, bool):
+    log.debug('read_config_data() Started')
+    pathname = '/home/chia/harvester_health/' + file
+    config.read(pathname)
+    if bool:
+        return config.getboolean(section, item)
+    else:
+        return config.get(section, item)
+
+def update_config_data(file, section, item, value):
+    log.debug('update_config_data() Started')
+    pathname = '/home/chia/harvester_health/' + file
+    config.read(pathname)
+    cfgfile = open(pathname, 'w')
+    config.set(section, item, value)
+    config.write(cfgfile)
+    cfgfile.close()
+
+
+# First, let's try and detect how we were installed
+# Depending on how we were installed (APT vs. Git) will
+# depend on how we call chia. If you installed via Git,
+# it is assumed that you installed at '/home/chia/', if
+# this is incorrect, you need to update the paths below.
+
+def how_installed():
+    log.debug('how_installed() called')
+    cache = apt.Cache()
+    cache.open()
+    response = "apt"
+    try:
+        cache['chia-blockchain'].is_installed or cache['chia-blockchain-cli'].is_installed
+        log.debug('It appears Chia was installed via apt....')
+    except KeyError:
+        log.debug('A Binary APT installation of Chia was not found, falling back to VENV!')
+        if os.path.isfile('/home/chia/chia-blockchain/venv/bin/chia'):
+            log.debug('It appears Chia was installed via VNEV....')
+            response = "venv"
+        else:
+            log.debug('A Chia Installation was not found. Exiting!')
+            exit()
+    return (response)
+
+
+def get_process(process_name):
+    log.debug('get_process() called')
+    procs = {p.pid: p.info for p in psutil.process_iter(['status']) if p.name() == process_name}
+    for i in procs:
+            for j, k in procs[i].items():
+                    return (k)
+
+def return_process_status(process_name):
+    log.debug('return_process_status() called')
+    status = (get_process(process_name))
+    if status is None:
+        return False
+    else:
+        return(status)
+
+def check_status(process):
+    log.debug('check_status() called')
+    status = return_process_status(process)
+    if not status:
+        return ('not_running')
+    elif status == 'zombie':
+        return ('zombie')
+    else:
+        return (status)
+
+
+def start_chia_harvester():
+    log.debug('start_chia_harvester() called')
+    installed = how_installed()
+    if installed == 'apt':
+        subprocess.call(['/usr/bin/chia', 'stop', '-d', 'all'])
+        time.sleep(10)
+        subprocess.call(['/usr/bin/chia', 'start', 'harvester'])
+    else:
+        subprocess.call(['/home/chia/chia-blockchain/venv/bin/chia', 'stop', '-d', 'all'])
+        time.sleep(10)
+        subprocess.call(['/home/chia/chia-blockchain/venv/bin/chia', 'start', 'harvester', '-r'])
+
+def check_chia_harvester():
+    log.debug('check_chia_haravester() called')
+    status = check_status('chia_harvester')
+    if status == 'zombie':
+        start_chia_harvester()
+        time.sleep(10)
+        status = check_status('chia_harvester')
+        if status == 'zombie':
+            notify('Zombie Infection', f'Harvester {host} has a Zombie Infection - Attempted Restart Failed - HELP ME!')
+            log.debug('Zombie detected and we could not restart, we are exiting....')
+            exit()
+        else:
+            send_email(notify_email_address, host, 'Zombie Killed, Harvester Restarted')
+            notify('Zombie Infection Killed', f'Harvester {host} had a Zombie Infection - Attempted Restart Succeeded')
+            log.debug('Zombie detected and Harvester Restart successful....')
+    elif status == 'not_running':
+        start_chia_harvester()
+        time.sleep(5)
+        status = check_status('chia_harvester')
+        if status == 'not_running':
+            notify('Harvester Not Running!', f'Harvester {host} NOT Running - Attempted Start Failed - HELP ME!')
+            log.debug('Harvester not running and an attempted restart failed...we are exiting')
+            exit()
+        else:
+            notify('Harvester Restart Successful', f'Harvester {host} has been restarted.')
+            log.debug('Harvester not running and an attempted restart was successful.....')
+    else:
+        print(status)
+
+def send_email(recipient, subject, body):
+    """
+    Part of our notification system.
+    Setup to send email via the builtin linux mail command.
+    Your local system **must** be configured already to send mail or this will fail.
+    https://stackoverflow.com/questions/27874102/executing-shell-mail-command-using-python
+    https://nedbatchelder.com/text/unipain.html
+    https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-postfix-as-a-send-only-smtp-server-on-ubuntu-20-04
+    """
+    try:
+        subprocess.run(['mail', '-s', subject, '-r', from_address, recipient], input=body, encoding='utf-8')
+        log.debug(f"Email Notification Sent: Subject: {subject}, Recipient: {recipient}, Message: {body}")
+    except subprocess.CalledProcessError as e:
+        log.debug(f'send_email error: {e}')
+    except Exception as e:
+        log.debug(f'send_email: Unknown Error! Email not sent.')
+
+
+def notify(title, message):
+    """ Notify system for email, and sms (via Email to SMS)"""
+    log.debug(f'notify() called with Title: {title} and Message: {message}')
+    if (read_config_data('farmer_health_config', 'notifications', 'alerting', True)):
+        if (read_config_data('harvester_health_config', 'notifications', 'email', True)):
+            for email_address in notify_email_address:
+                send_email(email_address, title, message)
+        if (read_config_data('harvester_health_config', 'notifications', 'sms', True)):
+            for email_address in notify_sms_address:
+                send_email(email_address, title, message)
+    else:
+        pass
+
+
+
+
+def main():
+    log.debug('harvester_health_check.py started')
+    check_chia_harvester()
+
+if __name__ == '__main__':
+    main()

harvester_health_check/harvester_health_check.py

@@ -0,0 +1,8 @@
+[system_logging]
+logging = True
+log_level = DEBUG
+
+[notifications]
+alerting = True
+email = True
+sms = True