diff --git a/ChangeLog b/ChangeLog index 665f8bf..4022fa5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,16 @@ # ChangeLog +Sat, 18 Feb 2023 15:36:08 -0700 v1.7.4 + +- Update README.md and requirements.txt to reflect use of Python `requests` library (#78) + +Sat, 18 Feb 2023 14:22:52 -0700 v1.7.3 + +- Add `fs_disk_used` value +- Correct Disk Used % sensor to use "used" vs. "free" value +- Convert memory values from floting point string to integer (precision not necessary) +- Add new Mem Used % sensor (#42) + Sat, 11 Feb 2023 14:28:59 -0700 v1.7.2 - Repair fetch of versions, now every 12 hours diff --git a/Docs/images/DiscoveryV4.png b/Docs/images/DiscoveryV4.png new file mode 100644 index 0000000..5c949ed Binary files /dev/null and b/Docs/images/DiscoveryV4.png differ diff --git a/ISP-RPi-mqtt-daemon.py b/ISP-RPi-mqtt-daemon.py index aaa06ab..ec28398 100755 --- a/ISP-RPi-mqtt-daemon.py +++ b/ISP-RPi-mqtt-daemon.py @@ -24,10 +24,12 @@ import sdnotify from signal import signal, SIGPIPE, SIG_DFL signal(SIGPIPE, SIG_DFL) +import time import requests from urllib3.exceptions import InsecureRequestWarning -script_version = "1.7.2" + +script_version = "1.7.4" script_name = 'ISP-RPi-mqtt-daemon.py' script_info = '{} v{}'.format(script_name, script_version) project_name = 'RPi Reporter MQTT2HA Daemon' @@ -304,7 +306,7 @@ def getDaemonReleases(): daemon_version_list = newVersionList print_line('- RQST daemon_version_list=({})'.format(daemon_version_list), debug=True) - daemon_last_fetch_time = time() # record when we last fetched the versions + daemon_last_fetch_time = time.time() # record when we last fetched the versions getDaemonReleases() # and load them! print_line('* daemon_last_fetch_time=({})'.format(daemon_last_fetch_time), debug=True) @@ -347,6 +349,9 @@ def getDaemonReleases(): rpi_cpuload5 = '' rpi_cpuload15 = '' +# Time for network transfer calculation +previous_time = time.time() + # ----------------------------------------------------------------------------- # monitor variable fetch routines # @@ -454,7 +459,7 @@ def getDeviceMemory(): if 'MemAvail' in currLine: mem_avail = float(lineParts[1]) / 1024 # Tuple (Total, Free, Avail.) - rpi_memory_tuple = (mem_total, mem_free, mem_avail) + rpi_memory_tuple = (mem_total, mem_free, mem_avail) # [0]=total, [1]=free, [2]=avail. print_line('rpi_memory_tuple=[{}]'.format(rpi_memory_tuple), debug=True) def refreshPackageInfo(): @@ -617,7 +622,7 @@ def getNetworkIFsUsingIP(ip_cmd): def getSingleInterfaceDetails(interfaceName): - cmdString = '/sbin/ifconfig {} | /bin/egrep "Link|flags|inet |ether " | /bin/egrep -v -i "lo:|loopback|inet6|\:\:1|127\.0\.0\.1"'.format( + cmdString = '/sbin/ifconfig {} | /bin/egrep "Link|flags|inet |ether |TX packets |RX packets "'.format( interfaceName) out = subprocess.Popen(cmdString, shell=True, @@ -638,6 +643,7 @@ def getSingleInterfaceDetails(interfaceName): def loadNetworkIFDetailsFromLines(ifConfigLines): global rpi_interfaces global rpi_mac + global previous_time # # OLDER SYSTEMS # eth0 Link encap:Ethernet HWaddr b8:27:eb:c8:81:f2 @@ -647,14 +653,22 @@ def loadNetworkIFDetailsFromLines(ifConfigLines): # The following means eth0 (wired is NOT connected, and WiFi is connected) # eth0: flags=4099 mtu 1500 # ether b8:27:eb:1a:f3:bc txqueuelen 1000 (Ethernet) + # RX packets 0 bytes 0 (0.0 B) + # TX packets 0 bytes 0 (0.0 B) # wlan0: flags=4163 mtu 1500 # inet 192.168.100.189 netmask 255.255.255.0 broadcast 192.168.100.255 # ether b8:27:eb:4f:a6:e9 txqueuelen 1000 (Ethernet) + # RX packets 1358790 bytes 1197368205 (1.1 GiB) + # TX packets 916361 bytes 150440804 (143.4 MiB) # tmpInterfaces = [] haveIF = False imterfc = '' rpi_mac = '' + current_time = time.time() + if current_time == previous_time: + current_time += 1 + for currLine in ifConfigLines: lineParts = currLine.split() #print_line('- currLine=[{}]'.format(currLine), debug=True) @@ -693,12 +707,33 @@ def loadNetworkIFDetailsFromLines(ifConfigLines): rpi_mac = lineParts[1] print_line('rpi_mac=[{}]'.format(rpi_mac), debug=True) print_line('newTuple=[{}]'.format(newTuple), debug=True) + elif 'RX' in currLine: # NEWER ONLY + previous_value = getPreviousNetworkData(imterfc, 'rx_data') + current_value = int(lineParts[4]) + rx_data = round((current_value - previous_value) / (current_time - previous_time) * 8 / 1024) + newTuple = (imterfc, 'rx_data', rx_data) + tmpInterfaces.append(newTuple) + print_line('newTuple=[{}]'.format(newTuple), debug=True) + elif 'TX' in currLine: # NEWER ONLY + previous_value = getPreviousNetworkData(imterfc, 'tx_data') + current_value = int(lineParts[4]) + tx_data = round((current_value - previous_value) / (current_time - previous_time) * 8 / 1024) + newTuple = (imterfc, 'tx_data', tx_data) + tmpInterfaces.append(newTuple) + print_line('newTuple=[{}]'.format(newTuple), debug=True) haveIF = False rpi_interfaces = tmpInterfaces print_line('rpi_interfaces=[{}]'.format(rpi_interfaces), debug=True) print_line('rpi_mac=[{}]'.format(rpi_mac), debug=True) +def getPreviousNetworkData(interface, field): + global rpi_interfaces + value = [item for item in rpi_interfaces if item[0] == interface and item[1] == field] + if len(value) > 0: + return value[0][2] + else: + return 0 def getNetworkIFs(): ip_cmd = getIPCmd() @@ -875,7 +910,9 @@ def getSystemTemperature(): if cmd_fspec == '': rpi_system_temp = float('-1.0') rpi_gpu_temp = float('-1.0') - rpi_cpu_temp = float('-1.0') + rpi_cpu_temp = getSystemCPUTemperature() + if rpi_cpu_temp != -1.0: + rpi_system_temp = rpi_cpu_temp else: retry_count = 3 while retry_count > 0 and 'failed' in rpi_gpu_temp_raw: @@ -899,14 +936,7 @@ def getSystemTemperature(): rpi_gpu_temp = interpretedTemp print_line('rpi_gpu_temp=[{}]'.format(rpi_gpu_temp), debug=True) - out = subprocess.Popen("/bin/cat /sys/class/thermal/thermal_zone0/temp", - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - stdout, _ = out.communicate() - rpi_cpu_temp_raw = stdout.decode('utf-8').rstrip() - rpi_cpu_temp = float(rpi_cpu_temp_raw) / 1000.0 - print_line('rpi_cpu_temp=[{}]'.format(rpi_cpu_temp), debug=True) + rpi_cpu_temp = getSystemCPUTemperature() # fallback to CPU temp is GPU not available rpi_system_temp = rpi_gpu_temp @@ -914,6 +944,24 @@ def getSystemTemperature(): rpi_system_temp = rpi_cpu_temp +def getSystemCPUTemperature(): + cmd_locn1 = '/sys/class/thermal/thermal_zone0/temp' + cmdString = '/bin/cat {}'.format( + cmd_locn1) + if os.path.exists(cmd_locn1) == False: + rpi_cpu_temp = float('-1.0') + else: + out = subprocess.Popen(cmdString, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + stdout, _ = out.communicate() + rpi_cpu_temp_raw = stdout.decode('utf-8').rstrip() + rpi_cpu_temp = float(rpi_cpu_temp_raw) / 1000.0 + print_line('rpi_cpu_temp=[{}]'.format(rpi_cpu_temp), debug=True) + return rpi_cpu_temp + + def getSystemThermalStatus(): global rpi_throttle_status # sudo vcgencmd get_throttled @@ -1211,6 +1259,7 @@ def isAliveTimerRunning(): LD_FS_USED = "disk_used" LDS_PAYLOAD_NAME = "info" LD_CPU_USE = "cpu_load" +LD_MEM_USED = "mem_used" if interval_in_minutes < 5: LD_CPU_USE_JSON = "cpu.load_1min_prcnt" @@ -1403,6 +1452,8 @@ def isPeriodTimerRunning(): RPI_DATE_LAST_UPDATE = "last_update" RPI_FS_SPACE = 'fs_total_gb' # "fs_space_gbytes" RPI_FS_AVAIL = 'fs_free_prcnt' # "fs_available_prcnt" +RPI_FS_USED = 'fs_used_prcnt' # "fs_used_prcnt" +RPI_RAM_USED = 'mem_used_prcnt' # "mem_used_prcnt" RPI_SYSTEM_TEMP = "temperature_c" RPI_GPU_TEMP = "temp_gpu_c" RPI_CPU_TEMP = "temp_cpu_c" @@ -1466,7 +1517,9 @@ def send_status(timestamp, nothing): else: rpiData[RPI_DATE_LAST_UPDATE] = '' rpiData[RPI_FS_SPACE] = int(rpi_filesystem_space.replace('GB', ''), 10) - rpiData[RPI_FS_AVAIL] = int(rpi_filesystem_percent, 10) + # TODO: consider eliminating RPI_FS_AVAIL/fs_free_prcnt as used is needed but free is not... (can be calculated) + rpiData[RPI_FS_AVAIL] = 100 - int(rpi_filesystem_percent, 10) + rpiData[RPI_FS_USED] = int(rpi_filesystem_percent, 10) rpiData[RPI_NETWORK] = getNetworkDictionary() @@ -1477,6 +1530,11 @@ def send_status(timestamp, nothing): rpiRam = getMemoryDictionary() if len(rpiRam) > 0: rpiData[RPI_MEMORY] = rpiRam + ramSizeMB = int('{:.0f}'.format(rpi_memory_tuple[0], 10)) # "mem_space_mbytes" + # used is total - free + ramUsedMB = int('{:.0f}'.format(rpi_memory_tuple[0] - rpi_memory_tuple[2]), 10) + ramUsedPercent = int((ramUsedMB / ramSizeMB) * 100) + rpiData[RPI_RAM_USED] = ramUsedPercent # "mem_used_prcnt" rpiCpu = getCPUDictionary() if len(rpiCpu) > 0: @@ -1571,8 +1629,9 @@ def getMemoryDictionary(): # Tuple (Total, Free, Avail.) memoryData = OrderedDict() if rpi_memory_tuple != '': - memoryData[RPI_MEM_TOTAL] = '{:.3f}'.format(rpi_memory_tuple[0]) - memoryData[RPI_MEM_FREE] = '{:.3f}'.format(rpi_memory_tuple[2]) + # TODO: remove free fr + memoryData[RPI_MEM_TOTAL] = int(rpi_memory_tuple[0]) + memoryData[RPI_MEM_FREE] = int(rpi_memory_tuple[2]) #print_line('memoryData:{}"'.format(memoryData), debug=True) return memoryData @@ -1612,6 +1671,7 @@ def update_values(): getSystemThermalStatus() getLastUpdateDate() getDeviceMemory() + getNetworkIFs() # ----------------------------------------------------------------------------- @@ -1664,7 +1724,7 @@ def afterMQTTConnect(): # our INTERVAL timer does the work sleep(10000) - timeNow = time() + timeNow = time.time() if timeNow > daemon_last_fetch_time + kVersionCheckIntervalInSeconds: getDaemonReleases() # and load them! diff --git a/README.md b/README.md index db052ea..e006cbf 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ user, a simple workaround could be: A simple Linux python script to query the Raspberry Pi on which it is running for various configuration and status values which it then reports via via [MQTT](https://projects.eclipse.org/projects/iot.mosquitto) to your [Home Assistant](https://www.home-assistant.io/) installation. This allows you to install and run this on each of your RPi's so you can track them all via your own Home Assistant Dashboard. -![Discovery image](./Docs/images/DiscoveryV3.png) +![Discovery image](./Docs/images/DiscoveryV4.png) This script should be configured to be run in **daemon mode** continously in the background as a systemd service (or optionally as a SysV init script). Instructions are provided below. @@ -101,52 +101,57 @@ Each RPi device is reported as: ### RPi MQTT Topics -Each RPi device is reported as four topics: +Each RPi device is reported as five topics: | Name | Device Class | Units | Description | | --------------- | ------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `~/monitor` | 'timestamp' | n/a | Is a timestamp which shows when the RPi last sent information, carries a template payload conveying all monitored values (**attach the lovelace custom card to this sensor!**) | | `~/temperature` | 'temperature' | degrees C | Shows the latest system temperature | -| `~/disk_used` | none | percent (%) | Shows the amount of root file system used | +| `~/disk_used` | none | percent (%) | Shows the percent of root file system used | | `~/cpu_load` | none | percent (%) | Shows CPU load % over the last 5 minutes | +| `~/mem_used` | none | percent (%) | Shows the percent of RAM used | ### RPi Monitor Topic The monitored topic reports the following information: -| Name | Sub-name | Description | -| ------------------- | ------------------ | --------------------------------------------------------------------------------------------------- | -| `rpi_model` | | tinyfied hardware version string | -| `ifaces` | | comma sep list of interfaces on board [w,e,b] | -| `temperature_c` | | System temperature, in [°C] (0.1°C resolution) Note: this is GPU temp. if available, else CPU temp. | -| `temp_gpu_c` | | GPU temperature, in [°C] (0.1°C resolution) | -| `temp_cpu_c` | | CPU temperature, in [°C] (0.1°C resolution) | -| `up_time` | | duration since last booted, as [days] | -| `last_update` | | updates last applied, as [date] | -| `fs_total_gb` | | / total space in [GBytes] | -| `fs_free_prcnt` | | / free space [%] | -| `host_name` | | hostname | -| `fqdn` | | hostname.domain | -| `ux_release` | | os release name (e.g., buster) | -| `ux_version` | | os version (e.g., 4.19.66-v7+) | -| `reporter` | | script name, version running on RPi | -| `networking` | | lists for each interface: interface name, mac address (and IP if the interface is connected) | -| `drives` | | lists for each drive mounted: size in GB, % used, device and mount point | -| `cpu` | | lists the model of cpu, number of cores, etc. | -| | `hardware` | typically the Broadcom chip ID (e.g. BCM2835) | -| | `model` | model description string (e.g., ARMv7 Processor rev 4 (v7l)) | -| | `number_cores` | number of cpu cores [1,4] | -| | `bogo_mips` | reported performance of this RPi | -| | `serial` | serial number of this RPi | -| | `load_1min_prcnt` | average % cpu load during prior minute (avg per core) | -| | `load_5min_prcnt` | average % cpu load during prior 5 minutes (avg per core) | -| | `load_15min_prcnt` | average % cpu load during prior 15 minutes (avg per core) | -| `memory` | | shows the total amount of RAM in MB and the available ram in MB | -| `reporter` | | name and version of the script reporting these values | -| `reporter_releases` | | list of latest reporter formal versions | -| `report_interval` | | interval in minutes between reports from this script | -| `throttle` | | reports the throttle status value plus interpretation thereof | -| `timestamp` | | date, time when this report was generated | +| Name | Sub-name | Description | +| ------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------- | +| `rpi_model` | | tinyfied hardware version string | +| `ifaces` | | comma sep list of interfaces on board [w,e,b] | +| `temperature_c` | | System temperature, in [°C] (0.1°C resolution) Note: this is GPU temp. if available, else CPU temp. (used by HA sensor) | +| `temp_gpu_c` | | GPU temperature, in [°C] (0.1°C resolution) | +| `temp_cpu_c` | | CPU temperature, in [°C] (0.1°C resolution) | +| `up_time` | | duration since last booted, as [days] | +| `last_update` | | updates last applied, as [date] | +| `fs_total_gb` | | / (root) total space in [GBytes] | +| `fs_free_prcnt` | | / (root) available space [%] | +| `fs_used_prcnt` | | / (root) used space [%] (used by HA sensor) | +| `host_name` | | hostname | +| `fqdn` | | hostname.domain | +| `ux_release` | | os release name (e.g., buster) | +| `ux_version` | | os version (e.g., 4.19.66-v7+) | +| `reporter` | | script name, version running on RPi | +| `networking` | | lists for each interface: interface name, mac address (and IP if the interface is connected) | +| `drives` | | lists for each drive mounted: size in GB, % used, device and mount point | +| `cpu` | | lists the model of cpu, number of cores, etc. | +| | `hardware` | - typically the Broadcom chip ID (e.g. BCM2835) | +| | `model` | - model description string (e.g., ARMv7 Processor rev 4 (v7l)) | +| | `number_cores` | - number of cpu cores [1,4] | +| | `bogo_mips` | - reported performance of this RPi | +| | `serial` | - serial number of this RPi | +| | `load_1min_prcnt` | - average % cpu load during prior minute (avg per core) | +| | `load_5min_prcnt` | - average % cpu load during prior 5 minutes (avg per core) | +| | `load_15min_prcnt` | - average % cpu load during prior 15 minutes (avg per core) | +| `memory` | | shows the RAM configuration in MB for this RPi | +| | `size_mb` | - total memory Size in MBytes | +| | `free_mb` | - available memory in MBytes | +| `mem_used_prcnt` | | shows the amount of RAM currently in use (used by HA sensor) | +| `reporter` | | name and version of the script reporting these values | +| `reporter_releases` | | list of latest reporter formal versions | +| `report_interval` | | interval in minutes between reports from this script | +| `throttle` | | reports the throttle status value plus interpretation thereof | +| `timestamp` | | date, time when this report was generated | _NOTE: cpu load averages are divided by the number of cores_ @@ -180,9 +185,48 @@ sudo apt-get install libraspberrypi-bin net-tools ### Packages for Arch Linux ```shell -sudo pacman -S python python-pip python-tzlocal python-notify2 python-colorama python-unidecode python-paho-mqtt inetutils +sudo pacman -S python python-pip python-tzlocal python-notify2 python-colorama python-unidecode python-paho-mqtt python-requests inetutils ``` +### With these extra packages installed, verify access to network information + +The Daemon script needs access to information about how your RPi connects to the network. It uses `ifconfig(8)` to look up connection names and get the RPi IP address, etc. + +Let's run `ifconfig` to insure you have it installed. + +```shell +# run ifconfig(8) to see your RPi networking info +ifconfig +eth0: flags=4099 mtu 1500 + ether xx:xx:xx:xx:xx:xx txqueuelen 1000 (Ethernet) + RX packets 0 bytes 0 (0.0 B) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 0 bytes 0 (0.0 B) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +lo: flags=73 mtu 65536 + inet 127.0.0.1 netmask 255.0.0.0 + inet6 ::1 prefixlen 128 scopeid 0x10 + loop txqueuelen 1000 (Local Loopback) + RX packets 41342 bytes 2175319 (2.0 MiB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 41342 bytes 2175319 (2.0 MiB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +wlan0: flags=4163 mtu 1500 + inet xxx.xxx.xxx.xxx netmask 255.255.255.0 broadcast xxx.xxx.xxx.xxx + inet6 ... {omitted} ... + inet6 ... {omitted} ... + ether xx:xx:xx:xx:xx:xx txqueuelen 1000 (Ethernet) + RX packets 1458134 bytes 344599963 (328.6 MiB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 299694 bytes 51281531 (48.9 MiB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +``` + +If you are seeing output from the `ifconfig` tool then continue on with the following steps. If you don't you may have missed installing `net-utils` in an earlier step. + ### Now finish with the script install Now that the extra packages are installed let's install our script and any remaining supporting python modules. @@ -382,59 +426,69 @@ An example: ```json { "info": { - "timestamp": "2021-12-02T17:45:48-07:00", - "rpi_model": "RPi 3 Model B r1.2", + "timestamp": "2023-02-23T15:38:43-07:00", + "rpi_model": "RPi 4 Model B r1.5", "ifaces": "e,w,b", - "host_name": "pibtle", - "fqdn": "pibtle.home", - "ux_release": "stretch", - "ux_version": "4.19.66-v7+", - "up_time": "46 days, 23:14", - "last_update": "2021-11-30T18:29:45-07:00", + "host_name": "pip2iotgw", + "fqdn": "pip2iotgw.home", + "ux_release": "bullseye", + "ux_version": "5.15.84-v8+", + "up_time": "10 days, 35 min", + "last_update": "2023-02-23T15:04:15-07:00", "fs_total_gb": 32, - "fs_free_prcnt": 41, + "fs_free_prcnt": 81, + "fs_used_prcnt": 19, "networking": { "eth0": { - "IP": "192.168.100.81", - "mac": "b8:27:eb:d1:16:42" + "mac": "e4:5f:01:f8:18:01", + "rx_data": 0, + "tx_data": 0 }, "wlan0": { - "mac": "b8:27:eb:84:43:17" + "IP": "192.168.100.196", + "mac": "e4:5f:01:f8:18:02", + "rx_data": 6948, + "tx_data": 977 } }, "drives": { "root": { "size_gb": 32, - "used_prcnt": 41, + "used_prcnt": 19, "device": "/dev/root", "mount_pt": "/" } }, "memory": { - "size_mb": "926.078", - "free_mb": "215.750" + "size_mb": 1849, + "free_mb": 806 }, + "mem_used_prcnt": 56, "cpu": { "hardware": "BCM2835", - "model": "ARMv7 Processor rev 4 (v7l)", + "model": "", "number_cores": 4, - "bogo_mips": "307.20", - "serial": "00000000a8d11642", - "load_1min_prcnt": 23.2, - "load_5min_prcnt": 22.5, - "load_15min_prcnt": 22.8 + "bogo_mips": "432.00", + "serial": "1000000081ae88c7",`` + "load_1min_prcnt": 0.5, + "load_5min_prcnt": 0.8, + "load_15min_prcnt": 3.8 }, - "throttle": ["throttled = 0x0", "Not throttled"], - "temperature_c": 63.4, - "temp_gpu_c": 63.4, - "temp_cpu_c": 62.8, - "reporter": "ISP-RPi-mqtt-daemon v1.6.0", + "throttle": [ + "throttled = 0x0", + "Not throttled" + ], + "temperature_c": 28.2, + "temp_gpu_c": 28.2, + "temp_cpu_c": 29.2, + "reporter": "ISP-RPi-mqtt-daemon v1.7.5", + "reporter_releases": "v1.7.5,v1.7.2,v1.7.3,v1.7.4", "report_interval": 5 } } ``` -**NOTE:** Where there's an IP address that interface is connected. +**NOTE:** Where there's an IP address that interface is connected. Also, there are new `tx_data` and `rx_data` values which show traffic in bytes for this reporting interval for each network interface. This data can be subscribed to and processed by your home assistant installation. How you build your RPi dashboard from here is up to you! diff --git a/requirements.txt b/requirements.txt index ca8a59d..fbf3c67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ sdnotify>=0.3.1 Unidecode>=0.4.21 colorama>=0.4.3 tzlocal>=2.1.0 +requests>=2.28.2