fix merge upstream/v1.1.9

Author: utdrmac

.github/workflows/python-3.13.yml

@@ -0,0 +1,42 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python 3.13
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.13"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+
+        # don't care about coding standards
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

README.md

@@ -3,6 +3,7 @@
 ![Python 3.10](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/python-3.10.yml/badge.svg)
 ![Python 3.11](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/python-3.11.yml/badge.svg)
 ![Python 3.12](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/python-3.12.yml/badge.svg)
+![Python 3.13](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/python-3.13.yml/badge.svg)
 
 [![CodeQL](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/HotNoob/PythonProtocolGateway/actions/workflows/github-code-scanning/codeql)
 

classes/protocol_settings.py

@@ -3,6 +3,7 @@
 from enum import Enum
 import glob
 import logging
+import time
 from typing import Union
 from defs.common import strtoint
 import itertools
@@ -11,6 +12,11 @@
 import os
 import ast
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from configparser import SectionProxy
+
 class Data_Type(Enum):
     BYTE = 1
     '''8bit byte'''
@@ -206,6 +212,12 @@ class registry_map_entry:
     read_command : bytes = None
     ''' for transports/protocols that require sending a command ontop of "register" '''
 
+    read_interval : int = 1000
+    ''' how often to read register in ms'''
+
+    next_read_timestamp : int = 0
+    ''' unix timestamp in ms '''
+
     write_mode : WriteMode = WriteMode.READ
     ''' enable disable reading/writing '''
 
@@ -240,12 +252,14 @@ class protocol_settings:
     settings : dict[str, str]
     ''' default settings provided by protocol json '''
 
+    transport_settings : 'SectionProxy' = None
+
     byteorder : str = "big"
 
     _log : logging.Logger = None
 
 
-    def __init__(self, protocol : str, settings_dir : str = 'protocols'):
+    def __init__(self, protocol : str, transport_settings : 'SectionProxy' = None, settings_dir : str = 'protocols'):
 
         #apply log level to logger
         self._log_level = getattr(logging, logging.getLevelName(logging.getLogger().getEffectiveLevel()), logging.INFO)
@@ -254,6 +268,7 @@ def __init__(self, protocol : str, settings_dir : str = 'protocols'):
 
         self.protocol = protocol
         self.settings_dir = settings_dir
+        self.transport_settings = transport_settings
 
         #load variable mask
         self.variable_mask = []
@@ -358,13 +373,22 @@ def load__registry(self, path, registry_type : Registry_Type = Registry_Type.INP
         registry_map : list[registry_map_entry] = []
         register_regex = re.compile(r'(?P<register>(?:0?x[\da-z]+|[\d]+))\.(b(?P<bit>x?\d{1,2})|(?P<byte>x?\d{1,2}))')
 
+        read_interval_regex = re.compile(r'(?P<value>[\.\d]+)(?P<unit>[xs]|ms)')
+
+
         data_type_regex = re.compile(r'(?P<datatype>\w+)\.(?P<length>\d+)')
 
         range_regex = re.compile(r'(?P<reverse>r|)(?P<start>(?:0?x[\da-z]+|[\d]+))[\-~](?P<end>(?:0?x[\da-z]+|[\d]+))')
         ascii_value_regex = re.compile(r'(?P<regex>^\[.+\]$)')
         list_regex = re.compile(r'\s*(?:(?P<range_start>(?:0?x[\da-z]+|[\d]+))-(?P<range_end>(?:0?x[\da-z]+|[\d]+))|(?P<element>[^,\s][^,]*?))\s*(?:,|$)')
 
 
+        #load read_interval from transport settings, for #x per register read intervals
+        transport_read_interval : int = 1000
+        if self.transport_settings is not None:
+            transport_read_interval = self.transport_settings.getint("read_interval", transport_read_interval)
+
+
         if not os.path.exists(path): #return empty is file doesnt exist.
             return registry_map
 
@@ -392,10 +416,38 @@ def process_row(row):
             # Initialize variables to hold numeric and character parts
             unit_multiplier : float = 1
             unit_symbol : str = ''
+            read_interval : int = 0
+            ''' read interval in ms '''
 
              #clean up doc name, for extra parsing
             row['documented name'] = row['documented name'].strip().lower().replace(' ', '_')
 
+            #region read_interval
+
+
+            if 'read interval' in row:
+                row['read interval'] = row['read interval'].lower() #ensure is all lower case
+                match = read_interval_regex.search(row['read interval'])
+                if match:
+                    unit = match.group('unit')
+                    value = match.group('value')
+                    if value:
+                        value = float(value)
+                        if unit == 'x':
+                            read_interval = int((transport_read_interval * 1000) * value)
+                        else: # seconds or ms
+                            read_interval = value
+                            if unit != 'ms':
+                                read_interval *= 1000
+
+            if read_interval == 0:
+                read_interval = transport_read_interval * 1000
+                if "read_interval" in self.settings:
+                    try:
+                        read_interval = int(self.settings['read_interval'])
+                    except ValueError:
+                        read_interval = transport_read_interval * 1000
+
 
             #region overrides
             if overrides is not None:
@@ -589,6 +641,9 @@ def process_row(row):
             if "writable" in row:
                 writeMode = WriteMode.fromString(row['writable'])
 
+            if "write" in row:
+                writeMode = WriteMode.fromString(row['write'])
+
             for i in r:
                 item = registry_map_entry(
                                             registry_type = registry_type,
@@ -608,6 +663,7 @@ def process_row(row):
                                             value_max=value_max,
                                             value_regex=value_regex,
                                             read_command = read_command,
+                                            read_interval=read_interval,
                                             write_mode=writeMode
                                         )
                 registry_map.append(item)
@@ -706,7 +762,8 @@ def process_row(row):
 
             return registry_map
 
-    def calculate_registry_ranges(self, map : list[registry_map_entry], max_register : int) -> list[tuple]:
+    def calculate_registry_ranges(self, map : list[registry_map_entry], max_register : int, init : bool = False) -> list[tuple]:
+
         ''' read optimization; calculate which ranges to read'''
         max_batch_size = 45 #see manual; says max batch is 45
 
@@ -719,6 +776,10 @@ def calculate_registry_ranges(self, map : list[registry_map_entry], max_register
         start = -max_batch_size
         ranges : list[tuple] = []
 
+        timestamp_ms = int(time.time() * 1000)
+        if init : #hack so that all registers are read initially without adding extra if in loop
+            timestamp_ms = 0
+
         while (start := start+max_batch_size) <= max_register:
 
             registers : list[int] = [] #use a list, im too lazy to write logic
@@ -731,7 +792,10 @@ def calculate_registry_ranges(self, map : list[registry_map_entry], max_register
                     if register.write_mode == WriteMode.WRITEONLY: ##Write Only; skip
                         continue
 
-                    registers.append(register.register)
+                    #we are assuming calc registry ranges is being called EVERY READ.
+                    if register.next_read_timestamp < timestamp_ms:
+                        register.next_read_timestamp = timestamp_ms + register.read_interval
+                        registers.append(register.register)
 
             if registers: #not empty
                 ranges.append((min(registers), max(registers)-min(registers)+1)) ## APPENDING A TUPLE!
@@ -781,7 +845,7 @@ def load_registry_map(self, registry_type : Registry_Type, file : str = '', sett
                 size = item.register
 
         self.registry_map_size[registry_type] = size
-        self.registry_map_ranges[registry_type] = self.calculate_registry_ranges(self.registry_map[registry_type], self.registry_map_size[registry_type])
+        self.registry_map_ranges[registry_type] = self.calculate_registry_ranges(self.registry_map[registry_type], self.registry_map_size[registry_type], init=True)
 
     def process_register_bytes(self, registry : dict[int,bytes], entry : registry_map_entry):
         ''' process bytes into data'''

classes/transports/modbus_base.py

@@ -43,7 +43,7 @@ class modbus_base(transport_base):
     send_input_register : bool = True
 
     def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_settings' = None):
-        super().__init__(settings, protocolSettings=protocolSettings)
+        super().__init__(settings)
 
         self.analyze_protocol_enabled = settings.getboolean('analyze_protocol', fallback=self.analyze_protocol_enabled)
         self.analyze_protocol_save_load = settings.getboolean('analyze_protocol_save_load', fallback=self.analyze_protocol_save_load)
@@ -155,7 +155,10 @@ def read_data(self) -> dict[str, str]:
             if registry_type == Registry_Type.HOLDING and not self.send_holding_register:
                 continue
 
-            registry = self.read_modbus_registers(ranges=self.protocolSettings.get_registry_ranges(registry_type=registry_type), registry_type=registry_type)
+            #calculate ranges dynamically -- for variable read timing
+            ranges = self.protocolSettings.calculate_registry_ranges(self.protocolSettings.registry_map[registry_type], self.protocolSettings.registry_map_size[registry_type])
+
+            registry = self.read_modbus_registers(ranges=ranges, registry_type=registry_type)
             new_info = self.protocolSettings.process_registery(registry, self.protocolSettings.get_registry_map(registry_type))
 
             if False:

classes/transports/transport_base.py

@@ -32,7 +32,7 @@ class transport_base:
 
     _log : logging.Logger = None
 
-    def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_settings' = None) -> None:
+    def __init__(self, settings : 'SectionProxy') -> None:
 
         self.transport_name = settings.name #section name
 
@@ -45,17 +45,6 @@ def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_setti
 
         self.type = self.__class__.__name__
 
-        self.protocolSettings = protocolSettings
-        if not self.protocolSettings: #if not, attempt to load. lazy i know
-            self.protocol_version = settings.get('protocol_version')
-            if self.protocol_version:
-                self.protocolSettings = protocol_settings(self.protocol_version)
-
-        if self.protocolSettings:
-            self.protocol_version = self.protocolSettings.protocol
-
-            #todo, reimplement default settings from protocolsettings
-
         if settings:
             self.device_serial_number = settings.get(["device_serial_number", "serial_number"], self.device_serial_number)
             self.device_manufacturer = settings.get(["device_manufacturer", "manufacturer"], self.device_manufacturer)
@@ -68,6 +57,17 @@ def __init__(self, settings : 'SectionProxy', protocolSettings : 'protocol_setti
             else:
                 self.write_enabled = settings.getboolean("write", self.write_enabled)
 
+            #load a protocol_settings class for every transport; required for adv features. ie, variable timing.
+            #must load after settings
+            self.protocol_version = settings.get('protocol_version')
+            if self.protocol_version:
+                self.protocolSettings = protocol_settings(self.protocol_version, transport_settings=settings)
+
+                if self.protocolSettings:
+                    self.protocol_version = self.protocolSettings.protocol
+
+            #todo, reimplement default settings from protocolsettings
+
         self.update_identifier()
 
 

documentation/usage/creating_and_editing_protocols.md

@@ -11,8 +11,8 @@ The .csv files hold the registry or address definitions.
 CSV = comma seperated values... spreadsheets. 
 delimeter for csv can be , or ; ( not both )
 
-| variable name | data type | register|documented name|description|writable|values |
-| -- | -- | -- | -- | -- | -- | -- | 
+| variable name | data type | register| documented name|description|writable|values | read interval |
+| -- | -- | -- | -- | -- | -- | -- | -- |
 
 
 ### variable name
@@ -79,6 +79,28 @@ for example:
 the format for these flags is json. these flags / codes can also be defined via the .json file by naming them as such:
 "{{document_name}}_codes"
 
+### read interval
+provides a per register read interval; the minimum value is the configured transport read interval.
+
+#x for read_interval from transport config * #
+#s for plainold seconds
+#ms for miliseconds
+
+for example:
+``` 
+[transport.modbus]
+read_interval = 7
+ ```
+
+```7x```
+would set the read interval for that register to 49 seconds.
+
+```7s```
+would set the read interval to 7s
+
+```1s```
+because the transport read interval is 7 seconds, the read interval would effectively be 7 seconds
+
 ##### Bit Flag Example
 ```
 {"b0" : "StandBy", "b1" : "On"}

documentation/usage/protocols.md

@@ -37,29 +37,29 @@ protocol_version = eg4_v58.custom
 {protocol_name}.registry_map.csv contains configuration for generic "registers". 
 
 ### csv format:
-https://github.com/HotNoob/PythonProtocolGateway/wiki/Creating-and-Editing-Protocols-%E2%80%90-JSON-%E2%80%90-CSV#csv
+[creating_and_editing_protocols.md](creating_and_editing_protocols.md) - Creating and editing protocolss
 
 ## egv_v58
 ```
 protocol_version = eg4_v58
 ```
-[Devices\EG4 to MQTT](https://github.com/HotNoob/PythonProtocolGateway/wiki/Devices%5CEG4-to-MQTT)
+[Devices\EG4 to MQTT](/documentation/devices/EG4.md)
 
 ## v0.14
 ```
 protocol_version = v0.14
 ```
-[Devices\Growatt To MQTT](https://github.com/HotNoob/PythonProtocolGateway/wiki/Devices%5CGrowatt-To-MQTT)
+[Devices\Growatt To MQTT](/documentation/devices/Growatt.md)
 
 ## sigineer_v0.11
 
 ```
 protocol_version = sigineer_v0.11
 ```
-[Devices\Sigineer to MQTT](https://github.com/HotNoob/PythonProtocolGateway/wiki/Devices%5CSigineer-to-MQTT)
+[Devices\Sigineer to MQTT](/documentation/devices/Sigineer.md)
 
 ## pace_bms_v1.3
 ```
 protocol_version = pace_bms_v1.3
 ```
-[Devices\SOK to MQTT](https://github.com/HotNoob/PythonProtocolGateway/wiki/Devices%5CSOK-to-MQTT)
+[Devices\SOK to MQTT](/documentation/devices/SOK.md)