Fix charge log (#1757)

* fix chargelog

* fix rebase

* test

* test

* fix

* f

* fix

* fix

* test

* running test only local

* undo

packages/conftest.py

@@ -1,6 +1,7 @@
 import datetime
 from unittest.mock import MagicMock, Mock
 import pytest
+import pytz
 
 from control import data
 from control.bat import Bat, BatData
@@ -27,6 +28,16 @@ def mock_today(monkeypatch) -> None:
     monkeypatch.setattr(timecheck, "create_timestamp", mock_today_timestamp)
 
 
+@pytest.fixture(autouse=False)
+def mock_strptime_timestamp(monkeypatch):
+    # Timezone-Objekt muss vor dem Mock von open initialisiert werden, da dies auch open verwendet und mit dem Mock
+    # nicht mehr funktioniert
+    timezone = pytz.timezone('Europe/Berlin')
+    original_strptime = datetime.datetime.strptime
+    monkeypatch.setattr(datetime.datetime, "strptime",
+                        lambda s, fmt: timezone.localize(original_strptime(s, fmt)))
+
+
 @pytest.fixture(autouse=True)
 def mock_pub(monkeypatch) -> Mock:
     pub_mock = Mock()

packages/control/chargelog/chargelog.py

@@ -450,8 +450,7 @@ def get_reference_time(cp, reference_position):
         return timecheck.create_timestamp() - 3540
     elif reference_position == ReferenceTime.END:
         # Wenn der Ladevorgang erst innerhalb der letzten Stunde gestartet wurde.
-        one_hour_back = timecheck.create_timestamp() - 3600
-        if (one_hour_back - cp.data.set.log.timestamp_start_charging) < 0:
+        if timecheck.create_unix_timestamp_current_full_hour() <= cp.data.set.log.timestamp_start_charging:
             return cp.data.set.log.timestamp_start_charging
         else:
             return timecheck.create_unix_timestamp_current_full_hour() + 60

packages/control/chargelog/chargelog_test.py

@@ -0,0 +1,154 @@
+
+import datetime
+from unittest.mock import Mock
+
+import pytest
+
+from control import data
+from control.chargelog import chargelog
+from control.chargelog.chargelog import calculate_charge_cost
+from control.chargepoint.chargepoint import Chargepoint
+from helpermodules import timecheck
+from test_utils.test_environment import running_on_github
+
+
+def mock_daily_log_with_charging(date: str, num_of_intervalls, monkeypatch):
+    """erzeugt ein daily_log, im ersten Eintrag gibt es keine Änderung, danach wird bis inklusive dem letzten Beitrag
+    geladen"""
+    bat_exported = pv_exported = cp_imported = counter_imported = 2000
+    date = datetime.datetime.strptime(date, "%m/%d/%Y, %H:%M")
+    daily_log = {"entries": []}
+    for i in range(0, num_of_intervalls):
+        if i != 0:
+            bat_exported += 1000
+            pv_exported += 500
+            cp_imported += 2000
+            counter_imported += 500
+        daily_log["entries"].append({'bat': {'all': {'exported': bat_exported, 'imported': 2000, 'soc': 100},
+                                             'bat2': {'exported': bat_exported, 'imported': 2000, 'soc': 100}},
+                                     'counter': {'counter0': {'exported': 2000,
+                                                              'grid': True,
+                                                              'imported': counter_imported}},
+                                     'cp': {'all': {'exported': 0, 'imported': cp_imported},
+                                            'cp4': {'exported': 0, 'imported': cp_imported}},
+                                     'date': date.strftime("%H:%M"),
+                                     'ev': {'ev0': {'soc': None}},
+                                     'hc': {'all': {'imported': 0}},
+                                     'pv': {'all': {'exported': pv_exported}, 'pv1': {'exported': pv_exported}},
+                                     'sh': {},
+                                     'timestamp': date.timestamp()})
+        date += datetime.timedelta(minutes=5)
+    mock_todays_daily_log = Mock(return_value=daily_log)
+    monkeypatch.setattr(chargelog, "get_todays_daily_log", mock_todays_daily_log)
+    return daily_log
+
+
+@pytest.fixture()
+def mock_data() -> None:
+    data.data_init(Mock())
+    data.data.optional_data.et_module = None
+
+
+def mock_create_entry_reference_end(clock, daily_log, monkeypatch):
+    current_log = daily_log["entries"][-1]
+    current_log["cp"]["all"]["imported"] += 500
+    current_log["cp"]["cp4"]["imported"] += 500
+    current_log["counter"]["counter0"]["imported"] += 500
+    current_log["date"] = clock
+    current_log["timestamp"] = datetime.datetime.strptime(f"05/16/2022, {clock}", "%m/%d/%Y, %H:%M").timestamp()
+    mock_create_entry = Mock(return_value=current_log)
+    monkeypatch.setattr(chargelog, "create_entry", mock_create_entry)
+
+
+def init_cp(charged_energy, costs, start_hour, start_minute=47):
+    cp = Chargepoint(4, None)
+    cp.data.set.log.imported_since_plugged = cp.data.set.log.imported_since_mode_switch = charged_energy
+    cp.data.set.log.timestamp_start_charging = datetime.datetime(2022, 5, 16, start_hour, start_minute).timestamp()
+    cp.data.get.imported = charged_energy + 2000
+    cp.data.set.log.costs = costs
+    return cp
+
+
+def test_calc_charge_cost_no_hour_change_reference_end(mock_data, monkeypatch):
+    cp = init_cp(6500, 0, 10, start_minute=27)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 10:25", 4, monkeypatch)
+    mock_create_entry_reference_end("10:42", daily_log, monkeypatch)
+
+    calculate_charge_cost(cp, True)
+
+    assert cp.data.set.log.costs == 1.425
+
+
+def test_calc_charge_cost_first_hour_change_reference_begin(mock_data, monkeypatch):
+    cp = init_cp(6000, 0, 7)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 07:45", 4, monkeypatch)
+    current_log = daily_log["entries"][-1]
+    current_log["date"] = "08:00"
+    current_log["timestamp"] = datetime.datetime.strptime("05/16/2022, 08:00", "%m/%d/%Y, %H:%M").timestamp()
+    mock_create_entry = Mock(return_value=current_log)
+    monkeypatch.setattr(chargelog, "create_entry", mock_create_entry)
+
+    calculate_charge_cost(cp, False)
+
+    assert cp.data.set.log.costs == 1.275
+
+
+def test_calc_charge_cost_first_hour_change_reference_begin_day_change(mock_data, monkeypatch):
+    cp = init_cp(6000, 0, 23)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 23:45", 4, monkeypatch)
+    current_log = daily_log["entries"][-1]
+    current_log["date"] = "00:00"
+    current_log["timestamp"] = datetime.datetime.strptime("05/17/2022, 00:00", "%m/%d/%Y, %H:%M").timestamp()
+    mock_create_entry = Mock(return_value=current_log)
+    monkeypatch.setattr(chargelog, "create_entry", mock_create_entry)
+    mock_today_timestamp = Mock(return_value=1652738421)
+    monkeypatch.setattr(timecheck, "create_timestamp", mock_today_timestamp)
+
+    calculate_charge_cost(cp, False)
+
+    assert cp.data.set.log.costs == 1.275
+
+
+def test_calc_charge_cost_one_hour_change_reference_end(mock_data, monkeypatch):
+    if running_on_github():
+        # ToDo Zeitzonen berücksichtigen, damit Tests auf Github laufen
+        return
+    cp = init_cp(22500, 1.275, 7)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 07:45", 12, monkeypatch)
+    mock_create_entry_reference_end("08:40", daily_log, monkeypatch)
+
+    calculate_charge_cost(cp, True)
+
+    assert cp.data.set.log.costs == 4.8248999999999995
+
+
+def test_calc_charge_cost_two_hour_change_reference_middle(mock_data, monkeypatch):
+    if running_on_github():
+        # ToDo Zeitzonen berücksichtigen, damit Tests auf Github laufen
+        return
+    cp = init_cp(22500, 1.275, 6)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 06:45", 16, monkeypatch)
+    current_log = daily_log["entries"][-1]
+    current_log["date"] = "08:00"
+    current_log["timestamp"] = datetime.datetime(2022, 5, 16, 8).timestamp()
+    mock_create_entry = Mock(return_value=current_log)
+    monkeypatch.setattr(chargelog, "create_entry", mock_create_entry)
+    mock_today_timestamp = Mock(return_value=1652680801)
+    monkeypatch.setattr(timecheck, "create_timestamp", mock_today_timestamp)
+
+    calculate_charge_cost(cp, False)
+
+    assert cp.data.set.log.costs == 6.375
+
+
+def test_calc_charge_cost_two_hour_change_reference_end(mock_data, monkeypatch):
+    if running_on_github():
+        # ToDo Zeitzonen berücksichtigen, damit Tests auf Github laufen
+        return
+    cp = init_cp(46500, 6.375, 6)
+    daily_log = mock_daily_log_with_charging("05/16/2022, 06:45", 24, monkeypatch)
+    mock_create_entry_reference_end("08:40", daily_log, monkeypatch)
+
+    calculate_charge_cost(cp, True)
+
+    assert cp.data.set.log.costs == 9.924900000000001

packages/helpermodules/update_config.py

@@ -1,24 +1,28 @@
 from dataclasses import asdict
 import datetime
 import glob
+import importlib
 import json
 import logging
 from pathlib import Path
 import re
+from shutil import copyfile
 import time
 from typing import List, Optional
 from paho.mqtt.client import Client as MqttClient, MQTTMessage
 from control.bat_all import BatConsiderationMode
+from control.chargelog.chargelog import ReferenceTime
 from control.general import ChargemodeConfig
 import dataclass_utils
 
 from control.chargepoint.chargepoint_template import get_chargepoint_template_default
+from dataclass_utils._dataclass_from_dict import dataclass_from_dict
 from helpermodules import timecheck
 from helpermodules import hardware_configuration
 from helpermodules.broker import InternalBrokerClient
 from helpermodules.constants import NO_ERROR
 from helpermodules.hardware_configuration import get_hardware_configuration_setting, update_hardware_configuration
-from helpermodules.measurement_logging.process_log import get_totals
+from helpermodules.measurement_logging.process_log import CalculationType, analyse_percentage, get_totals, process_entry
 from helpermodules.measurement_logging.write_log import get_names
 from helpermodules.messaging import MessageType, pub_system_message
 from helpermodules.pub import Pub
@@ -42,7 +46,7 @@
 
 
 class UpdateConfig:
-    DATASTORE_VERSION = 52
+    DATASTORE_VERSION = 53
     valid_topic = [
         "^openWB/bat/config/configured$",
         "^openWB/bat/set/charging_power_left$",
@@ -1615,14 +1619,109 @@ def upgrade(topic: str, payload) -> Optional[dict]:
         self._loop_all_received_topics(upgrade)
         self.__update_topic("openWB/system/datastore_version", 51)
 
+    def upgrade_datastore_51(self) -> None:
+        def upgrade(topic: str, payload) -> None:
+            if re.search("openWB/system/device/[0-9]+", topic) is not None:
+                payload = decode_payload(payload)
+                # update version and firmware of GoodWe
+                if payload.get("type") == "deye" and "device_type" in payload["configuration"]:
+                    payload["configuration"].pop("device_type")
+                Pub().pub(topic, payload)
+        self._loop_all_received_topics(upgrade)
+        self.__update_topic("openWB/system/datastore_version", 52)
 
-def upgrade_datastore_51(self) -> None:
-    def upgrade(topic: str, payload) -> None:
-        if re.search("openWB/system/device/[0-9]+", topic) is not None:
-            payload = decode_payload(payload)
-            # update version and firmware of GoodWe
-            if payload.get("type") == "deye" and "device_type" in payload["configuration"]:
-                payload["configuration"].pop("device_type")
-            Pub().pub(topic, payload)
-    self._loop_all_received_topics(upgrade)
-    self.__update_topic("openWB/system/datastore_version", 52)
+    def upgrade_datastore_52(self) -> None:
+        # alle Einträge des ladelogs seit januar 2024 durchgehen
+        path_list = [f"{self.base_path}/data/charge_log/2024{i:02d}.json" for i in range(1, 8)]
+        for topic, payload in self.all_received_topics.items():
+            if "openWB/general/prices/bat" in topic:
+                bat_price = decode_payload(payload)
+            elif "openWB/general/prices/grid" in topic:
+                grid_price = decode_payload(payload)
+            elif "openWB/general/prices/pv" in topic:
+                pv_price = decode_payload(payload)
+            elif "openWB/optional/et/provider" in topic:
+                et_active = True if decode_payload(payload)["type"] is not None else False
+        for chargelog in path_list:
+            fixed = False
+            try:
+                with open(chargelog, "r") as jsonFile:
+                    content = json.load(jsonFile)
+                for chargelog_entry in content:
+                    # prüfen, ob der Ladevorgang nur einen Stundenwechsel enthält
+                    begin = datetime.datetime.strptime(chargelog_entry["time"]["begin"], "%m/%d/%Y, %H:%M:%S")
+                    end = datetime.datetime.strptime(chargelog_entry["time"]["end"], "%m/%d/%Y, %H:%M:%S")
+                    hour_change = datetime.datetime.strptime(end.strftime("%m/%d/%Y, %H"), "%m/%d/%Y, %H")
+                    if ((end.hour - begin.hour) == 1 or
+                            ((end.day - begin.day) == 1 and begin.hour == 23 and end.hour == 0)):
+                        def calc(first: datetime.datetime, second: datetime.datetime, reference: ReferenceTime):
+                            with open(f"{self.base_path}/data/daily_log/2024{first.month:02d}{first.day:02d}.json",
+                                      "r") as jsonFile:
+                                daily_log = json.load(jsonFile)
+                            first_entry, second_entry = None, None
+                            if ReferenceTime.MIDDLE == reference:
+                                for daily_entry in reversed(daily_log["entries"]):
+                                    # -1, damit auch Einträge um XX:00:00 berücksichtigt werden
+                                    if daily_entry["timestamp"] <= first.timestamp() - 1:
+                                        first_entry = daily_entry
+                                        break
+                            else:
+                                for daily_entry in daily_log["entries"]:
+                                    if daily_entry["timestamp"] >= first.timestamp() - 1:
+                                        first_entry = daily_entry
+                                        break
+                            if begin.day != end.day:
+                                with open(f"{self.base_path}/data/daily_log/2024{end.month:02d}{end.day:02d}.json",
+                                          "r") as jsonFile:
+                                    daily_log = json.load(jsonFile)
+                            for daily_entry in daily_log["entries"]:
+                                if daily_entry["timestamp"] >= second.timestamp() - 1:
+                                    second_entry = daily_entry
+                                    break
+                            energy_entry = process_entry(first_entry, second_entry, CalculationType.ENERGY)
+                            energy_source = analyse_percentage(energy_entry)["energy_source"]
+                            cp_num = chargelog_entry['chargepoint']['id']
+                            charged_energy = (second_entry["cp"][f"cp{cp_num}"]["imported"]
+                                              - first_entry["cp"][f"cp{cp_num}"]["imported"])
+
+                            bat_costs = bat_price * charged_energy * energy_source["bat"]
+                            if et_active:
+                                if reference == ReferenceTime.MIDDLE:
+                                    price = et_prices[0]
+                                else:
+                                    price = et_prices[1]
+                                grid_costs = price * charged_energy * \
+                                    energy_source["grid"]
+                            else:
+                                grid_costs = grid_price * charged_energy * energy_source["grid"]
+                            pv_costs = pv_price * charged_energy * energy_source["pv"]
+
+                            log.debug(
+                                f'Ladepreis für die letzte Stunde: {bat_costs}€ Speicher ({energy_source["bat"]}%), '
+                                f'{grid_costs}€ Netz ({energy_source["grid"]}%), {pv_costs}€ Pv ({energy_source["pv"]}%'
+                                ')')
+                            return round(bat_costs + grid_costs + pv_costs, 4)
+
+                        if et_active:
+                            config_dict = decode_payload(payload)
+                            mod = importlib.import_module(
+                                f".electricity_tariffs.{config_dict['type']}.tariff", "modules")
+                            config = dataclass_from_dict(
+                                mod.device_descriptor.configuration_factory, config_dict)
+                            tariff_state = mod.create_electricity_tariff(config)(begin.timestamp())
+                            et_prices = [tariff_state.prices[hour_change.timestamp()-3600],
+                                         tariff_state.prices[hour_change.timestamp()]]
+                            log.debug(f'ET-Preise {chargelog_entry["time"]["begin"]} {begin.timestamp()}: '
+                                      f'{et_prices}')
+                        new_costs = calc(begin, hour_change, ReferenceTime.MIDDLE)
+                        new_costs += calc(hour_change, end, ReferenceTime.END)
+                        if chargelog_entry["data"]["costs"] != new_costs:
+                            fixed = True
+                            chargelog_entry["data"]["costs"] = new_costs
+                if fixed:
+                    copyfile(chargelog, chargelog+".1")
+                    with open(chargelog, "w") as jsonFile:
+                        json.dump(content, jsonFile)
+            except FileNotFoundError:
+                log.debug(f"Keine Logdatei {chargelog} gefunden")
+        self.__update_topic("openWB/system/datastore_version", 53)