From bb1ffce5f60e7e63229544a69bed8a29dae21406 Mon Sep 17 00:00:00 2001 From: LKuemmel <76958050+LKuemmel@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:21:01 +0200 Subject: [PATCH] Revert "Onedrive_Backup (#968)" This reverts commit 0b4c1dd03fbe009936510b0d6efc300082595d3c. --- .github/workflows/github-actions-python.yml | 2 +- packages/helpermodules/command.py | 27 --- .../backup_clouds/onedrive/__init__.py | 0 .../modules/backup_clouds/onedrive/api.py | 170 -------------- .../backup_clouds/onedrive/backup_cloud.py | 42 ---- .../modules/backup_clouds/onedrive/config.py | 30 --- .../onedrive/msdrive/constants.py | 3 - .../backup_clouds/onedrive/msdrive/drive.py | 208 ------------------ .../onedrive/msdrive/exceptions.py | 14 -- .../onedrive/msdrive/onedrive.py | 30 --- requirements.txt | 1 - 11 files changed, 1 insertion(+), 526 deletions(-) delete mode 100644 packages/modules/backup_clouds/onedrive/__init__.py delete mode 100644 packages/modules/backup_clouds/onedrive/api.py delete mode 100644 packages/modules/backup_clouds/onedrive/backup_cloud.py delete mode 100644 packages/modules/backup_clouds/onedrive/config.py delete mode 100644 packages/modules/backup_clouds/onedrive/msdrive/constants.py delete mode 100644 packages/modules/backup_clouds/onedrive/msdrive/drive.py delete mode 100644 packages/modules/backup_clouds/onedrive/msdrive/exceptions.py delete mode 100644 packages/modules/backup_clouds/onedrive/msdrive/onedrive.py diff --git a/.github/workflows/github-actions-python.yml b/.github/workflows/github-actions-python.yml index 7dc26fc5ee..1d2a1020b4 100644 --- a/.github/workflows/github-actions-python.yml +++ b/.github/workflows/github-actions-python.yml @@ -14,7 +14,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2 msal + pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2 - name: Flake8 with annotations in packages folder uses: TrueBrain/actions-flake8@v2.1 with: diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 6b20eab52c..83a4502f4d 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -10,10 +10,8 @@ import traceback from pathlib import Path import paho.mqtt.client as mqtt - from control.chargepoint import chargepoint from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default -from modules.backup_clouds.onedrive.api import generateMSALAuthCode, retrieveMSALTokens from helpermodules import measurement_log from helpermodules.broker import InternalBrokerClient @@ -30,7 +28,6 @@ import dataclass_utils from modules.common.configurable_vehicle import IntervalConfig - log = logging.getLogger(__name__) @@ -698,30 +695,6 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: f'Restore-Status: {result.returncode}
Meldung: {result.stdout.decode("utf-8")}', MessageType.ERROR) - def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: - ''' fordert einen Authentifizierungscode für MSAL (Microsoft Authentication Library) - an um Onedrive Backup zu ermöglichen''' - cloudbackupconfig = SubData.system_data["system"].backup_cloud - if cloudbackupconfig is None: - pub_user_message(payload, connection_id, - "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " - "und erneut versuchen.
", MessageType.WARNING) - return - result = generateMSALAuthCode(cloudbackupconfig.config) - pub_user_message(payload, connection_id, result["message"], result["MessageType"]) - - def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: - """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen - """ - cloudbackupconfig = SubData.system_data["system"].backup_cloud - if cloudbackupconfig is None: - pub_user_message(payload, connection_id, - "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " - "und erneut versuchen.
", MessageType.WARNING) - return - result = retrieveMSALTokens(cloudbackupconfig.config) - pub_user_message(payload, connection_id, result["message"], result["MessageType"]) - def factoryReset(self, connection_id: str, payload: dict) -> None: Path(Path(__file__).resolve().parents[2] / 'data' / 'restore' / 'factory_reset').touch() pub_user_message(payload, connection_id, diff --git a/packages/modules/backup_clouds/onedrive/__init__.py b/packages/modules/backup_clouds/onedrive/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/modules/backup_clouds/onedrive/api.py b/packages/modules/backup_clouds/onedrive/api.py deleted file mode 100644 index 51e85dfdb1..0000000000 --- a/packages/modules/backup_clouds/onedrive/api.py +++ /dev/null @@ -1,170 +0,0 @@ -import logging -import pickle -import json -import paho.mqtt.publish as publish -import msal -import base64 - -from msal import PublicClientApplication -from helpermodules.messaging import MessageType -from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration - - -log = logging.getLogger(__name__) - - -def encode_str_base64(string: str) -> str: - string_bytes = string.encode("ascii") - string_base64_bytes = base64.b64encode(string_bytes) - string_base64_string = string_base64_bytes.decode("ascii") - return string_base64_string - - -def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None: - # encode cache to base64 and save to config - log.debug("saving updated tokencache to config") - config.persistent_tokencache = encode_str_base64(cache) - - # construct full configuartion object for cloud backup - backupcloud = OneDriveBackupCloud() - backupcloud.configuration = config - backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) - log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) - - publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost") - - -def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: - result = None - cache = msal.SerializableTokenCache() - - if config.persistent_tokencache: - cache.deserialize(base64.b64decode(config.persistent_tokencache)) - else: - raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") - - # Create a public client application with msal - log.debug("creating MSAL public client application") - app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) - - log.debug("getting accounts") - accounts = app.get_accounts() - if accounts: - chosen = accounts[0] # assume that we only will have a single account in cache - log.debug("selected account " + str(chosen["username"])) - # Now let's try to find a token in cache for this account - result = app.acquire_token_silent(scopes=config.scope, account=chosen) - else: - raise Exception("No matching account found,please re-configure and re-authorize access Cloud backup settings.") - - log.debug("done acquring tokens") - if not result: # We have no token for this account, so the end user shall sign-in - raise Exception("No token found, please re-configure and re-authorize access Cloud backup settings.") - - if "access_token" in result: - log.debug("access token retrieved") - save_tokencache(config=config, cache=cache.serialize()) - else: - # Print the error - raise Exception("Error retrieving access token", result.get("error"), result.get("error_description")) - return result - - -def generateMSALAuthCode(cloudbackup: OneDriveBackupCloud) -> dict: - """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup - und speichert den AuthCode in der Konfiguration""" - result = dict( - message="", - MessageType=MessageType.SUCCESS - ) - - if cloudbackup is None: - result["message"] = """Es ist keine Backup-Cloud konfiguriert. - Bitte Konfiguration speichern und erneut versuchen.
""" - result["MessageType"] = MessageType.WARNING - return result - - # Create a public client application with msal - app = PublicClientApplication( - client_id=cloudbackup.configuration.clientID, - authority=cloudbackup.configuration.authority - ) - - # create device flow to obtain auth code - flow = app.initiate_device_flow(cloudbackup.configuration.scope) - if "user_code" not in flow: - raise Exception( - "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - - flow["expires_at"] = 0 # Mark it as expired immediately to prevent - pickleString = str(pickle.dumps(flow), encoding='latin1') - - cloudbackup.configuration.flow = str(pickleString) - cloudbackup.configuration.authcode = flow["user_code"] - cloudbackup.configuration.authurl = flow["verification_uri"] - cloudbackupconfig_to_mqtt = json.dumps(cloudbackup.__dict__, default=lambda o: o.__dict__) - - publish.single( - "openWB/set/system/backup_cloud/config", cloudbackupconfig_to_mqtt, retain=True, hostname="localhost" - ) - - result["message"] = """Authorisierung gestartet, bitte den Link öffen, Code eingeben, - und Zugang authorisieren. Anschließend Zugangsberechtigung abrufen.""" - result["MessageType"] = MessageType.SUCCESS - - return result - - -def retrieveMSALTokens(cloudbackup: OneDriveBackupCloud) -> dict: - result = dict( - message="", - MessageType=MessageType.SUCCESS - ) - if cloudbackup is None: - result["message"] = """Es ist keine Backup-Cloud konfiguriert. - Bitte Konfiguration speichern und erneut versuchen.
""" - result["MessageType"] = MessageType.WARNING - return result - - # Create a public client application with msal - tokens = None - cache = msal.SerializableTokenCache() - app = PublicClientApplication(client_id=cloudbackup.configuration.clientID, - authority=cloudbackup.configuration.authority, token_cache=cache) - - f = cloudbackup.configuration.flow - if f is None: - result["message"] = """Es ist wurde kein Auth-Code erstellt. - Bitte zunächst Auth-Code erstellen und den Authorisierungsprozess beenden.
""" - result["MessageType"] = MessageType.WARNING - return result - flow = pickle.loads(bytes(f, encoding='latin1')) - - tokens = app.acquire_token_by_device_flow(flow) - # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow - # https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-device-code - # Check if the token was obtained successfully - if "access_token" in tokens: - log.debug("retrieved access token") - - # Tokens retrieved, remove auth codes as they are single use only. - cloudbackup.configuration.flow = None - cloudbackup.configuration.authcode = None - cloudbackup.configuration.authurl = None - - # save tokens - save_tokencache(config=cloudbackup.configuration, cache=cache.serialize()) - result["message"] = """Zugangsberechtigung erfolgreich abgerufen.""" - result["MessageType"] = MessageType.SUCCESS - return result - - else: - result["message"] = """"Es konnten keine Tokens abgerufen werden: - %s
%s""" % (tokens.get("error"), tokens.get("error_description")) - result["MessageType"] = MessageType.WARNING - '''pub_user_message(payload, connection_id, - "Es konnten keine Tokens abgerufen werden: %s
%s" - % (result.get("error"), result.get("error_description")), MessageType.WARNING - ) - ''' - return result diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py deleted file mode 100644 index 8f2a4d5f96..0000000000 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -import logging -import os -import pathlib - -from modules.backup_clouds.onedrive.msdrive.onedrive import OneDrive -from modules.backup_clouds.onedrive.api import get_tokens -from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration -from modules.common.abstract_device import DeviceDescriptor -from modules.common.configurable_backup_cloud import ConfigurableBackupCloud - - -log = logging.getLogger(__name__) - - -def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: - # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration - # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online - tokens = get_tokens(config) # type: ignore - log.debug("token object retrieved, access_token: %s", tokens.__len__) - log.debug("instantiate OneDrive connection") - onedrive = OneDrive(access_token=tokens["access_token"]) - - localbackup = os.path.join(pathlib.Path().resolve(), 'data', 'backup', backup_filename) - remote_filename = backup_filename.replace(':', '-') # file won't upload when name contains ':' - - if not config.backuppath.endswith("/"): - log.debug("fixing missing ending slash in backuppath: " + config.backuppath) - config.backuppath = config.backuppath + "/" - - log.debug("uploading file %s to OneDrive", backup_filename) - onedrive.upload_item(item_path=(config.backuppath+remote_filename), file_path=localbackup, - conflict_behavior="replace") - - -def create_backup_cloud(config: OneDriveBackupCloud): - def updater(backup_filename: str, backup_file: bytes): - upload_backup(config.configuration, backup_filename, backup_file) - return ConfigurableBackupCloud(config=config, component_updater=updater) - - -device_descriptor = DeviceDescriptor(configuration_factory=OneDriveBackupCloud) diff --git a/packages/modules/backup_clouds/onedrive/config.py b/packages/modules/backup_clouds/onedrive/config.py deleted file mode 100644 index c221cafe34..0000000000 --- a/packages/modules/backup_clouds/onedrive/config.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Optional - - -class OneDriveBackupCloudConfiguration: - def __init__(self, backuppath: str = "/openWB/Backup/", - persistent_tokencache: Optional[str] = None, - authurl: Optional[str] = None, - authcode: Optional[str] = None, - scope: Optional[list] = ["https://graph.microsoft.com/Files.ReadWrite"], - authority: Optional[str] = "https://login.microsoftonline.com/consumers/", - clientID: Optional[str] = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2", - flow: Optional[str] = None) -> None: - self.backuppath = backuppath - self.persistent_tokencache = persistent_tokencache - self.authurl = authurl - self.authcode = authcode - self.scope = scope - self.authority = authority - self.clientID = clientID - self.flow = flow - - -class OneDriveBackupCloud: - def __init__(self, - name: str = "OneDrive", - type: str = "onedrive", - configuration: OneDriveBackupCloudConfiguration = None) -> None: - self.name = name - self.type = type - self.configuration = configuration or OneDriveBackupCloudConfiguration() diff --git a/packages/modules/backup_clouds/onedrive/msdrive/constants.py b/packages/modules/backup_clouds/onedrive/msdrive/constants.py deleted file mode 100644 index 16bace4960..0000000000 --- a/packages/modules/backup_clouds/onedrive/msdrive/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -BASE_GRAPH_URL = "https://graph.microsoft.com/v1.0" -SIMPLE_UPLOAD_MAX_SIZE = 4000000 # 4MB -CHUNK_UPLOAD_MAX_SIZE = 3276800 # ~3MB must be divisible by 327680 bytes diff --git a/packages/modules/backup_clouds/onedrive/msdrive/drive.py b/packages/modules/backup_clouds/onedrive/msdrive/drive.py deleted file mode 100644 index e2b8addf22..0000000000 --- a/packages/modules/backup_clouds/onedrive/msdrive/drive.py +++ /dev/null @@ -1,208 +0,0 @@ -import os -from .exceptions import InvalidAccessToken, ItemNotFound, RateLimited, DriveException -from requests import Session -from abc import ABC, abstractmethod -from urllib3.util.retry import Retry -from requests.adapters import HTTPAdapter -from requests.exceptions import HTTPError -from .constants import SIMPLE_UPLOAD_MAX_SIZE, CHUNK_UPLOAD_MAX_SIZE - - -class MSDrive(ABC): - """Abstract class for accessing files stored in OneDrive and SharePoint using the Microsoft Graph API.""" - - def __init__(self, access_token: str) -> None: - """Class constructor that accepts a Microsoft access token for use with the API - - Args: - access_token (str): The access token - """ - self.access_token = access_token - - def get_item_data(self, **kwargs) -> dict: - """Get metadata for a DriveItem. - - Args: - drive_id (str): The drive ID (only for SharePoint) - item_id (str): [EITHER] The item ID - item_path (str): [EITHER] The item path - - Returns: - dict: JSON representation of a DriveItem resource - """ - r = self._session().get(self._get_drive_item_url(**kwargs)) - - return r.json() - - def list_items(self, **kwargs) -> dict: - """List the DriveItems in a specific folder path. - - Args: - drive_id (str): The drive ID (only for SharePoint) - folder_path (str): The folder path (or leave out for root) - - Returns: - dict: JSON representation of a collection of DriveItem resources - """ - r = self._session().get(self._get_drive_children_url(**kwargs)) - - return r.json() - - def download_item(self, **kwargs) -> None: - """Download a DriveItem file to a specific local path. - - Args: - drive_id (str): The drive ID (only for SharePoint) - item_id (str): [EITHER] The item ID - item_path (str): [EITHER] The item path - file_path (str): Local path to save the file to (e.g. /tmp/blah.csv) - """ - if not kwargs.get("file_path"): - raise ValueError("Missing file_path argument") - - data = self.get_item_data(**kwargs) - - with Session().get(data["@microsoft.graph.downloadUrl"], stream=True) as r: - r.raise_for_status() - - with open(kwargs["file_path"], "wb") as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) - - def upload_item(self, **kwargs) -> None: - """Upload a local file to an existing or new DriveItem. - - Specify the item_path for a new file. - Specify the item_path or item_id for an existing file. - - Args: - drive_id (str): The drive ID (only for SharePoint) - item_id (str): [EITHER] The item ID - item_path (str): [EITHER] The item path - file_path (str): Local path to upload the file from (e.g. /tmp/blah.csv) - """ - if not kwargs.get("file_path"): - raise ValueError("Missing file_path argument") - - file_size = os.stat(kwargs["file_path"]).st_size - - if file_size <= SIMPLE_UPLOAD_MAX_SIZE: - self._upload_item_small(**kwargs) - else: - self._upload_item_large(**kwargs) - - @abstractmethod - def _get_drive_item_url(self, **kwargs) -> str: - raise NotImplementedError("Must be overridden") - - @abstractmethod - def _get_drive_children_url(self, **kwargs) -> str: - raise NotImplementedError("Must be overridden") - - def _session(self) -> Session: - s = Session() - s.hooks["response"] = [self.raise_error_hook] - s.headers.update({"Authorization": "Bearer " + self.access_token}) - - return s - - def _session_upload(self) -> Session: - retries = Retry( - total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504] - ) - - adapter = HTTPAdapter(max_retries=retries) - - s = Session() - s.mount("http://", adapter) - s.mount("https://", adapter) - s.hooks["response"] = [self.raise_error_hook] - - return s - - def _upload_item_small(self, **kwargs) -> None: - url = self._get_drive_item_url(**kwargs) - file_data = open(kwargs["file_path"], "rb") - - if kwargs.get("item_id"): - url += "/content" - else: - url += ":/content" - - try: - self._session().put(url, data=file_data) - finally: - file_data.close() - - def _upload_item_large(self, **kwargs) -> None: - upload_url = self._get_upload_url(**kwargs) - file_size = os.stat(kwargs["file_path"]).st_size - - with open(kwargs["file_path"], "rb") as f: - chunk_size = CHUNK_UPLOAD_MAX_SIZE - chunk_number = file_size // chunk_size - chunk_leftover = file_size - chunk_size * chunk_number - chunk_data = f.read(chunk_size) - i = 0 - - while chunk_data: - start_index = i * chunk_size - end_index = start_index + chunk_size - - if i == chunk_number: - end_index = start_index + chunk_leftover - - s = self._session_upload() - - # Setting the header with the appropriate chunk data location in the file - headers = { - "Content-Length": str(chunk_size), - "Content-Range": "bytes {}-{}/{}".format( - start_index, end_index - 1, file_size - ), - } - - s.headers.update(headers) - s.put(upload_url, data=chunk_data) - - i = i + 1 - chunk_data = f.read(chunk_size) - - def _get_upload_url(self, **kwargs) -> str: - url = self._get_drive_item_url(**kwargs) - - if kwargs.get("item_id"): - url += "/createUploadSession" - else: - url += ":/createUploadSession" - - r = self._session().post(url) - - return r.json()["uploadUrl"] - - def raise_error_hook(self, resp, *args, **kwargs) -> None: - try: - resp.raise_for_status() - except HTTPError as err: - self._handle_http_error(err) - - def _handle_http_error(self, err: HTTPError) -> None: - if err.response is None: - raise err - - try: - body = err.response.json() - message = body["error"]["message"] - except Exception: - raise err - - if err.response.status_code == 401: - raise InvalidAccessToken(message) - - if err.response.status_code == 404: - raise ItemNotFound(message) - - if err.response.status_code == 429: - raise RateLimited(message) - - raise DriveException(message) diff --git a/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py b/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py deleted file mode 100644 index 8537d69d3d..0000000000 --- a/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py +++ /dev/null @@ -1,14 +0,0 @@ -class DriveException(Exception): - """There was an ambiguous exception that occurred""" - - -class InvalidAccessToken(DriveException): - """Invalid access token""" - - -class ItemNotFound(DriveException): - """Item not found""" - - -class RateLimited(DriveException): - """Rate limit exceeded""" diff --git a/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py b/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py deleted file mode 100644 index ff75a980c5..0000000000 --- a/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py +++ /dev/null @@ -1,30 +0,0 @@ -from .drive import MSDrive -from urllib.parse import quote -from .constants import BASE_GRAPH_URL - - -class OneDrive(MSDrive): - """Class for accessing DriveItems stored in OneDrive. - - A DriveItem resource represents a file, folder, or other item stored in a drive. - - All file system objects in OneDrive are returned as DriveItem resources (see https://bit.ly/3HAAxrh). - - """ - - def _get_drive_item_url(self, **kwargs) -> str: - if kwargs.get("item_id"): - return f"{BASE_GRAPH_URL}/me/drive/items/{kwargs['item_id']}" - - if kwargs.get("item_path"): - path = quote(kwargs["item_path"].lstrip("/")) - return f"{BASE_GRAPH_URL}/me/drive/root:/{path}" - - raise ValueError("Missing argument: item_id or item_path") - - def _get_drive_children_url(self, **kwargs) -> str: - if not kwargs.get("folder_path"): - return f"{BASE_GRAPH_URL}/me/drive/root/children" - else: - path = quote(kwargs["folder_path"].lstrip("/").rstrip("/")) - return f"{BASE_GRAPH_URL}/me/drive/root:/{path}:/children" diff --git a/requirements.txt b/requirements.txt index ccc649f713..2e658cfa68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,4 @@ pkce==1.0.3 # skodaconnect==1.3.4 evdev==1.5.0 #telnetlib3==2.0.2 -msal==1.22.0 python-dateutil==2.8.2