diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 384637a..9a490ee 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index a6399e3..6f54fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ +site/ +docker/actual-data/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..015eb5d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,19 @@ +# Read the Docs configuration file for MkDocs projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +mkdocs: + configuration: mkdocs.yml + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/README.md b/README.md index 9b04e08..5651584 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,15 @@ from actual import Actual from actual.queries import get_transactions with Actual( - base_url="http://localhost:5006", # Url of the Actual Server - password="", # Password for authentication - encryption_password=None, # Optional: Password for the file encryption. Will not use it if set to None. - file="", # Set the file to work with. Can be either the file id or file name, if name is unique - data_dir="", # Optional: Directory to store downloaded files. Will use a temporary if not provided - cert="" # Optional: Path to the certificate file to use for the connection, can also be set as False to disable SSL verification + base_url="http://localhost:5006", # Url of the Actual Server + password="", # Password for authentication + encryption_password=None, # Optional: Password for the file encryption. Will not use it if set to None. + # Set the file to work with. Can be either the file id or file name, if name is unique + file="", + # Optional: Directory to store downloaded files. Will use a temporary if not provided + data_dir="", + # Optional: Path to the certificate file to use for the connection, can also be set as False to disable SSL verification + cert="" ) as actual: transactions = get_transactions(actual.session) for t in transactions: @@ -56,7 +59,7 @@ The `file` will be matched to either one of the following: - The ID of the budget, a UUID that is only available if you inspect the result of the method `list_user_files` - The Sync ID of the budget, a UUID available on the frontend on the "Advanced options" - If none of those options work for you, you can search for the file manually with `list_user_files` and provide the -object directly: + object directly: ```python from actual import Actual @@ -66,150 +69,7 @@ with Actual("http://localhost:5006", password="mypass") as actual: actual.download_budget() ``` -## Adding new transactions - -After you created your first budget (or when updating an existing budget), you can add new transactions by adding them -using the `actual.session.add()` method. You cannot use the SQLAlchemy session directly because that adds the entries to your -local database, but will not sync the results back to the server (that is only possible when re-uploading the file). - -The method will make sure the local database is updated, but will also send a SYNC request with the added data so that -it will be immediately available on the frontend: - -```python -import decimal -import datetime -from actual import Actual -from actual.queries import create_transaction, create_account - -with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - act = create_account(actual.session, "My account") - t = create_transaction( - actual.session, - datetime.date.today(), - act, - "My payee", - notes="My first transaction", - amount=decimal.Decimal(-10.5), - ) - actual.commit() # use the actual.commit() instead of session.commit()! -``` - -Will produce: - -![added-transaction](https://github.com/bvanelli/actualpy/blob/main/docs/static/added-transaction.png?raw=true) - -## Updating existing transactions - -You may also update transactions using the SQLModel directly, you just need to make sure to commit the results at the -end: - -```python -from actual import Actual -from actual.queries import get_transactions - - -with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - for transaction in get_transactions(actual.session): - # change the transactions notes - if transaction.notes is not None and "my pattern" in transaction.notes: - transaction.notes = transaction.notes + " my suffix!" - # commit your changes! - actual.commit() - -``` - -> [!IMPORTANT] -> You can also modify the relationships, for example the `transaction.payee.name`, but you to be aware that -> this payee might be used for more than one transaction. Whenever the relationship is anything but 1:1, you have to -> track the changes already done to prevent modifying a field twice. - -## Generating backups - -You can use actualpy to generate regular backups of your server files. Here is a script that will backup your server -file on the current folder: - -```python -from actual import Actual -from datetime import datetime - -with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - current_date = datetime.now().strftime("%Y%m%d-%H%M") - actual.export_data(f"actual_backup_{current_date}.zip") -``` - -# Experimental features - -> [!CAUTION] -> Experimental features do not have all the testing necessary to ensure correctness in comparison to the -> files generated by the Javascript API. This means that this operations could in theory corrupt the data. Make sure -> you have backups of your data before trying any of those operations. - -## Bootstraping a new server and uploading a first file - -The following script would generate a new empty budget on the Actual server, even if the server was not bootstrapped -with an initial password. - -```python -from actual import Actual - -with Actual(base_url="http://localhost:5006", password="mypass", bootstrap=True) as actual: - actual.create_budget("My budget") - actual.upload_budget() -``` - -You will then have a freshly created new budget to use: - -![created-budget](https://github.com/bvanelli/actualpy/blob/main/docs/static/new-budget.png?raw=true) - -If the `encryption_password` is set, the budget will additionally also be encrypted on the upload step to the server. - -## Updating transactions using Bank Sync - -If you have either [goCardless](https://actualbudget.org/docs/advanced/bank-sync/#gocardless-setup) or -[simplefin](https://actualbudget.org/docs/experimental/simplefin-sync/) integration configured, it is possible to -update the transactions using just the Python API alone. This is because the actual queries to the third-party service -are handled on the server, so the client does not have to do any custom API queries. - -To sync your account, simply call the `run_bank_sync` method: - -```python -from actual import Actual - -with Actual(base_url="http://localhost:5006", password="mypass") as actual: - synchronized_transactions = actual.run_bank_sync() - for transaction in synchronized_transactions: - print(f"Added of modified {transaction}") - # sync changes back to the server - actual.commit() -``` - -## Running rules - -You can also automatically run rules using the library: - -```python -from actual import Actual - -with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - actual.run_rules() - # sync changes back to the server - actual.commit() -``` - -You can also manipulate the rules individually: - -```python -from actual import Actual -from actual.queries import get_ruleset, get_transactions - -with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: - rs = get_ruleset(actual.session) - transactions = get_transactions(actual.session) - for rule in rs: - for t in transactions: - if rule.evaluate(t): - print(f"Rule {rule} matches for {t}") -``` +Checkout [the full documentation](https://actualpy.readthedocs.io) for more examples. # Understanding how Actual handles changes @@ -225,11 +85,12 @@ change, done locally, a SYNC request is sent to the server with a list of the fo - `row`: the row identifier for the entry that was added/update. This would be the primary key of the row (a uuid value) - `column`: the column that had the value changed - `value`: the new value. Since it's a string, the values are either prefixed by `S:` to denote a string, `N:` to denote -a numeric value and `0:` to denote a null value. + a numeric value and `0:` to denote a null value. All individual column changes are computed on an insert, serialized with protobuf and sent to the server to be stored. Null values and server defaults are not required to be present in the SYNC message, unless a column is changed to null. -If the file is encrypted, the protobuf content will also be encrypted, so that the server does not know what was changed. +If the file is encrypted, the protobuf content will also be encrypted, so that the server does not know what was +changed. New clients can use this individual changes to then sync their local copies and add the changes executed on other users. Whenever a SYNC request is done, the response will also contain changes that might have been done in other browsers, so @@ -237,17 +98,43 @@ that the user the retrieve the information and update its local copy. But this also means that new users need to download a long list of changes, possibly making the initialization slow. Thankfully, user is also allowed to reset the sync. When doing a reset of the file via frontend, the browser is then -resetting the file completely and clearing the list of changes. This would make sure all changes are actually stored in the +resetting the file completely and clearing the list of changes. This would make sure all changes are actually stored in +the database. This is done on the frontend under *Settings > Reset sync*, and causes the current file to be reset (removed from the server) and re-uploaded again, with all changes already in place. This means that, when using this library to operate changes on the database, you have to make sure that either: - do a sync request is made using the `actual.commit()` method. This only handles pending operations that haven't yet -been committed, generates a change list with them and posts them on the sync endpoint. + been committed, generates a change list with them and posts them on the sync endpoint. - do a full re-upload of the database is done. # Contributing The goal is to have more features implemented and tested on the Actual API. If you have ideas, comments, bug fixes or requests feel free to open an issue or submit a pull request. + +To install requirements, install both requirements files: + +```bash +# optionally setup a venv (recommended) +python3 -m venv venv && source venv/bin/activate +# install requirements +pip install -r requirements.txt +pip install -r requirements-dev.txt +``` + +We use [`pre-commit`](https://pre-commit.com/) to ensure consistent formatting across different developers. To develop +locally, make sure you install all development requirements, then install `pre-commit` hooks. This would make sure the +formatting runs on every commit. + +``` +pre-commit install +``` + +To run tests, make sure you have docker installed ([how to install docker](https://docs.docker.com/engine/install/)). +Run the tests on your machine: + +```bash +pytest +``` diff --git a/actual/__init__.py b/actual/__init__.py index d14fc1f..c9fa1dc 100644 --- a/actual/__init__.py +++ b/actual/__init__.py @@ -11,13 +11,13 @@ import warnings import zipfile from os import PathLike -from typing import IO, Union +from typing import IO, List, Union from sqlalchemy import insert, update from sqlmodel import MetaData, Session, create_engine, select from actual.api import ActualServer -from actual.api.models import RemoteFileListDTO +from actual.api.models import BankSyncErrorDTO, RemoteFileListDTO from actual.crypto import create_key_buffer, decrypt_from_meta, encrypt, make_salt from actual.database import ( Accounts, @@ -28,7 +28,12 @@ reflect_model, strong_reference_session, ) -from actual.exceptions import ActualError, InvalidZipFile, UnknownFileId +from actual.exceptions import ( + ActualBankSyncError, + ActualError, + InvalidZipFile, + UnknownFileId, +) from actual.migrations import js_migration_statements from actual.protobuf_models import HULC_Client, Message, SyncRequest from actual.queries import ( @@ -50,7 +55,7 @@ def __init__( file: str = None, encryption_password: str = None, data_dir: Union[str, pathlib.Path] = None, - cert: str | bool = False, + cert: str | bool = None, bootstrap: bool = False, sa_kwargs: dict = None, ): @@ -58,8 +63,8 @@ def __init__( Implements the Python API for the Actual Server in order to be able to read and modify information on Actual books using Python. - Parts of the implementation are available at the following file: - https://github.com/actualbudget/actual/blob/2178da0414958064337b2c53efc95ff1d3abf98a/packages/loot-core/src/server/cloud-storage.ts + Parts of the implementation are [available at the following file.]( + https://github.com/actualbudget/actual/blob/2178da0414958064337b2c53efc95ff1d3abf98a/packages/loot-core/src/server/cloud-storage.ts) :param base_url: url of the running Actual server :param token: the token for authentication, if this is available (optional) @@ -68,10 +73,12 @@ def __init__( :param encryption_password: password used to configure encryption, if existing :param data_dir: where to store the downloaded files from the server. If not specified, a temporary folder will be created instead. + :param cert: if a custom certificate should be used (i.e. self-signed certificate), it's path can be provided + as a string. Set to `False` for no certificate check. :param bootstrap: if the server is not bootstrapped, bootstrap it with the password. :param sa_kwargs: additional kwargs passed to the SQLAlchemy session maker. Examples are `autoflush` (enabled - by default), `autocommit` (disabled by default). For a list of all parameters, check the SQLAlchemy - documentation: https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.__init__ + by default), `autocommit` (disabled by default). For a list of all parameters, check the [SQLAlchemy + documentation.](https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.__init__) """ super().__init__(base_url, token, password, bootstrap, cert) self._file: RemoteFileListDTO | None = None @@ -104,7 +111,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): @property def session(self) -> Session: if not self._session: - raise ActualError("No session defined. Use `with Actual() as actual:` construct to generate one.") + raise ActualError( + "No session defined. Use `with Actual() as actual:` construct to generate one.\n" + "If you are already using the context manager, try setting a file to use the session." + ) return self._session def set_file(self, file_id: Union[str, RemoteFileListDTO]) -> RemoteFileListDTO: @@ -128,7 +138,7 @@ def set_file(self, file_id: Union[str, RemoteFileListDTO]) -> RemoteFileListDTO: raise UnknownFileId(f"Multiple files found with identifier '{file_id}'") return self.set_file(selected_files[0]) - def run_migrations(self, migration_files: list[str]): + def run_migrations(self, migration_files: List[str]): """Runs the migration files, skipping the ones that have already been run. The files can be retrieved from .data_file_index() method. This first file is the base database, and the following files are migrations. Migrations can also be .js files. In this case, we have to extract and execute queries from the standard JS.""" @@ -149,6 +159,8 @@ def run_migrations(self, migration_files: list[str]): conn.execute(f"INSERT INTO __migrations__ (id) VALUES ({file_id});") conn.commit() conn.close() + # update the metadata by reflecting the model + self._meta = reflect_model(self.engine) def create_budget(self, budget_name: str): """Creates a budget using the remote server default database and migrations. If password is provided, the @@ -176,23 +188,23 @@ def create_budget(self, budget_name: str): } ) self._file = RemoteFileListDTO(name=budget_name, fileId=file_id, groupId=None, deleted=0, encryptKeyId=None) - # create engine for downloaded database and run migrations - self.run_migrations(migration_files[1:]) # generate a session self.engine = create_engine(f"sqlite:///{self._data_dir}/db.sqlite") + # create engine for downloaded database and run migrations + self.run_migrations(migration_files[1:]) if self._in_context: self._session = strong_reference_session(Session(self.engine, **self._sa_kwargs)) - # reflect the session - self._meta = reflect_model(self.engine) # create a clock self.load_clock() def rename_budget(self, budget_name: str): + """Renames the budget with the given name.""" if not self._file: raise UnknownFileId("No current file loaded.") self.update_user_file_name(self._file.file_id, budget_name) def delete_budget(self): + """Deletes the currently loaded file from the server.""" if not self._file: raise UnknownFileId("No current file loaded.") self.delete_user_file(self._file.file_id) @@ -241,7 +253,9 @@ def encrypt(self, encryption_password: str): self.set_file(self._file.file_id) def upload_budget(self): - """Uploads the current file to the Actual server.""" + """Uploads the current file to the Actual server. If attempting to upload your first budget, make sure you use + [actual.Actual.create_budget][] first. + """ if not self._data_dir: raise UnknownFileId("No current file loaded.") if not self._file: @@ -262,10 +276,13 @@ def upload_budget(self): self.encrypt(self._encryption_password) def reupload_budget(self): + """Similar to the reset sync option from the frontend, resets the user file on the backend and reuploads the + current copy instead. **This operation can be destructive**, so make sure you generate a copy before + attempting to reupload your budget.""" self.reset_user_file(self._file.file_id) self.upload_budget() - def apply_changes(self, messages: list[Message]): + def apply_changes(self, messages: List[Message]): """Applies a list of sync changes, based on what the sync method returned on the remote.""" if not self.engine: raise UnknownFileId("No valid file available, download one with download_budget()") @@ -304,7 +321,8 @@ def update_metadata(self, patch: dict): then be merged on the metadata and written again to a file.""" metadata_file = self._data_dir / "metadata.json" if metadata_file.is_file(): - config = self.get_metadata() | patch + config = self.get_metadata() + config.update(patch) else: config = patch metadata_file.write_text(json.dumps(config, separators=(",", ":"))) @@ -340,6 +358,8 @@ def download_budget(self, encryption_password: str = None): self._session = strong_reference_session(Session(self.engine, **self._sa_kwargs)) def import_zip(self, file_bytes: str | PathLike[str] | IO[bytes]): + """Imports a zip file as the current database, as well as generating the local reflected session. Enables you + to inspect backups by loading them directly, instead of unzipping the contents.""" try: zip_file = zipfile.ZipFile(file_bytes) except zipfile.BadZipfile as e: @@ -354,6 +374,9 @@ def import_zip(self, file_bytes: str | PathLike[str] | IO[bytes]): self.load_clock() def sync(self): + """Does a sync request and applies all changes that are stored on the server on the local copy of the database. + Since all changes are retrieved, this function cannot be used for partial changes (since the budget is online). + """ # after downloading the budget, some pending transactions still need to be retrieved using sync request = SyncRequest( { @@ -370,8 +393,9 @@ def sync(self): self._client = HULC_Client.from_timestamp(changes.messages[-1].timestamp) def load_clock(self) -> MessagesClock: - """See implementation at: - https://github.com/actualbudget/actual/blob/5bcfc71be67c6e7b7c8b444e4c4f60da9ea9fdaa/packages/loot-core/src/server/db/index.ts#L81-L98 + """Loads the HULC Clock from the database. This clock tells the server from when the messages should be + retrieved. See the [original implementation.]( + https://github.com/actualbudget/actual/blob/5bcfc71be67c6e7b7c8b444e4c4f60da9ea9fdaa/packages/loot-core/src/server/db/index.ts#L81-L98) """ with Session(self.engine) as session: clock = session.exec(select(MessagesClock)).one_or_none() @@ -411,17 +435,24 @@ def commit(self): self.sync_sync(req) def run_rules(self): + """Runs all the stored rules on the database on all transactions, without any filters.""" ruleset = get_ruleset(self.session) transactions = get_transactions(self.session, is_parent=True) ruleset.run(transactions) - def _run_bank_sync_account(self, acct: Accounts, start_date: datetime.date) -> list[Transactions]: + def _run_bank_sync_account(self, acct: Accounts, start_date: datetime.date) -> List[Transactions]: sync_method = acct.account_sync_source account_id = acct.account_id requisition_id = acct.bank.bank_id if sync_method == "goCardless" else None new_transactions_data = self.bank_sync_transactions( sync_method.lower(), account_id, start_date, requisition_id=requisition_id ) + if isinstance(new_transactions_data, BankSyncErrorDTO): + raise ActualBankSyncError( + new_transactions_data.data.error_type, + new_transactions_data.data.status, + new_transactions_data.data.reason, + ) new_transactions = new_transactions_data.data.transactions.all imported_transactions = [] for transaction in new_transactions: @@ -444,7 +475,7 @@ def _run_bank_sync_account(self, acct: Accounts, start_date: datetime.date) -> l def run_bank_sync( self, account: str | Accounts | None = None, start_date: datetime.date | None = None - ) -> list[Transactions]: + ) -> List[Transactions]: """ Runs the bank synchronization for the selected account. If missing, all accounts are synchronized. If a start_date is provided, is used as a reference, otherwise, the last timestamp of each account will be used. If diff --git a/actual/api/__init__.py b/actual/api/__init__.py index 790974d..9f03605 100644 --- a/actual/api/__init__.py +++ b/actual/api/__init__.py @@ -8,8 +8,8 @@ from actual.api.models import ( BankSyncAccountResponseDTO, + BankSyncResponseDTO, BankSyncStatusDTO, - BankSyncTransactionResponseDTO, BootstrapInfoDTO, Endpoints, GetUserFileInfoDTO, @@ -37,8 +37,20 @@ def __init__( token: str = None, password: str = None, bootstrap: bool = False, - cert: str | bool = False, + cert: str | bool = None, ): + """ + Implements the low-level API for interacting with the Actual server by just implementing the API calls and + response models. + + :param base_url: url of the running Actual server + :param token: the token for authentication, if this is available (optional) + :param password: the password for authentication. It will be used on the .login() method to retrieve the token. + be created instead. + :param bootstrap: if the server is not bootstrapped, bootstrap it with the password. + :param cert: if a custom certificate should be used (i.e. self-signed certificate), it's path can be provided + as a string. Set to `False` for no certificate check. + """ self.api_url = base_url self._token = token self.cert = cert @@ -58,8 +70,8 @@ def login(self, password: str, method: Literal["password", "header"] = "password authenticate the user. :param password: password of the Actual server. - :param method: the method used to authenticate with the server. Check - https://actualbudget.org/docs/advanced/http-header-auth/ for information. + :param method: the method used to authenticate with the server. Check the [official auth header documentation]( + https://actualbudget.org/docs/advanced/http-header-auth/) for information. """ if not password: raise AuthorizationError("Trying to login but not password was provided.") @@ -96,7 +108,7 @@ def headers(self, file_id: str = None, extra_headers: dict = None) -> dict: if file_id: headers["X-ACTUAL-FILE-ID"] = file_id if extra_headers: - headers = headers | extra_headers + headers.update(extra_headers) return headers def info(self) -> InfoDTO: @@ -284,7 +296,7 @@ def bank_sync_transactions( account_id: str, start_date: datetime.date, requisition_id: str = None, - ) -> BankSyncTransactionResponseDTO: + ) -> BankSyncResponseDTO: if bank_sync == "gocardless" and requisition_id is None: raise ActualInvalidOperationError("Retrieving transactions with goCardless requires `requisition_id`") endpoint = Endpoints.BANK_SYNC_TRANSACTIONS.value.format(bank_sync=bank_sync) @@ -292,4 +304,4 @@ def bank_sync_transactions( if requisition_id: payload["requisitionId"] = requisition_id response = requests.post(f"{self.api_url}/{endpoint}", headers=self.headers(), json=payload, verify=self.cert) - return BankSyncTransactionResponseDTO.model_validate(response.json()) + return BankSyncResponseDTO.validate_python(response.json()) diff --git a/actual/api/bank_sync.py b/actual/api/bank_sync.py index c8dd600..c6146ec 100644 --- a/actual/api/bank_sync.py +++ b/actual/api/bank_sync.py @@ -76,7 +76,7 @@ class TransactionItem(BaseModel): ) date: datetime.date remittance_information_unstructured: str = Field(None, alias="remittanceInformationUnstructured") - remittance_information_unstructured_array: list[str] = Field( + remittance_information_unstructured_array: List[str] = Field( default_factory=list, alias="remittanceInformationUnstructuredArray" ) additional_information: Optional[str] = Field(None, alias="additionalInformation") @@ -117,3 +117,10 @@ class BankSyncTransactionData(BaseModel): # goCardless specific iban: Optional[str] = None institution_id: Optional[str] = Field(None, alias="institutionId") + + +class BankSyncErrorData(BaseModel): + error_type: str + error_code: str + status: Optional[str] = None + reason: Optional[str] = None diff --git a/actual/api/models.py b/actual/api/models.py index d7d4227..8be9ebd 100644 --- a/actual/api/models.py +++ b/actual/api/models.py @@ -1,11 +1,15 @@ from __future__ import annotations import enum -from typing import List, Optional +from typing import List, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, TypeAdapter -from actual.api.bank_sync import BankSyncAccountData, BankSyncTransactionData +from actual.api.bank_sync import ( + BankSyncAccountData, + BankSyncErrorData, + BankSyncTransactionData, +) class Endpoints(enum.Enum): @@ -158,3 +162,10 @@ class BankSyncAccountResponseDTO(StatusDTO): class BankSyncTransactionResponseDTO(StatusDTO): data: BankSyncTransactionData + + +class BankSyncErrorDTO(StatusDTO): + data: BankSyncErrorData + + +BankSyncResponseDTO = TypeAdapter(Union[BankSyncErrorDTO, BankSyncTransactionResponseDTO]) diff --git a/actual/crypto.py b/actual/crypto.py index 65d1d01..bb0b332 100644 --- a/actual/crypto.py +++ b/actual/crypto.py @@ -71,25 +71,18 @@ def make_test_message(key_id: str, key: bytes) -> dict: def is_uuid(text: str, version: int = 4): """ - Check if uuid_to_test is a valid UUID. + Check if uuid_to_test is a valid UUID. Taken from [this thread](https://stackoverflow.com/a/54254115/12681470) - Taken from https://stackoverflow.com/a/54254115/12681470 + Examples: - Parameters - ---------- - uuid_to_test : str - version : {1, 2, 3, 4} - - Returns - ------- - `True` if uuid_to_test is a valid UUID, otherwise `False`. - - Examples - -------- >>> is_uuid('c9bf9e57-1685-4c89-bafb-ff5af830be8a') True >>> is_uuid('c9bf9e58') False + + :param text: UUID string to test + :param version: expected version for the UUID + :return: `True` if `text` is a valid UUID, otherwise `False`. """ try: uuid.UUID(str(text), version=version) diff --git a/actual/database.py b/actual/database.py index 07cd232..5a315cc 100644 --- a/actual/database.py +++ b/actual/database.py @@ -2,9 +2,13 @@ This file was partially generated using sqlacodegen using the downloaded version of the db.sqlite file export in order to update this file, you can generate the code with: -> sqlacodegen --generator sqlmodels sqlite:///db.sqlite +```bash +sqlacodegen --generator sqlmodels sqlite:///db.sqlite +``` -and patch the necessary models by merging the results. +and patch the necessary models by merging the results. The [actual.database.BaseModel][] defines all models that can +be updated from the user, and must contain a unique `id`. Those models can then be converted automatically into a +protobuf change message using [actual.database.BaseModel.convert][]. """ import datetime @@ -35,18 +39,20 @@ """ This variable contains the internal model mappings for all databases. It solves a couple of issues, namely having the -mapping from __tablename__ to the actual SQLAlchemy class, and later mapping the SQL column into the Pydantic field, +mapping from `__tablename__` to the actual SQLAlchemy class, and later mapping the SQL column into the Pydantic field, which could be different and follows the Python naming convention. An example is the field `Transactions.is_parent`, that converts into the SQL equivalent `transactions.isParent`. In this case, we would have the following entries: - __TABLE_COLUMNS_MAP__ = { - "transactions": { - "entity": , - "columns": { - "isParent": "is_parent" - } +``` +__TABLE_COLUMNS_MAP__ = { + "transactions": { + "entity": , + "columns": { + "isParent": "is_parent" } } +} +``` """ __TABLE_COLUMNS_MAP__ = dict() @@ -61,7 +67,7 @@ def reflect_model(eng: engine.Engine) -> MetaData: def get_class_from_reflected_table_name(metadata: MetaData, table_name: str) -> Union[Table, None]: """ Returns, based on the defined tables on the reflected model the corresponding SQLAlchemy table. - If not found, returns None. + If not found, returns `None`. """ return metadata.tables.get(table_name, None) @@ -70,7 +76,7 @@ def get_attribute_from_reflected_table_name( metadata: MetaData, table_name: str, column_name: str ) -> Union[Column, None]: """ - Returns, based, on the defined reflected model the corresponding and the SAColumn. If not found, returns None. + Returns, based, on the defined reflected model the corresponding and the SAColumn. If not found, returns `None`. """ table = get_class_from_reflected_table_name(metadata, table_name) return table.columns.get(column_name, None) @@ -78,16 +84,17 @@ def get_attribute_from_reflected_table_name( def get_class_by_table_name(table_name: str) -> Union[SQLModel, None]: """ - Returns, based on the defined tables __tablename__ the corresponding SQLModel object. If not found, returns None. + Returns, based on the defined tables `__tablename__` the corresponding SQLModel object. If not found, returns + `None`. """ return __TABLE_COLUMNS_MAP__.get(table_name, {}).get("entity", None) def get_attribute_by_table_name(table_name: str, column_name: str, reverse: bool = False) -> Union[str, None]: """ - Returns, based, on the defined tables __tablename__ and the SAColumn name, the correct pydantic attribute. Search + Returns, based, on the defined tables `__tablename__` and the SAColumn name, the correct pydantic attribute. Search can be reversed by setting the `reverse` flag to `True`. - If not found, returns None. + If not found, returns `None`. :param table_name: SQL table name. :param column_name: SQL column name. @@ -135,9 +142,8 @@ class BaseModel(SQLModel): id: str = Field(sa_column=Column("id", Text, primary_key=True)) def convert(self, is_new: bool = True) -> List[Message]: - """Convert the object into distinct entries for sync method. Based on the original implementation: - - https://github.com/actualbudget/actual/blob/98c17bd5e0f13e27a09a7f6ac176510530572be7/packages/loot-core/src/server/aql/schema-helpers.ts#L146 + """Convert the object into distinct entries for sync method. Based on the [original implementation]( + https://github.com/actualbudget/actual/blob/98c17bd5e0f13e27a09a7f6ac176510530572be7/packages/loot-core/src/server/aql/schema-helpers.ts#L146) """ row = getattr(self, "id", None) # also helps lazy loading the instance if row is None: @@ -157,7 +163,7 @@ def convert(self, is_new: bool = True) -> List[Message]: changes.append(m) return changes - def changed(self) -> list[str]: + def changed(self) -> List[str]: """Returns list of model changed attributes.""" changed_attributes = [] inspr = inspect(self) @@ -594,7 +600,7 @@ def set_date(self, date: datetime.date): self.date = int(datetime.date.strftime(date, "%Y%m%d")) def set_amount(self, amount: Union[decimal.Decimal, int, float]): - self.amount = int(amount * 100) + self.amount = int(round(amount * 100)) def get_amount(self) -> decimal.Decimal: return decimal.Decimal(self.amount) / decimal.Decimal(100) diff --git a/actual/exceptions.py b/actual/exceptions.py index 55d5aa8..fbea0ea 100644 --- a/actual/exceptions.py +++ b/actual/exceptions.py @@ -50,3 +50,8 @@ class ActualDecryptionError(ActualError): class ActualSplitTransactionError(ActualError): pass + + +class ActualBankSyncError(ActualError): + def __init__(self, error_type: str, status: str = None, reason: str = None): + self.error_type, self.status, self.reason = error_type, status, reason diff --git a/actual/migrations.py b/actual/migrations.py index c280583..b6b0875 100644 --- a/actual/migrations.py +++ b/actual/migrations.py @@ -1,9 +1,10 @@ import re import uuid import warnings +from typing import List -def js_migration_statements(js_file: str) -> list[str]: +def js_migration_statements(js_file: str) -> List[str]: queries = [] matches = re.finditer(r"db\.(execQuery|runQuery)", js_file) for match in matches: diff --git a/actual/protobuf_models.py b/actual/protobuf_models.py index 7c13423..1186368 100644 --- a/actual/protobuf_models.py +++ b/actual/protobuf_models.py @@ -3,6 +3,7 @@ import base64 import datetime import uuid +from typing import List import proto @@ -10,13 +11,11 @@ from actual.exceptions import ActualDecryptionError """ -Protobuf message definitions taken from: +Protobuf message definitions taken from the [sync.proto file]( +https://github.com/actualbudget/actual/blob/029e2f09bf6caf386523bbfa944ab845271a3932/packages/crdt/src/proto/sync.proto). -https://github.com/actualbudget/actual/blob/029e2f09bf6caf386523bbfa944ab845271a3932/packages/crdt/src/proto/sync.proto - -They should represent how the server take requests from the client. The server side implementation is available here: - -https://github.com/actualbudget/actual-server/blob/master/src/app-sync.js#L32 +They should represent how the server take requests from the client. The server side implementation is available [here]( +https://github.com/actualbudget/actual-server/blob/master/src/app-sync.js#L32). """ @@ -33,11 +32,13 @@ def from_timestamp(cls, ts: str) -> HULC_Client: def timestamp(self, now: datetime.datetime = None) -> str: """Actual uses Hybrid Unique Logical Clock (HULC) timestamp generator. - Timestamps serialize into a 46-character collatable string - * example: 2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF - * example: 2015-04-24T22:23:42.123Z-1000-A219E7A71CC18912 + Timestamps serialize into a 46-character collatable string. Examples: + + - `2015-04-24T22:23:42.123Z-1000-0123456789ABCDEF` + - `2015-04-24T22:23:42.123Z-1000-A219E7A71CC18912` - See https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts + See [original source code]( + https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts) for reference. """ if not now: @@ -47,9 +48,8 @@ def timestamp(self, now: datetime.datetime = None) -> str: return f"{now.isoformat(timespec='milliseconds')}Z-{count}-{self.client_id}" def get_client_id(self): - """Creates a client id for the HULC request. Copied implementation from: - - https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts#L80 + """Creates a client id for the HULC request. Implementation copied [from the source code]( + https://github.com/actualbudget/actual/blob/a9362cc6f9b974140a760ad05816cac51c849769/packages/crdt/src/crdt/timestamp.ts#L80) """ return ( self.client_id if getattr(self, "client_id", None) is not None else str(uuid.uuid4()).replace("-", "")[-16:] @@ -69,9 +69,8 @@ class Message(proto.Message): value = proto.Field(proto.STRING, number=4) def get_value(self) -> str | int | float | None: - """Serialization types from Actual. Source: - - https://github.com/actualbudget/actual/blob/998efb9447da6f8ce97956cbe83d6e8a3c18cf53/packages/loot-core/src/server/sync/index.ts#L154-L160 + """Serialization types from Actual. [Original source code]( + https://github.com/actualbudget/actual/blob/998efb9447da6f8ce97956cbe83d6e8a3c18cf53/packages/loot-core/src/server/sync/index.ts#L154-L160) """ datatype, _, value = self.value.partition(":") if datatype == "S": @@ -120,7 +119,7 @@ def set_timestamp(self, client_id: str = None, now: datetime.datetime = None) -> def set_null_timestamp(self, client_id: str = None) -> str: return self.set_timestamp(client_id, datetime.datetime(1970, 1, 1, 0, 0, 0, 0)) - def set_messages(self, messages: list[Message], client: HULC_Client, master_key: bytes = None): + def set_messages(self, messages: List[Message], client: HULC_Client, master_key: bytes = None): if not self.messages: self.messages = [] for message in messages: @@ -146,7 +145,7 @@ class SyncResponse(proto.Message): messages = proto.RepeatedField(MessageEnvelope, number=1) merkle = proto.Field(proto.STRING, number=2) - def get_messages(self, master_key: bytes = None) -> list[Message]: + def get_messages(self, master_key: bytes = None) -> List[Message]: messages = [] for message in self.messages: # noqa if message.isEncrypted: diff --git a/actual/queries.py b/actual/queries.py index d19efe3..da05bd9 100644 --- a/actual/queries.py +++ b/actual/queries.py @@ -107,7 +107,7 @@ def match_transaction( payee: str | Payees = "", amount: decimal.Decimal | float | int = 0, imported_id: str | None = None, - already_matched: list[Transactions] = None, + already_matched: typing.List[Transactions] = None, ) -> typing.Optional[Transactions]: """Matches a transaction with another transaction based on the fuzzy matching described at `reconcileTransactions`: @@ -129,8 +129,8 @@ def match_transaction( # if not matched, look 7 days ahead and 7 days back when fuzzy matching query = _transactions_base_query( s, date - datetime.timedelta(days=7), date + datetime.timedelta(days=8), account=account - ).filter(Transactions.amount == amount * 100) - results: list[Transactions] = s.exec(query).all() # noqa + ).filter(Transactions.amount == round(amount * 100)) + results: typing.List[Transactions] = s.exec(query).all() # noqa # filter out the ones that were already matched if already_matched: matched = {t.id for t in already_matched} @@ -175,7 +175,7 @@ def create_transaction_from_ids( id=str(uuid.uuid4()), acct=account_id, date=date_int, - amount=int(amount * 100), + amount=int(round(amount * 100)), category_id=category_id, payee_id=payee_id, notes=notes, @@ -224,11 +224,16 @@ def create_transaction( acct = get_account(s, account) if acct is None: raise ActualError(f"Account {account} not found") + if imported_payee: + imported_payee = imported_payee.strip() + if not payee: + payee = imported_payee payee = get_or_create_payee(s, payee) if category: category_id = get_or_create_category(s, category).id else: category_id = None + return create_transaction_from_ids( s, date, acct.id, payee.id, notes, category_id, amount, imported_id, cleared, imported_payee ) @@ -269,7 +274,7 @@ def reconcile_transaction( cleared: bool = False, imported_payee: str = None, update_existing: bool = True, - already_matched: list[Transactions] = None, + already_matched: typing.List[Transactions] = None, ) -> Transactions: """Matches the transaction to an existing transaction using fuzzy matching. @@ -601,8 +606,8 @@ def get_ruleset(s: Session) -> RuleSet: """ rule_set = list() for rule in get_rules(s): - conditions = TypeAdapter(list[Condition]).validate_json(rule.conditions) - actions = TypeAdapter(list[Action]).validate_json(rule.actions) + conditions = TypeAdapter(typing.List[Condition]).validate_json(rule.conditions) + actions = TypeAdapter(typing.List[Action]).validate_json(rule.actions) rs = Rule(conditions=conditions, operation=rule.conditions_op, actions=actions, stage=rule.stage) # noqa rule_set.append(rs) return RuleSet(rules=rule_set) diff --git a/actual/rules.py b/actual/rules.py index 231b84a..de9dde5 100644 --- a/actual/rules.py +++ b/actual/rules.py @@ -40,12 +40,15 @@ class ConditionType(enum.Enum): NOT_ONE_OF = "notOneOf" IS_BETWEEN = "isbetween" MATCHES = "matches" + HAS_TAGS = "hasTags" class ActionType(enum.Enum): SET = "set" SET_SPLIT_AMOUNT = "set-split-amount" LINK_SCHEDULE = "link-schedule" + PREPEND_NOTES = "prepend-notes" + APPEND_NOTES = "append-notes" class BetweenValue(pydantic.BaseModel): @@ -74,6 +77,7 @@ class ValueType(enum.Enum): STRING = "string" NUMBER = "number" BOOLEAN = "boolean" + IMPORTED_PAYEE = "imported_payee" def is_valid(self, operation: ConditionType) -> bool: """Returns if a conditional operation for a certain type is valid. For example, if the value is of type string, @@ -81,8 +85,17 @@ def is_valid(self, operation: ConditionType) -> bool: greater than defined for strings.""" if self == ValueType.DATE: return operation.value in ("is", "isapprox", "gt", "gte", "lt", "lte") - elif self == ValueType.STRING: - return operation.value in ("is", "contains", "oneOf", "isNot", "doesNotContain", "notOneOf", "matches") + elif self in (ValueType.STRING, ValueType.IMPORTED_PAYEE): + return operation.value in ( + "is", + "contains", + "oneOf", + "isNot", + "doesNotContain", + "notOneOf", + "matches", + "hasTags", + ) elif self == ValueType.ID: return operation.value in ("is", "isNot", "oneOf", "notOneOf") elif self == ValueType.NUMBER: @@ -91,7 +104,7 @@ def is_valid(self, operation: ConditionType) -> bool: # must be BOOLEAN return operation.value in ("is",) - def validate(self, value: typing.Union[int, list[str], str, None], operation: ConditionType = None) -> bool: + def validate(self, value: typing.Union[int, typing.List[str], str, None], operation: ConditionType = None) -> bool: if isinstance(value, list) and operation in (ConditionType.ONE_OF, ConditionType.NOT_ONE_OF): return all(self.validate(v, None) for v in value) if value is None: @@ -99,7 +112,7 @@ def validate(self, value: typing.Union[int, list[str], str, None], operation: Co if self == ValueType.ID: # make sure it's an uuid return isinstance(value, str) and is_uuid(value) - elif self == ValueType.STRING: + elif self in (ValueType.STRING, ValueType.IMPORTED_PAYEE): return isinstance(value, str) elif self == ValueType.DATE: try: @@ -120,8 +133,10 @@ def validate(self, value: typing.Union[int, list[str], str, None], operation: Co def from_field(cls, field: str | None) -> ValueType: if field in ("acct", "category", "description"): return ValueType.ID - elif field in ("notes", "imported_description"): + elif field in ("notes",): return ValueType.STRING + elif field in ("imported_description",): + return ValueType.IMPORTED_PAYEE elif field in ("date",): return ValueType.DATE elif field in ("cleared", "reconciled"): @@ -133,8 +148,8 @@ def from_field(cls, field: str | None) -> ValueType: def get_value( - value: typing.Union[int, list[str], str, None], value_type: ValueType -) -> typing.Union[int, datetime.date, list[str], str, None]: + value: typing.Union[int, typing.List[str], str, None], value_type: ValueType +) -> typing.Union[int, datetime.date, typing.List[str], str, None]: """Converts the value to an actual value according to the type.""" if value_type is ValueType.DATE: if isinstance(value, str): @@ -143,7 +158,7 @@ def get_value( return datetime.datetime.strptime(str(value), "%Y%m%d").date() elif value_type is ValueType.BOOLEAN: return int(value) # database accepts 0 or 1 - elif value_type is ValueType.STRING: + elif value_type in (ValueType.STRING, ValueType.IMPORTED_PAYEE): if isinstance(value, list): return [get_value(v, value_type) for v in value] else: @@ -153,8 +168,8 @@ def get_value( def condition_evaluation( op: ConditionType, - true_value: typing.Union[int, list[str], str, datetime.date, None], - self_value: typing.Union[int, list[str], str, datetime.date, BetweenValue, None], + true_value: typing.Union[int, typing.List[str], str, datetime.date, None], + self_value: typing.Union[int, typing.List[str], str, datetime.date, BetweenValue, None], options: dict = None, ) -> bool: """Helper function to evaluate the condition based on the true_value, value found on the transaction, and the @@ -193,7 +208,7 @@ def condition_evaluation( elif op == ConditionType.CONTAINS: return self_value in true_value elif op == ConditionType.MATCHES: - return bool(re.match(self_value, true_value, re.IGNORECASE)) + return bool(re.search(self_value, true_value, re.IGNORECASE)) elif op == ConditionType.NOT_ONE_OF: return true_value not in self_value elif op == ConditionType.DOES_NOT_CONTAIN: @@ -208,6 +223,11 @@ def condition_evaluation( return self_value >= true_value elif op == ConditionType.IS_BETWEEN: return self_value.num_1 <= true_value <= self_value.num_2 + elif op == ConditionType.HAS_TAGS: + # this regex is not correct, but is good enough according to testing + # taken from https://stackoverflow.com/a/26740753/12681470 + tags = re.findall(r"\#[\U00002600-\U000027BF\U0001f300-\U0001f64F\U0001f680-\U0001f6FF\w-]+", self_value) + return any(tag in true_value for tag in tags) else: raise ActualError(f"Operation {op} not supported") @@ -248,7 +268,16 @@ class Condition(pydantic.BaseModel): ] op: ConditionType value: typing.Union[ - int, float, str, list[str], Schedule, list[BaseModel], BetweenValue, BaseModel, datetime.date, None + int, + float, + str, + typing.List[str], + Schedule, + typing.List[BaseModel], + BetweenValue, + BaseModel, + datetime.date, + None, ] type: typing.Optional[ValueType] = None options: typing.Optional[dict] = None @@ -264,7 +293,7 @@ def as_dict(self): ret.pop("options", None) return ret - def get_value(self) -> typing.Union[int, datetime.date, list[str], str, None]: + def get_value(self) -> typing.Union[int, datetime.date, typing.List[str], str, None]: return get_value(self.value, self.type) @pydantic.model_validator(mode="after") @@ -330,7 +359,7 @@ class Action(pydantic.BaseModel): op: ActionType = pydantic.Field(ActionType.SET, description="Action type to apply (default changes a column).") value: typing.Union[str, bool, int, float, pydantic.BaseModel, None] type: typing.Optional[ValueType] = None - options: dict[str, typing.Union[str, int]] = None + options: typing.Dict[str, typing.Union[str, int]] = None def __str__(self) -> str: if self.op in (ActionType.SET, ActionType.LINK_SCHEDULE): @@ -343,6 +372,12 @@ def __str__(self) -> str: method = self.options.get("method") or "" split_index = self.options.get("splitIndex") or "" return f"allocate a {method} at Split {split_index}: {self.value}" + elif self.op in (ActionType.APPEND_NOTES, ActionType.PREPEND_NOTES): + return ( + f"append to notes '{self.value}'" + if self.op == ActionType.APPEND_NOTES + else f"prepend to notes '{self.value}'" + ) def as_dict(self): """Returns valid dict for database insertion.""" @@ -369,6 +404,9 @@ def check_operation_type(self): self.type = ValueType.ID elif self.op == ActionType.SET_SPLIT_AMOUNT: self.type = ValueType.NUMBER + # questionable choice from the developers to set it to ID, I hope they fix it at some point, but we change it + if self.op in (ActionType.APPEND_NOTES, ActionType.PREPEND_NOTES): + self.type = ValueType.STRING # if a pydantic object is provided and id is expected, extract the id if isinstance(self.value, pydantic.BaseModel) and hasattr(self.value, "id"): self.value = str(self.value.id) @@ -392,6 +430,16 @@ def run(self, transaction: Transactions) -> None: setattr(transaction, attr, value) elif self.op == ActionType.LINK_SCHEDULE: transaction.schedule_id = self.value + # for the notes rule, check if the rule was already applied since actual does not do that. + # this should ensure the prefix or suffix is not applied multiple times + elif self.op == ActionType.APPEND_NOTES: + notes = transaction.notes or "" + if not notes.endswith(self.value): + transaction.notes = f"{notes}{self.value}" + elif self.op == ActionType.PREPEND_NOTES: + notes = transaction.notes or "" + if not notes.startswith(self.value): + transaction.notes = f"{self.value}{notes}" else: raise ActualError(f"Operation {self.op} not supported") @@ -406,13 +454,13 @@ class Rule(pydantic.BaseModel): automatically. """ - conditions: list[Condition] = pydantic.Field( + conditions: typing.List[Condition] = pydantic.Field( ..., description="List of conditions that need to be met (one or all) in order for the actions to be applied." ) operation: typing.Literal["and", "or"] = pydantic.Field( "and", description="Operation to apply for the rule evaluation. If 'all' or 'any' need to be evaluated." ) - actions: list[Action] = pydantic.Field(..., description="List of actions to apply to the transaction.") + actions: typing.List[Action] = pydantic.Field(..., description="List of actions to apply to the transaction.") stage: typing.Literal["pre", "post", None] = pydantic.Field( None, description="Stage in which the rule" "will be evaluated (default None)" ) @@ -525,7 +573,7 @@ class RuleSet(pydantic.BaseModel): >>> ]) """ - rules: list[Rule] + rules: typing.List[Rule] def __str__(self): return "\n".join([str(r) for r in self.rules]) @@ -534,7 +582,9 @@ def __iter__(self) -> typing.Iterator[Rule]: return self.rules.__iter__() def _run( - self, transaction: typing.Union[Transactions, list[Transactions]], stage: typing.Literal["pre", "post", None] + self, + transaction: typing.Union[Transactions, typing.List[Transactions]], + stage: typing.Literal["pre", "post", None], ): for rule in [r for r in self.rules if r.stage == stage]: if isinstance(transaction, list): diff --git a/actual/schedules.py b/actual/schedules.py index 111a5a3..1a07e3b 100644 --- a/actual/schedules.py +++ b/actual/schedules.py @@ -101,7 +101,7 @@ class Schedule(pydantic.BaseModel): start: datetime.date = pydantic.Field(..., description="Start date of the schedule.") interval: int = pydantic.Field(1, description="Repeat every interval at frequency unit.") frequency: Frequency = pydantic.Field(Frequency.MONTHLY, description="Unit for the defined interval.") - patterns: list[Pattern] = pydantic.Field(default_factory=list) + patterns: typing.List[Pattern] = pydantic.Field(default_factory=list) skip_weekend: bool = pydantic.Field( False, alias="skipWeekend", description="If should move schedule before or after a weekend." ) @@ -192,10 +192,13 @@ def rruleset(self) -> rruleset: # for the month or weekday rules, add a different rrule to the ruleset. This is because otherwise the rule # would only look for, for example, days that are 15 that are also Fridays, and that is not desired if by_month_day: - monthly_config = config.copy() | {"bymonthday": by_month_day} + monthly_config = config.copy() + monthly_config.update({"bymonthday": by_month_day}) rule_sets_configs.append(monthly_config) if by_weekday: - rule_sets_configs.append(config.copy() | {"byweekday": by_weekday}) + weekly_config = config.copy() + weekly_config.update({"byweekday": by_weekday}) + rule_sets_configs.append(weekly_config) # if ruleset does not contain multiple rules, add the current rule as default if not rule_sets_configs: rule_sets_configs.append(config) @@ -214,11 +217,11 @@ def do_skip_weekend( if self.end_mode == EndMode.ON_DATE and value > date_to_datetime(self.end_date): return None else: # BEFORE - value_after = value - datetime.timedelta(days=value.weekday() - 4) - if value_after < dt_start: + value_before = value - datetime.timedelta(days=value.weekday() - 4) + if value_before < dt_start: # value is in the past, skip and look for another return None - value = value_after + value = value_before return value def before(self, date: datetime.date = None) -> typing.Optional[datetime.date]: @@ -230,9 +233,12 @@ def before(self, date: datetime.date = None) -> typing.Optional[datetime.date]: before_datetime = rs.before(dt_start) if not before_datetime: return None - return self.do_skip_weekend(dt_start, before_datetime).date() + with_weekend_skip = self.do_skip_weekend(date_to_datetime(self.start), before_datetime) + if not with_weekend_skip: + return None + return with_weekend_skip.date() - def xafter(self, date: datetime.date = None, count: int = 1) -> list[datetime.date]: + def xafter(self, date: datetime.date = None, count: int = 1) -> typing.List[datetime.date]: if not date: date = datetime.date.today() # dateutils only accepts datetime for evaluation diff --git a/actual/utils/title.py b/actual/utils/title.py index c41f90c..da2f368 100644 --- a/actual/utils/title.py +++ b/actual/utils/title.py @@ -1,4 +1,5 @@ import re +from typing import List conjunctions = [ "for", @@ -159,7 +160,7 @@ ) -def convert_to_regexp(special_characters: list[str]): +def convert_to_regexp(special_characters: List[str]): return [(re.compile(rf"\b{s}\b", re.IGNORECASE), s) for s in special_characters] @@ -184,7 +185,7 @@ def replace_func(m: re.Match): return (lead or "") + (lower or forced or "").upper() + (rest or "") -def title(title_str: str, custom_specials: list[str] = None): +def title(title_str: str, custom_specials: List[str] = None): title_str = title_str.lower() title_str = regex.sub(replace_func, title_str) diff --git a/actual/version.py b/actual/version.py index 446b117..147950a 100644 --- a/actual/version.py +++ b/actual/version.py @@ -1,2 +1,2 @@ -__version_info__ = ("0", "4", "0") +__version_info__ = ("0", "5", "1") __version__ = ".".join(__version_info__) diff --git a/docs/API-reference/actual.md b/docs/API-reference/actual.md new file mode 100644 index 0000000..4d3ce9b --- /dev/null +++ b/docs/API-reference/actual.md @@ -0,0 +1,5 @@ +# Actual + +::: actual.Actual + options: + inherited_members: false diff --git a/docs/API-reference/endpoints.md b/docs/API-reference/endpoints.md new file mode 100644 index 0000000..0ec42be --- /dev/null +++ b/docs/API-reference/endpoints.md @@ -0,0 +1,6 @@ +# Endpoints + +::: actual.api +::: actual.api.models +::: actual.api.bank_sync +::: actual.protobuf_models diff --git a/docs/API-reference/models.md b/docs/API-reference/models.md new file mode 100644 index 0000000..af17886 --- /dev/null +++ b/docs/API-reference/models.md @@ -0,0 +1,5 @@ +# Database models + +::: actual.database + options: + inherited_members: false diff --git a/docs/API-reference/queries.md b/docs/API-reference/queries.md new file mode 100644 index 0000000..facc4bd --- /dev/null +++ b/docs/API-reference/queries.md @@ -0,0 +1,3 @@ +# Queries + +::: actual.queries diff --git a/docs/experimental-features.md b/docs/experimental-features.md new file mode 100644 index 0000000..38fb49c --- /dev/null +++ b/docs/experimental-features.md @@ -0,0 +1,74 @@ +# Experimental features + +!!! danger + Experimental features do not have all the testing necessary to ensure correctness in comparison to the + files generated by the Javascript API. This means that this operations could in theory corrupt the data. Make sure + you have backups of your data before trying any of those operations. + +## Bootstraping a new server and uploading a first file + +The following script would generate a new empty budget on the Actual server, even if the server was not bootstrapped +with an initial password. + +```python +from actual import Actual + +with Actual(base_url="http://localhost:5006", password="mypass", bootstrap=True) as actual: + actual.create_budget("My budget") + actual.upload_budget() +``` + +You will then have a freshly created new budget to use: + +![created-budget](./static/new-budget.png?raw=true) + +If the `encryption_password` is set, the budget will additionally also be encrypted on the upload step to the server. + +## Updating transactions using Bank Sync + +If you have either [goCardless](https://actualbudget.org/docs/advanced/bank-sync/#gocardless-setup) or +[simplefin](https://actualbudget.org/docs/experimental/simplefin-sync/) integration configured, it is possible to +update the transactions using just the Python API alone. This is because the actual queries to the third-party service +are handled on the server, so the client does not have to do any custom API queries. + +To sync your account, simply call the `run_bank_sync` method: + +```python +from actual import Actual + +with Actual(base_url="http://localhost:5006", password="mypass") as actual: + synchronized_transactions = actual.run_bank_sync() + for transaction in synchronized_transactions: + print(f"Added of modified {transaction}") + # sync changes back to the server + actual.commit() + +``` + +## Running rules + +You can also automatically run rules using the library: + +```python +from actual import Actual + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + actual.run_rules() + # sync changes back to the server + actual.commit() +``` + +You can also manipulate the rules individually: + +```python +from actual import Actual +from actual.queries import get_ruleset, get_transactions + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + rs = get_ruleset(actual.session) + transactions = get_transactions(actual.session) + for rule in rs: + for t in transactions: + if rule.evaluate(t): + print(f"Rule {rule} matches for {t}") +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..615ae6b --- /dev/null +++ b/docs/index.md @@ -0,0 +1,72 @@ +# Quickstart + +## Adding new transactions + +After you created your first budget (or when updating an existing budget), you can add new transactions by adding them +using the `actual.session.add()` method. You cannot use the SQLAlchemy session directly because that adds the entries to your +local database, but will not sync the results back to the server (that is only possible when re-uploading the file). + +The method will make sure the local database is updated, but will also send a SYNC request with the added data so that +it will be immediately available on the frontend: + +```python +import decimal +import datetime +from actual import Actual +from actual.queries import create_transaction, create_account + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + act = create_account(actual.session, "My account") + t = create_transaction( + actual.session, + datetime.date.today(), + act, + "My payee", + notes="My first transaction", + amount=decimal.Decimal(-10.5), + ) + actual.commit() # use the actual.commit() instead of session.commit()! +``` + +Will produce: + +![added-transaction](./static/added-transaction.png?raw=true) + +## Updating existing transactions + +You may also update transactions using the SQLModel directly, you just need to make sure to commit the results at the +end: + +```python +from actual import Actual +from actual.queries import get_transactions + + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + for transaction in get_transactions(actual.session): + # change the transactions notes + if transaction.notes is not None and "my pattern" in transaction.notes: + transaction.notes = transaction.notes + " my suffix!" + # commit your changes! + actual.commit() + +``` + +!!! warning + You can also modify the relationships, for example the `transaction.payee.name`, but you to be aware that + this payee might be used for more than one transaction. Whenever the relationship is anything but 1:1, you have to + track the changes already done to prevent modifying a field twice. + +## Generating backups + +You can use actualpy to generate regular backups of your server files. Here is a script that will backup your server +file on the current folder: + +```python +from actual import Actual +from datetime import datetime + +with Actual(base_url="http://localhost:5006", password="mypass", file="My budget") as actual: + current_date = datetime.now().strftime("%Y%m%d-%H%M") + actual.export_data(f"actual_backup_{current_date}.zip") +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..b8a1467 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +mkdocs-material +mkdocs +mkdocstrings-python +griffe-fieldz +black diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..d0ed4f2 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,62 @@ +site_name: actualpy Documentation +strict: true +site_description: A Python re-implementation of the NodeJS API for Actual Budget +repo_name: bvanelli/actualpy +repo_url: https://github.com/bvanelli/actualpy +edit_uri: edit/main/docs/ + +theme: + name: material + palette: + # Palette toggle for light mode + - scheme: default + toggle: + icon: material/weather-night + name: Switch to dark mode + # Palette toggle for dark mode + - scheme: slate + toggle: + icon: material/weather-sunny + name: Switch to light mode + features: + - content.code.copy + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + +plugins: +- search +- mkdocstrings: + handlers: + python: + import: + - https://docs.python.org/3/objects.inv + options: + docstring_style: sphinx + docstring_options: + ignore_init_summary: true + docstring_section_style: list + filters: ["!^_"] + heading_level: 1 + inherited_members: true + merge_init_into_class: true + parameter_headings: true + separate_signature: true + members_order: source + show_root_heading: true + show_root_full_path: false + show_signature_annotations: true + show_symbol_type_heading: true + show_symbol_type_toc: true + signature_crossrefs: true + summary: true + extensions: + - griffe_fieldz: { include_inherited: true } diff --git a/requirements-dev.txt b/requirements-dev.txt index f7fe509..047523e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ pytest-mock pytest pytest-cov testcontainers +pre-commit diff --git a/setup.py b/setup.py index 2de6b9e..6503d8c 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ version=__version__, packages=find_packages(), description="Implementation of the Actual API to interact with Actual over Python.", - long_description=open("README.md").read(), + long_description=open("README.md").read().replace("> [!WARNING]", "⚠️**Warning**: "), long_description_content_type="text/markdown", author="Brunno Vanelli", author_email="brunnovanelli@gmail.com", diff --git a/tests/test_bank_sync.py b/tests/test_bank_sync.py index f7dd24f..0e9424f 100644 --- a/tests/test_bank_sync.py +++ b/tests/test_bank_sync.py @@ -4,7 +4,7 @@ import pytest -from actual import Actual +from actual import Actual, ActualBankSyncError from actual.database import Banks from actual.queries import create_account from tests.conftest import RequestsMock @@ -54,6 +54,12 @@ }, } +fail_response = { + "error_type": "ACCOUNT_NEEDS_ATTENTION", + "error_code": "ACCOUNT_NEEDS_ATTENTION", + "reason": "The account needs your attention.", +} + def create_accounts(session, protocol: str): bank = create_account(session, "Bank") @@ -140,3 +146,19 @@ def test_bank_sync_unconfigured(mocker, session): actual._session = session create_accounts(session, "simplefin") assert actual.run_bank_sync() == [] + + +def test_bank_sync_exception(session, mocker): + mocker.patch("requests.get").return_value = RequestsMock({"status": "ok", "data": {"validated": True}}) + main_mock = mocker.patch("requests.post") + main_mock.side_effect = [ + RequestsMock({"status": "ok", "data": {"configured": True}}), + RequestsMock({"status": "ok", "data": fail_response}), + ] + with Actual(token="foo") as actual: + actual._session = session + create_accounts(session, "simplefin") + + # now try to run the bank sync + with pytest.raises(ActualBankSyncError): + actual.run_bank_sync() diff --git a/tests/test_database.py b/tests/test_database.py index f09297a..4906ce7 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -4,7 +4,7 @@ import pytest -from actual import ActualError +from actual import Actual, ActualError from actual.database import Notes from actual.queries import ( create_account, @@ -63,6 +63,18 @@ def test_account_relationships(session): assert get_accounts(session, "Bank") == [bank] +def test_transaction(session): + today = date.today() + other = create_account(session, "Other") + coffee = create_transaction( + session, date=today, account="Other", payee="Starbucks", notes="coffee", amount=float(-9.95) + ) + session.commit() + assert coffee.amount == -995 + assert len(other.transactions) == 1 + assert other.balance == decimal.Decimal("-9.95") + + def test_reconcile_transaction(session): today = date.today() create_account(session, "Bank") @@ -191,3 +203,17 @@ def test_model_notes(session): session.commit() assert account_with_note.notes == "My note" assert account_without_note.notes is None + + +def test_default_imported_payee(session): + t = create_transaction(session, date(2024, 1, 4), create_account(session, "Bank"), imported_payee=" foo ") + session.flush() + assert t.payee.name == "foo" + assert t.imported_description == "foo" + + +def test_session_error(mocker): + mocker.patch("actual.Actual.validate") + with Actual(token="foo") as actual: + with pytest.raises(ActualError, match="No session defined"): + print(actual.session) # try to access the session, should raise an exception diff --git a/tests/test_integration.py b/tests/test_integration.py index 7aa4ea0..9de0be3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,11 +1,12 @@ import datetime import pytest +from sqlalchemy import delete, select from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs from actual import Actual, js_migration_statements -from actual.database import __TABLE_COLUMNS_MAP__, reflect_model +from actual.database import __TABLE_COLUMNS_MAP__, Dashboard, Migrations, reflect_model from actual.exceptions import ActualDecryptionError, ActualError, AuthorizationError from actual.queries import ( create_transaction, @@ -36,6 +37,7 @@ def test_create_user_file(actual_server): assert len(actual.list_user_files().data) == 0 actual.create_budget("My Budget") actual.upload_budget() + assert "userId" in actual.get_metadata() # add some entries to the budget acct = get_or_create_account(actual.session, "Bank") assert acct.balance == 0 @@ -156,6 +158,25 @@ def test_header_login(): assert response_login.data.token == response_header_login.data.token +def test_session_reflection_after_migrations(): + with DockerContainer("actualbudget/actual-server:24.9.0").with_exposed_ports(5006) as container: + port = container.get_exposed_port(5006) + wait_for_logs(container, "Listening on :::5006...") + with Actual(f"http://localhost:{port}", password="mypass", bootstrap=True) as actual: + actual.create_budget("My Budget") + actual.upload_budget() + # add a dashboard entry + actual.session.add(Dashboard(id="123", x=1, y=2)) + actual.commit() + # revert the last migration like it never happened + Dashboard.__table__.drop(actual.engine) + actual.session.exec(delete(Migrations).where(Migrations.id == 1722804019000)) + actual.session.commit() + # now try to download the budget, it should not fail + with Actual(f"http://localhost:{port}", file="My Budget", password="mypass") as actual: + assert len(actual.session.exec(select(Dashboard)).all()) > 2 # there are two default dashboards + + def test_empty_query_migrations(): # empty queries should not fail assert js_migration_statements("await db.runQuery('');") == [] diff --git a/tests/test_rules.py b/tests/test_rules.py index 71d22c4..273fcde 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -87,7 +87,7 @@ def test_string_condition(): assert Condition(field="notes", op="matches", value="g.*").run(t) is False assert Condition(field="notes", op="doesNotContain", value="foo").run(t) is False assert Condition(field="notes", op="doesNotContain", value="foobar").run(t) is True - # test the cases where the case do not match + # case insensitive entries assert Condition(field="notes", op="oneOf", value=["FOO", "BAR"]).run(t) is True assert Condition(field="notes", op="notOneOf", value=["FOO", "BAR"]).run(t) is False assert Condition(field="notes", op="contains", value="FO").run(t) is True @@ -98,6 +98,35 @@ def test_string_condition(): assert Condition(field="notes", op="doesNotContain", value="FOOBAR").run(t) is True +def test_has_tags(): + mock = MagicMock() + acct = create_account(mock, "Bank") + t = create_transaction(mock, datetime.date(2024, 1, 1), acct, "", "foo #bar #✨ #🙂‍↔️") + assert Condition(field="notes", op="hasTags", value="#bar").run(t) is True + assert Condition(field="notes", op="hasTags", value="#foo").run(t) is False + # test other unicode entries + assert Condition(field="notes", op="hasTags", value="#emoji #✨").run(t) is True + assert Condition(field="notes", op="hasTags", value="#🙂‍↔️").run(t) is True # new emojis should be supported + assert Condition(field="notes", op="hasTags", value="bar").run(t) is False # individual string will not match + + +@pytest.mark.parametrize( + "op,condition_value,value,expected_result", + [ + ("contains", "supermarket", "Best Supermarket", True), + ("contains", "supermarket", None, False), + ("oneOf", ["my supermarket", "other supermarket"], "MY SUPERMARKET", True), + ("oneOf", ["supermarket"], None, False), + ("matches", "market", "hypermarket", True), + ], +) +def test_imported_payee_condition(op, condition_value, value, expected_result): + t = create_transaction(MagicMock(), datetime.date(2024, 1, 1), "Bank", "", amount=5, imported_payee=value) + condition = {"field": "imported_description", "type": "imported_payee", "op": op, "value": condition_value} + cond = Condition.model_validate(condition) + assert cond.run(t) == expected_result + + def test_numeric_condition(): t = create_transaction(MagicMock(), datetime.date(2024, 1, 1), "Bank", "", amount=5) c1 = Condition(field="amount_inflow", op="gt", value=10.0) @@ -181,6 +210,8 @@ def test_value_type_condition_validation(): assert ValueType.ID.is_valid(ConditionType.CONTAINS) is False assert ValueType.STRING.is_valid(ConditionType.CONTAINS) is True assert ValueType.STRING.is_valid(ConditionType.GT) is False + assert ValueType.IMPORTED_PAYEE.is_valid(ConditionType.CONTAINS) is True + assert ValueType.IMPORTED_PAYEE.is_valid(ConditionType.GT) is False def test_value_type_value_validation(): @@ -196,6 +227,8 @@ def test_value_type_value_validation(): assert ValueType.ID.validate("foo") is False assert ValueType.BOOLEAN.validate(True) is True assert ValueType.BOOLEAN.validate("") is False + assert ValueType.IMPORTED_PAYEE.validate("") is True + assert ValueType.IMPORTED_PAYEE.validate(1) is False # list and NoneType assert ValueType.DATE.validate(None) assert ValueType.DATE.validate(["2024-10-04"], ConditionType.ONE_OF) is True @@ -207,6 +240,7 @@ def test_value_type_from_field(): assert ValueType.from_field("notes") == ValueType.STRING assert ValueType.from_field("date") == ValueType.DATE assert ValueType.from_field("cleared") == ValueType.BOOLEAN + assert ValueType.from_field("imported_description") == ValueType.IMPORTED_PAYEE with pytest.raises(ValueError): ValueType.from_field("foo") @@ -331,3 +365,23 @@ def test_set_split_amount_exception(session, mocker): session.flush() with pytest.raises(ActualSplitTransactionError): rs.run(t) + + +@pytest.mark.parametrize( + "operation,value,note,expected", + [ + ("append-notes", "bar", "foo", "foobar"), + ("prepend-notes", "bar", "foo", "barfoo"), + ("append-notes", "bar", None, "bar"), + ("prepend-notes", "bar", None, "bar"), + ], +) +def test_preppend_append_notes(operation, value, note, expected): + mock = MagicMock() + t = create_transaction(mock, datetime.date(2024, 1, 1), "Bank", "", notes=note) + action = Action(field="description", op=operation, value=value) + action.run(t) + assert t.notes == expected + action.run(t) # second iteration should not update the result + assert t.notes == expected + assert f"{operation.split('-')[0]} to notes '{value}'" in str(action) diff --git a/tests/test_schedules.py b/tests/test_schedules.py index fb51f47..7930267 100644 --- a/tests/test_schedules.py +++ b/tests/test_schedules.py @@ -70,6 +70,48 @@ def test_complex_schedules(): assert str(s) == "Every month on the last Sunday, 2nd Saturday, 10th, 31st, 5th (before weekend)" +def test_skip_weekend_after_schedule(): + s = Schedule.model_validate( + { + "start": "2024-08-14", + "interval": 1, + "frequency": "monthly", + "patterns": [], + "skipWeekend": True, + "weekendSolveMode": "after", + "endMode": "on_date", + "endOccurrences": 1, + "endDate": "2024-09-14", + } + ) + after = s.xafter(date(2024, 9, 10), count=2) + # we should ensure that dates that fall outside the endDate are not covered, even though actual will accept it + assert after == [] + + +def test_skip_weekend_before_schedule(): + s = Schedule.model_validate( + { + "start": "2024-04-10", + "interval": 1, + "frequency": "monthly", + "patterns": [], + "skipWeekend": True, + "weekendSolveMode": "before", + "endMode": "never", + "endOccurrences": 1, + "endDate": "2024-04-10", + } + ) + before = s.before(date(2024, 8, 14)) + assert before == date(2024, 8, 9) + # check that it wouldn't pick itself + assert s.before(date(2024, 7, 10)) == date(2024, 6, 10) + # we should ensure that dates that fall outside the endDate are not covered, even though actual will accept it + s.start = date(2024, 9, 21) + assert s.before(date(2024, 9, 22)) is None + + def test_is_approx(): # create schedule for every 1st and last day of the month (30th or 31st) s = Schedule.model_validate(