Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prevent sqlalchemy Transparent SQL Compilation Caching from filling up during purge #71015

Merged
merged 17 commits into from
Apr 29, 2022
Merged

Prevent sqlalchemy Transparent SQL Compilation Caching from filling up during purge #71015

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
efb2110
Prevent sqlalchemy Transparent SQL Compilation Caching from filling u…
bdraco Apr 28, 2022
df266f9
simplify expression
bdraco Apr 28, 2022
d68e5ca
restore caching
bdraco Apr 28, 2022
ce2231d
use an anonymous value to reduce sqlalchemy overhead
bdraco Apr 28, 2022
6e64672
tweak
bdraco Apr 28, 2022
e85d254
tweak
bdraco Apr 28, 2022
5d7df00
Revert "tweak"
bdraco Apr 28, 2022
1f59da4
100 block size
bdraco Apr 28, 2022
66106b4
Revert "100 block size"
bdraco Apr 28, 2022
17e7158
Revert "Revert "tweak""
bdraco Apr 28, 2022
1c43462
write it back out again as the other method did not work
bdraco Apr 28, 2022
4a220e3
tweak
bdraco Apr 28, 2022
db25f23
correct typing
bdraco Apr 28, 2022
2b2fc93
merge
bdraco Apr 28, 2022
e101b02
read the graph wrong its only 500 not 800
bdraco Apr 28, 2022
564ffd4
reduce some more
bdraco Apr 28, 2022
88cad5d
Merge branch 'dev' into sqlalchemy_caching
bdraco Apr 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
reduce some more
  • Loading branch information
@bdraco
bdraco committed Apr 28, 2022
commit 564ffd45e3b8aee733c01c624e48f887016c17ad
207 changes: 105 additions & 102 deletions homeassistant/components/recorder/purge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
from datetime import datetime
from itertools import zip_longest
import logging
from typing import TYPE_CHECKING, Final
from typing import TYPE_CHECKING

from sqlalchemy import func, lambda_stmt, select, union_all
from sqlalchemy.orm.session import Session
from sqlalchemy.sql.expression import distinct
from sqlalchemy.sql.lambdas import StatementLambdaElement
from sqlalchemy.sql.selectable import Select

from homeassistant.const import EVENT_STATE_CHANGED

Expand Down Expand Up @@ -114,7 +115,9 @@ def _select_event_state_and_attributes_ids_to_purge(
return event_ids, state_ids, attributes_ids


STATE_ATTRS_ID: Final = States.attributes_id
def _state_attrs_exist(attr: int | None) -> Select:
"""Check if a state attributes id exists in the states table."""
return select(func.min(States.attributes_id)).where(States.attributes_id == attr)


def _generate_find_attr_lambda(
Copy link
Member Author

@bdraco bdraco Apr 29, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The docs for sqlalchemy say

Avoid referring to non-SQL constructs inside of lambdas as they are not cacheable by default
....
The best way to resolve the above situation is to not refer to foo inside of the lambda, and refer to it outside instead:

>>> def my_stmt(foo):
...     x_param, y_param = foo.x, foo.y
...     stmt = lambda_stmt(lambda: select(func.max(x_param, y_param)))
...     return stmt

So even if I pass this as a list I'm stuck unpacking them as attr1, attr2, attr3, attr4, .... = attrs outside the lambda

Expand Down Expand Up @@ -225,106 +228,106 @@ def _generate_find_attr_lambda(
"""
return lambda_stmt(
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The lambda_stmt is actually looking at the code inside the lambda to generate the cache key so it seems like we are stuck writing this out if we want a single cache key with bindparams

lambda: union_all(
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr1),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr2),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr3),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr4),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr5),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr6),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr7),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr8),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr9),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr10),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr11),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr12),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr13),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr14),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr15),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr16),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr17),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr18),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr19),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr20),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr21),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr22),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr23),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr24),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr25),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr26),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr27),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr28),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr29),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr30),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr31),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr32),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr33),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr34),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr35),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr36),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr37),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr38),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr39),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr40),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr41),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr42),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr43),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr44),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr45),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr46),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr47),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr48),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr49),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr50),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr51),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr52),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr53),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr54),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr55),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr56),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr57),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr58),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr59),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr60),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr61),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr62),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr63),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr64),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr65),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr66),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr67),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr68),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr69),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr70),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr71),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr72),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr73),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr74),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr75),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr76),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr77),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr78),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr79),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr80),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr81),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr82),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr83),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr84),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr85),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr86),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr87),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr88),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr89),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr90),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr91),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr92),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr93),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr94),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr95),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr96),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr97),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr98),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr99),
select(func.min(STATE_ATTRS_ID)).where(STATE_ATTRS_ID == attr100),
_state_attrs_exist(attr1),
_state_attrs_exist(attr2),
_state_attrs_exist(attr3),
_state_attrs_exist(attr4),
_state_attrs_exist(attr5),
_state_attrs_exist(attr6),
_state_attrs_exist(attr7),
_state_attrs_exist(attr8),
_state_attrs_exist(attr9),
_state_attrs_exist(attr10),
_state_attrs_exist(attr11),
_state_attrs_exist(attr12),
_state_attrs_exist(attr13),
_state_attrs_exist(attr14),
_state_attrs_exist(attr15),
_state_attrs_exist(attr16),
_state_attrs_exist(attr17),
_state_attrs_exist(attr18),
_state_attrs_exist(attr19),
_state_attrs_exist(attr20),
_state_attrs_exist(attr21),
_state_attrs_exist(attr22),
_state_attrs_exist(attr23),
_state_attrs_exist(attr24),
_state_attrs_exist(attr25),
_state_attrs_exist(attr26),
_state_attrs_exist(attr27),
_state_attrs_exist(attr28),
_state_attrs_exist(attr29),
_state_attrs_exist(attr30),
_state_attrs_exist(attr31),
_state_attrs_exist(attr32),
_state_attrs_exist(attr33),
_state_attrs_exist(attr34),
_state_attrs_exist(attr35),
_state_attrs_exist(attr36),
_state_attrs_exist(attr37),
_state_attrs_exist(attr38),
_state_attrs_exist(attr39),
_state_attrs_exist(attr40),
_state_attrs_exist(attr41),
_state_attrs_exist(attr42),
_state_attrs_exist(attr43),
_state_attrs_exist(attr44),
_state_attrs_exist(attr45),
_state_attrs_exist(attr46),
_state_attrs_exist(attr47),
_state_attrs_exist(attr48),
_state_attrs_exist(attr49),
_state_attrs_exist(attr50),
_state_attrs_exist(attr51),
_state_attrs_exist(attr52),
_state_attrs_exist(attr53),
_state_attrs_exist(attr54),
_state_attrs_exist(attr55),
_state_attrs_exist(attr56),
_state_attrs_exist(attr57),
_state_attrs_exist(attr58),
_state_attrs_exist(attr59),
_state_attrs_exist(attr60),
_state_attrs_exist(attr61),
_state_attrs_exist(attr62),
_state_attrs_exist(attr63),
_state_attrs_exist(attr64),
_state_attrs_exist(attr65),
_state_attrs_exist(attr66),
_state_attrs_exist(attr67),
_state_attrs_exist(attr68),
_state_attrs_exist(attr69),
_state_attrs_exist(attr70),
_state_attrs_exist(attr71),
_state_attrs_exist(attr72),
_state_attrs_exist(attr73),
_state_attrs_exist(attr74),
_state_attrs_exist(attr75),
_state_attrs_exist(attr76),
_state_attrs_exist(attr77),
_state_attrs_exist(attr78),
_state_attrs_exist(attr79),
_state_attrs_exist(attr80),
_state_attrs_exist(attr81),
_state_attrs_exist(attr82),
_state_attrs_exist(attr83),
_state_attrs_exist(attr84),
_state_attrs_exist(attr85),
_state_attrs_exist(attr86),
_state_attrs_exist(attr87),
_state_attrs_exist(attr88),
_state_attrs_exist(attr89),
_state_attrs_exist(attr90),
_state_attrs_exist(attr91),
_state_attrs_exist(attr92),
_state_attrs_exist(attr93),
_state_attrs_exist(attr94),
_state_attrs_exist(attr95),
_state_attrs_exist(attr96),
_state_attrs_exist(attr97),
_state_attrs_exist(attr98),
_state_attrs_exist(attr99),
_state_attrs_exist(attr100),
)
)

Expand Down