Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(logging): Clean up statsd logging #25847

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions superset/advanced_data_type/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ class AdvancedDataTypeRestApi(BaseSupersetApi):
@permission_name("read")
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=False, # pylint: disable-arguments-renamed
)
@rison(advanced_data_type_convert_schema)
def get(self, **kwargs: Any) -> Response:
Expand Down Expand Up @@ -116,7 +115,6 @@ def get(self, **kwargs: Any) -> Response:
@permission_name("read")
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=False, # pylint: disable-arguments-renamed
)
def get_types(self) -> Response:
"""Return a list of available advanced data types.
Expand Down
11 changes: 2 additions & 9 deletions superset/annotation_layers/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,8 @@
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod
from superset.extensions import event_logger
from superset.models.annotations import AnnotationLayer
from superset.views.base_api import (
BaseSupersetModelRestApi,
requires_json,
statsd_metrics,
)
from superset.views.base import statsd_metrics
from superset.views.base_api import BaseSupersetModelRestApi, requires_json

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -113,7 +110,6 @@ class AnnotationLayerRestApi(BaseSupersetModelRestApi):
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.delete",
log_to_statsd=False,
)
@permission_name("delete")
def delete(self, pk: int) -> Response:
Expand Down Expand Up @@ -167,7 +163,6 @@ def delete(self, pk: int) -> Response:
@permission_name("post")
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post",
log_to_statsd=False,
)
@requires_json
def post(self) -> Response:
Expand Down Expand Up @@ -231,7 +226,6 @@ def post(self) -> Response:
@permission_name("put")
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.put",
log_to_statsd=False,
)
@requires_json
def put(self, pk: int) -> Response:
Expand Down Expand Up @@ -302,7 +296,6 @@ def put(self, pk: int) -> Response:
@rison(get_delete_ids_schema)
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.bulk_delete",
log_to_statsd=False,
)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Bulk delete annotation layers.
Expand Down
2 changes: 2 additions & 0 deletions superset/async_events/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

from superset.async_events.async_query_manager import AsyncQueryTokenException
from superset.extensions import async_query_manager, event_logger
from superset.views.base import statsd_metrics
from superset.views.base_api import BaseSupersetApi

logger = logging.getLogger(__name__)
Expand All @@ -33,6 +34,7 @@ class AsyncEventsRestApi(BaseSupersetApi):
allow_browser_login = True

@expose("/", methods=("GET",))
@statsd_metrics
@event_logger.log_this
@protect()
Comment on lines 36 to 39
Copy link
Contributor Author

@sebastianliebscher sebastianliebscher Nov 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most API endpoints already use @statsd_metrics. When this decorator is added to replace implicit log_to_statsd=True like in this example, the statsd metric

  • superset_<Endpoint>

will be replaced by

  • superset_<Class>_<Endpoint>_success|warning|error
  • superset_<Class>_<Endpoint>_time
  • superset_<Class>_<Endpoint>_time_count
  • superset_<Class>_<Endpoint>_time_sum

So it is kind of a breaking change.

@safe
Expand Down
4 changes: 2 additions & 2 deletions superset/available_domains/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
from superset.available_domains.schemas import AvailableDomainsSchema
from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
from superset.extensions import event_logger
from superset.views.base_api import BaseSupersetApi, statsd_metrics
from superset.views.base import statsd_metrics
from superset.views.base_api import BaseSupersetApi

logger = logging.getLogger(__name__)

Expand All @@ -44,7 +45,6 @@ class AvailableDomainsRestApi(BaseSupersetApi):
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.get",
log_to_statsd=True,
)
def get(self) -> Response:
"""
Expand Down
5 changes: 3 additions & 2 deletions superset/cachekeys/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
from superset.connectors.sqla.models import SqlaTable
from superset.extensions import cache_manager, db, event_logger, stats_logger_manager
from superset.models.cache import CacheKey
from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics
from superset.views.base import statsd_metrics
from superset.views.base_api import BaseSupersetModelRestApi

logger = logging.getLogger(__name__)

Expand All @@ -48,7 +49,7 @@ class CacheRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
@event_logger.log_this_with_context(log_to_statsd=False)
@event_logger.log_this
def invalidate(self) -> Response:
"""
Take a list of datasources, find and invalidate the associated cache records
Expand Down
15 changes: 1 addition & 14 deletions superset/charts/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,12 @@
from superset.tasks.utils import get_current_user
from superset.utils.screenshots import ChartScreenshot, DEFAULT_CHART_WINDOW_SIZE
from superset.utils.urls import get_url_path
from superset.views.base import statsd_metrics
from superset.views.base_api import (
BaseSupersetModelRestApi,
RelatedFieldFilter,
requires_form_data,
requires_json,
statsd_metrics,
)
from superset.views.filters import BaseFilterRelatedUsers, FilterRelatedOwners

Expand Down Expand Up @@ -281,7 +281,6 @@ def ensure_thumbnails_enabled(self) -> Optional[Response]:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.post",
log_to_statsd=False,
)
@requires_json
def post(self) -> Response:
Expand Down Expand Up @@ -346,7 +345,6 @@ def post(self) -> Response:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.put",
log_to_statsd=False,
)
@requires_json
def put(self, pk: int) -> Response:
Expand Down Expand Up @@ -422,7 +420,6 @@ def put(self, pk: int) -> Response:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.delete",
log_to_statsd=False,
)
def delete(self, pk: int) -> Response:
"""Delete a chart.
Expand Down Expand Up @@ -478,7 +475,6 @@ def delete(self, pk: int) -> Response:
@rison(get_delete_ids_schema)
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.bulk_delete",
log_to_statsd=False,
)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Bulk delete charts.
Expand Down Expand Up @@ -537,7 +533,6 @@ def bulk_delete(self, **kwargs: Any) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".cache_screenshot",
log_to_statsd=False,
)
def cache_screenshot(self, pk: int, **kwargs: Any) -> WerkzeugResponse:
"""Compute and cache a screenshot.
Expand Down Expand Up @@ -609,7 +604,6 @@ def trigger_celery() -> WerkzeugResponse:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.screenshot",
log_to_statsd=False,
)
def screenshot(self, pk: int, digest: str) -> WerkzeugResponse:
"""Get a computed screenshot from cache.
Expand Down Expand Up @@ -663,7 +657,6 @@ def screenshot(self, pk: int, digest: str) -> WerkzeugResponse:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.thumbnail",
log_to_statsd=False,
)
def thumbnail(self, pk: int, digest: str, **kwargs: Any) -> WerkzeugResponse:
"""Compute or get already computed chart thumbnail from cache.
Expand Down Expand Up @@ -751,7 +744,6 @@ def thumbnail(self, pk: int, digest: str, **kwargs: Any) -> WerkzeugResponse:
@rison(get_export_ids_schema)
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.export",
log_to_statsd=False,
)
def export(self, **kwargs: Any) -> Response:
"""Download multiple charts as YAML files.
Expand Down Expand Up @@ -815,7 +807,6 @@ def export(self, **kwargs: Any) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".favorite_status",
log_to_statsd=False,
)
def favorite_status(self, **kwargs: Any) -> Response:
"""Check favorited charts for current user.
Expand Down Expand Up @@ -863,7 +854,6 @@ def favorite_status(self, **kwargs: Any) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".add_favorite",
log_to_statsd=False,
)
def add_favorite(self, pk: int) -> Response:
"""Mark the chart as favorite for the current user.
Expand Down Expand Up @@ -906,7 +896,6 @@ def add_favorite(self, pk: int) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".remove_favorite",
log_to_statsd=False,
)
def remove_favorite(self, pk: int) -> Response:
"""Remove the chart from the user favorite list.
Expand Down Expand Up @@ -949,7 +938,6 @@ def remove_favorite(self, pk: int) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".warm_up_cache",
log_to_statsd=False,
)
def warm_up_cache(self) -> Response:
"""Warm up the cache for the chart.
Expand Down Expand Up @@ -1003,7 +991,6 @@ def warm_up_cache(self) -> Response:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.import_",
log_to_statsd=False,
)
@requires_form_data
def import_(self) -> Response:
Expand Down
11 changes: 6 additions & 5 deletions superset/charts/data/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,12 @@
from superset.extensions import event_logger
from superset.models.sql_lab import Query
from superset.utils.core import create_zip, get_user_id, json_int_dttm_ser
from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse
from superset.views.base_api import statsd_metrics
from superset.views.base import (
CsvResponse,
generate_download_headers,
statsd_metrics,
XlsxResponse,
)

if TYPE_CHECKING:
from superset.common.query_context import QueryContext
Expand All @@ -65,7 +69,6 @@ class ChartDataRestApi(ChartRestApi):
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.data",
log_to_statsd=False,
)
def get_data(self, pk: int) -> Response:
"""
Expand Down Expand Up @@ -178,7 +181,6 @@ def get_data(self, pk: int) -> Response:
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.data",
log_to_statsd=False,
)
def data(self) -> Response:
"""
Expand Down Expand Up @@ -263,7 +265,6 @@ def data(self) -> Response:
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}"
f".data_from_cache",
log_to_statsd=False,
)
def data_from_cache(self, cache_key: str) -> Response:
"""
Expand Down
4 changes: 2 additions & 2 deletions superset/css_templates/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@
)
from superset.extensions import event_logger
from superset.models.core import CssTemplate
from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics
from superset.views.base import statsd_metrics
from superset.views.base_api import BaseSupersetModelRestApi

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -93,7 +94,6 @@ class CssTemplateRestApi(BaseSupersetModelRestApi):
@statsd_metrics
@event_logger.log_this_with_context(
action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.bulk_delete",
log_to_statsd=False,
)
@rison(get_delete_ids_schema)
def bulk_delete(self, **kwargs: Any) -> Response:
Expand Down
Loading
Loading