Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flask App factory PR #1 #8418

Merged
merged 48 commits into from
Nov 20, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
56e6c1a
First cut at app factory
craig-rueda Oct 15, 2019
21e3494
Setting things back to master
craig-rueda Oct 18, 2019
be37d24
Working with new FLASK_APP
craig-rueda Oct 18, 2019
9d31fff
Merge branch 'master' into app_factory
craig-rueda Oct 18, 2019
259a307
Still need to refactor Celery
craig-rueda Oct 21, 2019
9f8fe89
Merge branch 'master' into app_factory
craig-rueda Oct 25, 2019
89bba87
CLI mostly working
craig-rueda Oct 28, 2019
fb9cf33
Working on unit tests
craig-rueda Oct 28, 2019
01aafb5
Moving cli stuff around a bit
craig-rueda Oct 28, 2019
9bf6333
Merge branch 'master' into app_factory
craig-rueda Nov 6, 2019
db25d4d
Removing get in config
craig-rueda Nov 6, 2019
0ab9f0a
Defaulting test config
craig-rueda Nov 6, 2019
921c285
Adding flask-testing
craig-rueda Nov 7, 2019
095529f
flask-testing casing
craig-rueda Nov 8, 2019
428655b
resultsbackend property bug
craig-rueda Nov 8, 2019
97e4cda
Fixing up cli
craig-rueda Nov 8, 2019
9f686cd
Quick fix for KV api
craig-rueda Nov 8, 2019
f1318d5
Working on save slice
craig-rueda Nov 8, 2019
947622b
Fixed core_tests
craig-rueda Nov 9, 2019
91a39f5
Fixed utils_tests
craig-rueda Nov 9, 2019
f675ad4
Most tests working - still need to dig into remaining app_context iss…
craig-rueda Nov 13, 2019
78990d1
All tests passing locally - need to update code comments
craig-rueda Nov 13, 2019
7e37cef
Fixing dashboard tests again
craig-rueda Nov 13, 2019
3b95e96
Blacking
craig-rueda Nov 13, 2019
7f1dadc
Sorting imports
craig-rueda Nov 13, 2019
01607c4
linting
craig-rueda Nov 13, 2019
3a66b07
removing envvar mangling
craig-rueda Nov 13, 2019
5764936
Merge remote-tracking branch 'remotes/origin/master' into app_factory
craig-rueda Nov 14, 2019
74cd1bf
blacking
craig-rueda Nov 14, 2019
8d53bd7
Fixing unit tests
craig-rueda Nov 14, 2019
037cb2c
isorting
craig-rueda Nov 14, 2019
3c0d7da
licensing
craig-rueda Nov 14, 2019
c3c480d
fixing mysql tests
craig-rueda Nov 15, 2019
90a25f2
fixing cypress?
craig-rueda Nov 15, 2019
d8e74a2
fixing .flaskenv
craig-rueda Nov 15, 2019
55d5c27
fixing test app_ctx
craig-rueda Nov 15, 2019
92e7922
fixing cypress
craig-rueda Nov 15, 2019
798d723
Merge remote-tracking branch 'remotes/upstream/master' into app_factory
craig-rueda Nov 18, 2019
b943d68
moving manifest processor around
craig-rueda Nov 18, 2019
ab7a69c
moving results backend manager around
craig-rueda Nov 18, 2019
f72438a
Cleaning up __init__ a bit more
craig-rueda Nov 19, 2019
b30ff82
Addressing PR comments
craig-rueda Nov 19, 2019
b333044
Addressing PR comments
craig-rueda Nov 19, 2019
a9ecc7a
Blacking
craig-rueda Nov 19, 2019
ccd0338
Fixes for running celery worker
craig-rueda Nov 19, 2019
03a3e4a
Tuning isort
craig-rueda Nov 19, 2019
cf0dac2
Merge remote-tracking branch 'remotes/origin/master' into app_factory
craig-rueda Nov 19, 2019
d2f4ada
Blacking
craig-rueda Nov 19, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Setting things back to master
  • Loading branch information
craig-rueda committed Oct 18, 2019
commit 21e349440b6659cec4568573ab58ba7845100e84
3 changes: 1 addition & 2 deletions superset/connectors/sqla/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,10 @@
from sqlalchemy.sql.expression import Label, Select, TextAsFrom
import sqlparse

from superset import app
from superset import app, db, security_manager
from superset.connectors.base.models import BaseColumn, BaseDatasource, BaseMetric
from superset.db_engine_specs.base import TimestampExpression
from superset.exceptions import DatabaseNotFound
from superset.extensions import db, security_manager
from superset.jinja_context import get_template_processor
from superset.models.annotations import Annotation
from superset.models.core import Database
Expand Down
2 changes: 1 addition & 1 deletion superset/connectors/sqla/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
from flask_babel import lazy_gettext as _
from wtforms.ext.sqlalchemy.fields import QuerySelectField

from superset import appbuilder, db, security_manager
from superset.connectors.base.views import DatasourceModelView
from superset.extensions import appbuilder, db, security_manager
from superset.utils import core as utils
from superset.views.base import (
DatasourceFilter,
Expand Down
7 changes: 5 additions & 2 deletions superset/db_engine_specs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@
import sqlparse
from werkzeug.utils import secure_filename

from superset import sql_parse
from superset.extensions import db
from superset import app, db, sql_parse
from superset.utils import core as utils

if TYPE_CHECKING:
Expand All @@ -53,7 +52,11 @@ class TimeGrain(NamedTuple): # pylint: disable=too-few-public-methods
duration: Optional[str]


config = app.config


QueryStatus = utils.QueryStatus
config = app.config

builtin_time_grains: Dict[Optional[str], str] = {
None: "Time Column",
Expand Down
9 changes: 2 additions & 7 deletions superset/db_engine_specs/hive.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
from typing import Any, Dict, List, Optional, Tuple
from urllib import parse

from flask import current_app
from sqlalchemy import Column
from sqlalchemy.engine import create_engine
from sqlalchemy.engine.base import Engine
Expand All @@ -31,12 +30,13 @@
from sqlalchemy.sql.expression import ColumnClause, Select
from werkzeug.utils import secure_filename

from superset import app, conf
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.presto import PrestoEngineSpec
from superset.utils import core as utils

QueryStatus = utils.QueryStatus
conf = current_app.config
config = app.config

tracking_url_trans = conf.get("TRACKING_URL_TRANSFORMER")
hive_poll_interval = conf.get("HIVE_POLL_INTERVAL")
Expand All @@ -63,11 +63,6 @@ class HiveEngineSpec(PrestoEngineSpec):
r"reduce = (?P<reduce_progress>[0-9]+)%.*"
)

def __init__(self) -> None:
super().__init__()
self.tracking_url_trans = conf.get("TRACKING_URL_TRANSFORMER")
self.hive_poll_interval = conf.get("HIVE_POLL_INTERVAL")

@classmethod
def patch(cls):
from pyhive import hive # pylint: disable=no-name-in-module
Expand Down
13 changes: 7 additions & 6 deletions superset/db_engine_specs/presto.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@
from sqlalchemy.engine.result import RowProxy
from sqlalchemy.sql.expression import ColumnClause, Select

from superset import app, is_feature_enabled, security_manager
from superset.db_engine_specs.base import BaseEngineSpec
from superset.exceptions import SupersetTemplateException
from superset.extensions import feature_flag_manager, security_manager
from superset.models.sql_types.presto_sql_types import type_map as presto_type_map
from superset.sql_parse import ParsedQuery
from superset.utils import core as utils
Expand All @@ -44,6 +44,7 @@
from superset.models.core import Database # pylint: disable=unused-import

QueryStatus = utils.QueryStatus
config = app.config

# map between Presto types and Pandas
pandas_dtype_map = {
Expand Down Expand Up @@ -134,7 +135,7 @@ def get_table_names(
cls, database: "Database", inspector: Inspector, schema: Optional[str]
) -> List[str]:
tables = super().get_table_names(database, inspector, schema)
if not feature_flag_manager.is_feature_enabled("PRESTO_SPLIT_VIEWS_FROM_TABLES"):
if not is_feature_enabled("PRESTO_SPLIT_VIEWS_FROM_TABLES"):
return tables

views = set(cls.get_view_names(database, inspector, schema))
Expand All @@ -151,7 +152,7 @@ def get_view_names(
and get_view_names() is not implemented in sqlalchemy_presto.py
https://github.com/dropbox/PyHive/blob/e25fc8440a0686bbb7a5db5de7cb1a77bdb4167a/pyhive/sqlalchemy_presto.py
"""
if not feature_flag_manager.is_feature_enabled("PRESTO_SPLIT_VIEWS_FROM_TABLES"):
if not is_feature_enabled("PRESTO_SPLIT_VIEWS_FROM_TABLES"):
return []

if schema:
Expand Down Expand Up @@ -335,7 +336,7 @@ def get_columns(
for column in columns:
try:
# parse column if it is a row or array
if feature_flag_manager.is_feature_enabled("PRESTO_EXPAND_DATA") and (
if is_feature_enabled("PRESTO_EXPAND_DATA") and (
"array" in column.Type or "row" in column.Type
):
structural_column_index = len(result)
Expand Down Expand Up @@ -422,7 +423,7 @@ def select_star( # pylint: disable=too-many-arguments
"""
cols = cols or []
presto_cols = cols
if feature_flag_manager.is_feature_enabled("PRESTO_EXPAND_DATA") and show_cols:
if is_feature_enabled("PRESTO_EXPAND_DATA") and show_cols:
dot_regex = r"\.(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)"
presto_cols = [
col for col in presto_cols if not re.search(dot_regex, col["name"])
Expand Down Expand Up @@ -576,7 +577,7 @@ def expand_data( # pylint: disable=too-many-locals
:return: list of all columns(selected columns and their nested fields),
expanded data set, listed of nested fields
"""
if not feature_flag_manager.is_feature_enabled("PRESTO_EXPAND_DATA"):
if not is_feature_enabled("PRESTO_EXPAND_DATA"):
return columns, data, []

# process each column, unnesting ARRAY types and
Expand Down
6 changes: 4 additions & 2 deletions superset/jinja_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,12 @@
import uuid

from dateutil.relativedelta import relativedelta
from flask import current_app, g, request
from flask import g, request
from jinja2.sandbox import SandboxedEnvironment

config = current_app.config
from superset import app

config = app.config
BASE_CONTEXT = {
"datetime": datetime,
"random": random,
Expand Down
9 changes: 4 additions & 5 deletions superset/models/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import textwrap
from typing import List

from flask import current_app, escape, g, Markup, request
from flask import escape, g, Markup, request
from flask_appbuilder import Model
from flask_appbuilder.models.decorators import renders
from flask_appbuilder.security.sqla.models import User
Expand Down Expand Up @@ -52,9 +52,8 @@
from sqlalchemy_utils import EncryptedType
import sqlparse

from superset import db_engine_specs
from superset import app, db, db_engine_specs, is_feature_enabled, security_manager
from superset.connectors.connector_registry import ConnectorRegistry
from superset.extensions import db, feature_flag_manager, security_manager
from superset.legacy import update_time_range
from superset.models.helpers import AuditMixinNullable, ImportMixin
from superset.models.tags import ChartUpdater, DashboardUpdater, FavStarUpdater
Expand All @@ -63,7 +62,7 @@
from superset.viz import viz_types
from urllib import parse # noqa

config = current_app.config
config = app.config
custom_password_store = config.get("SQLALCHEMY_CUSTOM_PASSWORD_STORE")
stats_logger = config.get("STATS_LOGGER")
log_query = config.get("QUERY_LOGGER")
Expand Down Expand Up @@ -1311,7 +1310,7 @@ def user_roles(self):


# events for updating tags
if feature_flag_manager.is_feature_enabled("TAGGING_SYSTEM"):
if is_feature_enabled("TAGGING_SYSTEM"):
sqla.event.listen(Slice, "after_insert", ChartUpdater.after_insert)
sqla.event.listen(Slice, "after_update", ChartUpdater.after_update)
sqla.event.listen(Slice, "after_delete", ChartUpdater.after_delete)
Expand Down
2 changes: 1 addition & 1 deletion superset/models/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship

from superset.extensions import security_manager
from superset import security_manager
from superset.models.helpers import AuditMixinNullable, ImportMixin


Expand Down
2 changes: 1 addition & 1 deletion superset/models/sql_lab.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)
from sqlalchemy.orm import backref, relationship

from superset.extensions import security_manager
from superset import security_manager
from superset.models.helpers import AuditMixinNullable, ExtraJSONMixin
from superset.models.tags import QueryUpdater
from superset.utils.core import QueryStatus, user_label
Expand Down
2 changes: 1 addition & 1 deletion superset/models/user_attributes.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from sqlalchemy import Column, ForeignKey, Integer
from sqlalchemy.orm import relationship

from superset.extensions import security_manager
from superset import security_manager
from superset.models.helpers import AuditMixinNullable


Expand Down
21 changes: 10 additions & 11 deletions superset/viz.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,8 @@
import polyline
import simplejson as json

from flask import current_app
from superset import app, cache, get_css_manifest_files
from superset.exceptions import NullValueException, SpatialException
from superset.extensions import cache_manager, manifest_processor
from superset.utils import core as utils
from superset.utils.core import (
DTTM_ALIAS,
Expand All @@ -58,7 +57,7 @@
)


config = current_app.config
config = app.config
stats_logger = config.get("STATS_LOGGER")
relative_start = config.get("DEFAULT_RELATIVE_START_TIME", "today")
relative_end = config.get("DEFAULT_RELATIVE_END_TIME", "today")
Expand Down Expand Up @@ -395,8 +394,8 @@ def get_df_payload(self, query_obj=None, **kwargs):
stacktrace = None
df = None
cached_dttm = datetime.utcnow().isoformat().split(".")[0]
if cache_key and cache_manager.cache and not self.force:
cache_value = cache_manager.cache.get(cache_key)
if cache_key and cache and not self.force:
cache_value = cache.get(cache_key)
if cache_value:
stats_logger.incr("loaded_from_cache")
try:
Expand Down Expand Up @@ -430,7 +429,7 @@ def get_df_payload(self, query_obj=None, **kwargs):
if (
is_loaded
and cache_key
and cache_manager.cache
and cache
and self.status != utils.QueryStatus.FAILED
):
try:
Expand All @@ -446,13 +445,13 @@ def get_df_payload(self, query_obj=None, **kwargs):
)

stats_logger.incr("set_cache_key")
cache_manager.cache.set(cache_key, cache_value, timeout=self.cache_timeout)
cache.set(cache_key, cache_value, timeout=self.cache_timeout)
except Exception as e:
# cache.set call can fail if the backend is down or if
# the key is too large or whatever other reasons
logging.warning("Could not cache key {}".format(cache_key))
logging.exception(e)
cache_manager.cache.delete(cache_key)
cache.delete(cache_key)
return {
"cache_key": self._any_cache_key,
"cached_dttm": self._any_cached_dttm,
Expand Down Expand Up @@ -736,7 +735,7 @@ def get_data(self, df):
code = self.form_data.get("code", "")
if markup_type == "markdown":
code = markdown(code)
return dict(html=code, theme_css=manifest_processor.get_css_manifest_files("theme"))
return dict(html=code, theme_css=get_css_manifest_files("theme"))


class SeparatorViz(MarkupViz):
Expand Down Expand Up @@ -1293,7 +1292,7 @@ def get_data(self, df):
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.core import Slice
from superset.extensions import db
from superset import db

slice_ids1 = fd.get("line_charts")
slices1 = db.session.query(Slice).filter(Slice.id.in_(slice_ids1)).all()
Expand Down Expand Up @@ -2095,7 +2094,7 @@ def get_data(self, df):
fd = self.form_data
# Late imports to avoid circular import issues
from superset.models.core import Slice
from superset.extensions import db
from superset import db

slice_ids = fd.get("deck_slices")
slices = db.session.query(Slice).filter(Slice.id.in_(slice_ids)).all()
Expand Down