diff --git a/UPDATING.md b/UPDATING.md index f7b8fa86a3207..5c7edbe5395a1 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -27,6 +27,11 @@ assists people when migrating to a new version. run `pip install superset[presto]` and/or `pip install superset[hive]` as required. +* [5445](https://github.com/apache/incubator-superset/pull/5445) : a change +which prevents encoding of empty string from form data in the datanbase. +This involves a non-schema changing migration which does potentially impact +a large number of records. Scheduled downtime may be advised. + ## Superset 0.31.0 * boto3 / botocore was removed from the dependency list. If you use s3 as a place to store your SQL Lab result set or Hive uploads, you may diff --git a/requirements.txt b/requirements.txt index 0b0fd48696fc4..5eef82f8fd4ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,8 +42,8 @@ kombu==4.2.1 # via celery mako==1.0.7 # via alembic markdown==3.0 markupsafe==1.0 # via jinja2, mako -numpy==1.15.2 # via numpy -pandas==0.23.4 # via pandas +numpy==1.15.2 # via pandas +pandas==0.23.4 parsedatetime==2.0.0 pathlib2==2.3.0 polyline==1.3.2 @@ -60,7 +60,7 @@ requests==2.20.0 retry==0.9.2 selenium==3.141.0 simplejson==3.15.0 -six==1.11.0 # via bleach, cryptography, isodate, pathlib2, polyline, pydruid, python-dateutil, sqlalchemy-utils +six==1.11.0 # via bleach, cryptography, isodate, pathlib2, polyline, pydruid, python-dateutil, sqlalchemy-utils, wtforms-json sqlalchemy-utils==0.32.21 sqlalchemy==1.2.2 sqlparse==0.2.4 @@ -69,4 +69,5 @@ urllib3==1.22 # via requests, selenium vine==1.1.4 # via amqp webencodings==0.5.1 # via bleach werkzeug==0.14.1 # via flask -wtforms==2.2.1 # via flask-wtf +wtforms-json==0.3.3 +wtforms==2.2.1 # via flask-wtf, wtforms-json diff --git a/setup.py b/setup.py index 9d3f700536fe1..fa82e10814c00 100644 --- a/setup.py +++ b/setup.py @@ -104,6 +104,7 @@ def get_git_sha(): 'sqlalchemy-utils', 'sqlparse', 'unicodecsv', + 'wtforms-json', ], extras_require={ 'cors': ['flask-cors>=2.0.0'], diff --git a/superset/__init__.py b/superset/__init__.py index 791e20bcadd07..8a03337c067ee 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -28,6 +28,7 @@ from flask_migrate import Migrate from flask_wtf.csrf import CSRFProtect from werkzeug.contrib.fixers import ProxyFix +import wtforms_json from superset import config from superset.connectors.connector_registry import ConnectorRegistry @@ -35,6 +36,8 @@ from superset.utils.core import ( get_update_perms_flag, pessimistic_connection_handling, setup_cache) +wtforms_json.init() + APP_DIR = os.path.dirname(__file__) CONFIG_MODULE = os.environ.get('SUPERSET_CONFIG', 'superset.config') diff --git a/superset/migrations/versions/c617da68de7d_form_nullable.py b/superset/migrations/versions/c617da68de7d_form_nullable.py new file mode 100644 index 0000000000000..b2281e82dc584 --- /dev/null +++ b/superset/migrations/versions/c617da68de7d_form_nullable.py @@ -0,0 +1,191 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""form nullable + +Revision ID: c617da68de7d +Revises: 18dc26817ad2 +Create Date: 2018-07-19 23:41:32.631556 + +""" + +# revision identifiers, used by Alembic. +revision = 'c617da68de7d' +down_revision = '18dc26817ad2' + +from alembic import op +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, String, Text + +from superset import db +from superset.utils.core import MediumText + +Base = declarative_base() + + +class BaseColumnMixin(object): + id = Column(Integer, primary_key=True) + column_name = Column(String(255)) + description = Column(Text) + type = Column(String(32)) + verbose_name = Column(String(1024)) + + +class BaseDatasourceMixin(object): + id = Column(Integer, primary_key=True) + description = Column(Text) + + +class BaseMetricMixin(object): + id = Column(Integer, primary_key=True) + d3format = Column(String(128)) + description = Column(Text) + metric_name = Column(String(512)) + metric_type = Column(String(32)) + verbose_name = Column(String(1024)) + warning_text = Column(Text) + + +class Annotation(Base): + __tablename__ = 'annotation' + + id = Column(Integer, primary_key=True) + long_descr = Column(Text) + json_metadata = Column(Text) + short_descr = Column(String(500)) + + +class Dashboard(Base): + __tablename__ = 'dashboards' + + id = Column(Integer, primary_key=True) + css = Column(Text) + dashboard_title = Column(String(500)) + description = Column(Text) + json_metadata = Column(Text) + position_json = Column(MediumText()) + slug = Column(String(255)) + + +class Database(Base): + __tablename__ = 'dbs' + + id = Column(Integer, primary_key=True) + database_name = Column(String(250)) + extra = Column(Text) + force_ctas_schema = Column(String(250)) + sqlalchemy_uri = Column(String(1024)) + verbose_name = Column(String(250)) + + +class DruidCluster(Base): + __tablename__ = 'clusters' + + id = Column(Integer, primary_key=True) + broker_host = Column(String(255)) + broker_endpoint = Column(String(255)) + cluster_name = Column(String(250)) + verbose_name = Column(String(250)) + + +class DruidColumn(BaseColumnMixin, Base): + __tablename__ = 'columns' + + dimension_spec_json = Column(Text) + + +class DruidDatasource(BaseDatasourceMixin, Base): + __tablename__ = 'datasources' + + datasource_name = Column(String(255)) + default_endpoint = Column(Text) + fetch_values_from = Column(String(100)) + + +class DruidMetric(BaseMetricMixin, Base): + __tablename__ = 'metrics' + + json = Column(Text) + + +class Slice(Base): + __tablename__ = 'slices' + + id = Column(Integer, primary_key=True) + description = Column(Text) + params = Column(Text) + slice_name = Column(String(250)) + viz_type = Column(String(250)) + + +class SqlaTable(BaseDatasourceMixin, Base): + __tablename__ = 'tables' + + default_endpoint = Column(MediumText()) + fetch_values_predicate = Column(String(1000)) + main_dttm_col = Column(String(250)) + schema = Column(String(255)) + sql = Column(Text) + table_name = Column(String(250)) + template_params = Column(Text) + + +class SqlMetric(BaseMetricMixin, Base): + __tablename__ = 'sql_metrics' + + expression = Column(Text) + + +class TableColumn(BaseColumnMixin, Base): + __tablename__ = 'table_columns' + + database_expression = Column(String(255)) + expression = Column(Text) + python_date_format = Column(String(255)) + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + tables = [ + Annotation, + Dashboard, + Database, + DruidCluster, + DruidColumn, + DruidDatasource, + DruidMetric, + Slice, + SqlaTable, + SqlMetric, + TableColumn, + ] + + for table in tables: + for record in session.query(table).all(): + for col in record.__table__.columns.values(): + if not col.primary_key: + if getattr(record, col.name) == '': + setattr(record, col.name, None) + + session.commit() + + session.close() + + +def downgrade(): + pass