Getting ready to gut panoramix

app/models.py

@@ -3,6 +3,8 @@
 from datetime import timedelta
 from flask.ext.appbuilder.models.mixins import AuditMixin, FileColumn
 from sqlalchemy import Column, Integer, String, ForeignKey, Text, Boolean, DateTime
+from sqlalchemy import create_engine, MetaData
+from sqlalchemy import Table as sqlaTable
 from sqlalchemy.orm import relationship
 from app import get_session
 from dateutil.parser import parse
@@ -11,6 +13,102 @@
 import json
 import requests
 
+from app import db
+
+class Queryable(object):
+    @property
+    def column_names(self):
+        return sorted([c.column_name for c in self.columns])
+
+    @property
+    def groupby_column_names(self):
+        return sorted([c.column_name for c in self.columns if c.groupby])
+
+    @property
+    def filterable_column_names(self):
+        return sorted([c.column_name for c in self.columns if c.filterable])
+
+class Database(Model, AuditMixin):
+    __tablename__ = 'databases'
+    id = Column(Integer, primary_key=True)
+    database_name = Column(String(256), unique=True)
+    sqlalchemy_uri = Column(String(1024))
+
+    def __repr__(self):
+        return self.database_name
+
+
+class Table(Model, AuditMixin, Queryable):
+    __tablename__ = 'tables'
+    id = Column(Integer, primary_key=True)
+    table_name = Column(String(256), unique=True)
+    default_endpoint = Column(Text)
+    database_id = Column(
+        String(256), ForeignKey('databases.id'))
+    database = relationship(
+        'Database', backref='tables', foreign_keys=[database_id])
+
+    @property
+    def table_link(self):
+        url = "/panoramix/table/{}/".format(self.id)
+        return '<a href="{url}">{self.table_name}</a>'.format(**locals())
+
+    @property
+    def metrics_combo(self):
+        return sorted(
+            [
+                (
+                    'sum__{}'.format(m.column_name),
+                    'SUM({})'.format(m.column_name),
+                )
+                for m in self.columns if m.sum],
+            key=lambda x: x[1])
+
+    def fetch_metadata(self):
+        engine = create_engine(self.database.sqlalchemy_uri)
+        meta = MetaData()
+        table = sqlaTable(
+            self.table_name, meta, autoload=True, autoload_with=engine)
+        TC = TableColumn
+        for col in table.columns:
+            dbcol = (
+                db.session
+                .query(TC)
+                .filter(TC.table==self)
+                .filter(TC.column_name==col.name)
+                .first()
+            )
+            db.session.flush()
+            if not dbcol:
+                dbcol = TableColumn(column_name=col.name)
+                if str(col.type) in ('VARCHAR', 'STRING'):
+                    dbcol.groupby = True
+                    dbcol.filterable = True
+                self.columns.append(dbcol)
+
+            dbcol.type = str(col.type)
+            db.session.commit()
+
+
+class TableColumn(Model, AuditMixin):
+    __tablename__ = 'table_columns'
+    id = Column(Integer, primary_key=True)
+    table_id = Column(
+        String(256),
+        ForeignKey('tables.id'))
+    table = relationship('Table', backref='columns')
+    column_name = Column(String(256))
+    is_dttm = Column(Boolean, default=True)
+    is_active = Column(Boolean, default=True)
+    type = Column(String(32), default='')
+    groupby = Column(Boolean, default=False)
+    count_distinct = Column(Boolean, default=False)
+    sum = Column(Boolean, default=False)
+    max = Column(Boolean, default=False)
+    min = Column(Boolean, default=False)
+    filterable = Column(Boolean, default=False)
+    description = Column(Text, default='')
+
 
 class Cluster(Model, AuditMixin):
     __tablename__ = 'clusters'
@@ -46,7 +144,7 @@ def refresh_datasources(self):
             #    logging.exception(e)
             #    logging.error("Failed at syncing " + datasource)
 
-class Datasource(Model, AuditMixin):
+class Datasource(Model, AuditMixin, Queryable):
     __tablename__ = 'datasources'
     id = Column(Integer, primary_key=True)
     datasource_name = Column(String(256), unique=True)
@@ -130,17 +228,6 @@ def sync_to_db(cls, name, cluster):
             col_obj.generate_metrics()
         #session.commit()
 
-    @property
-    def column_names(self):
-        return sorted([c.column_name for c in self.columns])
-
-    @property
-    def groupby_column_names(self):
-        return sorted([c.column_name for c in self.columns if c.groupby])
-
-    @property
-    def filterable_column_names(self):
-        return sorted([c.column_name for c in self.columns if c.filterable])
 
 
 class Metric(Model):

app/views.py

@@ -27,6 +27,22 @@ def muldelete(self, items):
         return redirect(self.get_redirect())
 
 
+class TableColumnInlineView(CompactCRUDMixin, ModelView):
+    datamodel = SQLAInterface(models.TableColumn)
+    can_delete = False
+    edit_columns = [
+        'column_name', 'description', 'table', 'groupby', 'filterable',
+        'count_distinct', 'sum', 'min', 'max']
+    list_columns = [
+        'column_name', 'type', 'groupby', 'count_distinct',
+        'sum', 'min', 'max']
+    page_size = 100
+    list_columns = [
+        'column_name', 'type', 'groupby', 'count_distinct',
+        'sum', 'min', 'max']
+appbuilder.add_view_no_menu(TableColumnInlineView)
+
+
 class ColumnInlineView(CompactCRUDMixin, ModelView):
     datamodel = SQLAInterface(models.Column)
     edit_columns = [
@@ -80,6 +96,39 @@ class ClusterModelView(ModelView, DeleteMixin):
     category_icon='fa-cogs',)
 
 
+class DatabaseView(ModelView, DeleteMixin):
+    datamodel = SQLAInterface(models.Database)
+    list_columns = ['database_name']
+    add_columns = ['database_name', 'sqlalchemy_uri']
+    edit_columns = add_columns
+
+appbuilder.add_view(
+    DatabaseView,
+    "Databases",
+    icon="fa-database",
+    category="Admin",
+    category_icon='fa-cogs',)
+
+
+class TableView(ModelView, DeleteMixin):
+    datamodel = SQLAInterface(models.Table)
+    list_columns = ['table_link', 'database']
+    add_columns = ['table_name', 'database']
+    edit_columns = add_columns
+    related_views = [TableColumnInlineView]
+
+    def post_insert(self, table):
+        table.fetch_metadata()
+
+    def post_update(self, table):
+        table.fetch_metadata()
+
+appbuilder.add_view(
+    TableView,
+    "Tables",
+    icon='fa-table',)
+
+
 class DatasourceModelView(ModelView, DeleteMixin):
     datamodel = SQLAInterface(models.Datasource)
     list_columns = [
@@ -101,8 +150,7 @@ def post_update(self, datasource):
 appbuilder.add_view(
     DatasourceModelView,
     "Druid Datasources",
-    icon="fa-cube",
-    category_icon='fa-envelope')
+    icon="fa-cube")
 
 
 @app.route('/health')
@@ -116,6 +164,34 @@ def ping():
 
 
 class Panoramix(BaseView):
+    @has_access
+    @permission_name('tables')
+    @expose("/table/<table_id>/")
+    def table(self, table_id):
+
+        table = (
+            db.session
+            .query(models.Table)
+            .filter_by(id=table_id)
+            .first()
+        )
+        viz_type = request.args.get("viz_type")
+        if not viz_type and table.default_endpoint:
+            return redirect(table.default_endpoint)
+        if not viz_type:
+            viz_type = "table"
+        obj = viz.viz_types[viz_type](
+            table,
+            form_data=request.args, view=self)
+        if request.args.get("json"):
+            return Response(
+                json.dumps(obj.get_query(), indent=4),
+                status=200,
+                mimetype="application/json")
+        if obj.df is None or obj.df.empty:
+            return obj.render_no_data()
+        return obj.render()
+
     @has_access
     @permission_name('datasources')
     @expose("/datasource/<datasource_name>/")