[WIP] Nosetest improvements

superset/cli.py

@@ -140,55 +140,95 @@ def version(verbose):
 
 
 def load_examples_run(load_test_data):
-    print('Loading examples into {}'.format(db))
-
-    data.load_css_templates()
+    #raise Exception("Where am I coming from?")
+    #import sys 
+    #print(sys._getframe(1).f_code.co_name)
+    #print(sys._getframe(2).f_code.co_name)
+    #print(sys._getframe(3).f_code.co_name)
+    #print("^^^^")
+    #shave the time it spends loading and then parallelize the tests. 
+
+    #do we need this? 
+    # parallelize the loading...
+
+    #print('Loading examples into {}'.format(db))
+    #start_css_tmp = utils.now_as_float()
+    #data.load_css_templates()
+    #print("took {}".format(utils.now_as_float() - start_css_tmp))
 
     print('Loading energy related dataset')
+    start_energy = utils.now_as_float()
     data.load_energy()
+    print("took {}".format(utils.now_as_float() - start_energy))
 
     print("Loading [World Bank's Health Nutrition and Population Stats]")
+    start_health = utils.now_as_float()
     data.load_world_bank_health_n_pop()
+    energy_time = utils.now_as_float() - start_health
 
     print('Loading [Birth names]')
+    start_birth = utils.now_as_float()
     data.load_birth_names()
-
-    print('Loading [Random time series data]')
-    data.load_random_time_series_data()
-
-    print('Loading [Random long/lat data]')
-    data.load_long_lat_data()
-
-    print('Loading [Country Map data]')
-    data.load_country_map_data()
-
-    print('Loading [Multiformat time series]')
-    data.load_multiformat_time_series_data()
-
-    print('Loading [Paris GeoJson]')
-    data.load_paris_iris_geojson()
-
-    print('Loading [San Francisco population polygons]')
-    data.load_sf_population_polygons()
-
-    print('Loading [Flights data]')
-    data.load_flights()
-
-    print('Loading [BART lines]')
-    data.load_bart_lines()
-
-    print('Loading [Multi Line]')
-    data.load_multi_line()
-
-    print('Loading [Misc Charts] dashboard')
-    data.load_misc_dashboard()
-
-    if load_test_data:
-        print('Loading [Unicode test data]')
-        data.load_unicode_test_data()
-
-    print('Loading DECK.gl demo')
-    data.load_deck_dash()
+    print(utils.now_as_float() - start_birth)
+
+    #print('Loading [Random time series data]')
+    #start_ts = utils.now_as_float()
+    #data.load_random_time_series_data()
+    #print("took {}".format(utils.now_as_float() - start_ts))
+
+    #print('Loading [Random long/lat data]')
+    #start_lola = utils.now_as_float()
+    #data.load_long_lat_data()
+    #print("took {}".format(utils.now_as_float() - start_lola))
+
+    #print('Loading [Country Map data]')
+    #start_map = utils.now_as_float()
+    #data.load_country_map_data()
+    #print("took {}".format(utils.now_as_float() - start_map))
+
+    #print('Loading [Multiformat time series]')
+    #start_mts = utils.now_as_float()
+    #data.load_multiformat_time_series_data()
+    #print("took {}".format(utils.now_as_float() - start_mts))
+
+    #print('Loading [Paris GeoJson]')
+    #start_geo = utils.now_as_float()
+    #data.load_paris_iris_geojson()
+    #print("took {}".format(utils.now_as_float() - start_geo))
+
+    #print('Loading [San Francisco population polygons]')
+    #start_polygons = utils.now_as_float()
+    #data.load_sf_population_polygons()
+    #print("took {}".format(utils.now_as_float() - start_polygons))
+
+    #print('Loading [Flights data]')
+    #start_flights = utils.now_as_float()
+    #data.load_flights()
+    #print("took {}".format(utils.now_as_float() - start_flights))
+
+    #print('Loading [BART lines]')
+    #start_bart = utils.now_as_float()
+    #data.load_bart_lines()
+    #print("took {}".format(utils.now_as_float() - start_bart))
+
+    #print('Loading [Multi Line]')
+    #start_ml = utils.now_as_float()
+    #data.load_multi_line()
+    #print("took {}".format(utils.now_as_float() - start_ml))
+
+    #print('Loading [Misc Charts] dashboard')
+    #start_misc = utils.now_as_float()
+    #data.load_misc_dashboard()
+    #print("took {}".format(utils.now_as_float() - start_misc))
+
+    #if load_test_data:
+    #    print('Loading [Unicode test data]')
+    #    data.load_unicode_test_data()
+
+    #print('Loading DECK.gl demo')
+    #start_deckgl = utils.now_as_float()
+    #data.load_deck_dash()
+    #print("took {}".format(utils.now_as_float() - start_deckgl))
 
 
 @app.cli.command()

tests/access_tests.py

@@ -87,6 +87,7 @@ def create_access_request(session, ds_type, ds_name, role_name, user_name):
 class RequestAccessTests(SupersetTestCase):
 
     requires_examples = False
+    #_multiprocess_shared_ = True
 
     @classmethod
     def setUpClass(cls):
@@ -170,10 +171,11 @@ def test_override_role_permissions_druid_and_table(self):
         self.assertEquals(3, len(perms))
 
     def test_override_role_permissions_drops_absent_perms(self):
+        #use energy usage instead?....
         override_me = security_manager.find_role('override_me')
         override_me.permissions.append(
             security_manager.find_permission_view_menu(
-                view_menu_name=self.get_table_by_name('long_lat').perm,
+                view_menu_name=self.get_table_by_name('energy_usage').perm,
                 permission_name='datasource_access'),
         )
         db.session.flush()
@@ -259,9 +261,9 @@ def test_clean_requests_after_db_grant(self):
 
         gamma_user = security_manager.find_user(username='gamma')
         access_request1 = create_access_request(
-            session, 'table', 'long_lat', TEST_ROLE_1, 'gamma')
+            session, 'table', 'energy_usage', TEST_ROLE_1, 'gamma')
         create_access_request(
-            session, 'table', 'long_lat', TEST_ROLE_2, 'gamma2')
+            session, 'table', 'energy_usage', TEST_ROLE_2, 'gamma2')
         ds_1_id = access_request1.datasource_id
         # gamma gets granted database access
         database = session.query(models.Database).first()
@@ -359,9 +361,9 @@ def test_approve(self, mock_send_mime):
             # Case 2. Extend the role to have access to the table
 
             access_request2 = create_access_request(
-                session, 'table', 'long_lat', TEST_ROLE_NAME, 'gamma')
+                session, 'table', 'energy_usage', TEST_ROLE_NAME, 'gamma')
             ds_2_id = access_request2.datasource_id
-            long_lat_perm = access_request2.datasource.perm
+            energy_usage_perm = access_request2.datasource.perm
 
             self.client.get(EXTEND_ROLE_REQUEST.format(
                 'table', access_request2.datasource_id, 'gamma', TEST_ROLE_NAME))
@@ -377,13 +379,13 @@ def test_approve(self, mock_send_mime):
                 '[Superset] Access to the datasource {} was granted'.format(
                     self.get_table(ds_2_id).full_name), call_args[2]['Subject'])
             self.assertIn(TEST_ROLE_NAME, call_args[2].as_string())
-            self.assertIn('long_lat', call_args[2].as_string())
+            self.assertIn('energy_usage', call_args[2].as_string())
 
             # request was removed
             self.assertFalse(access_requests)
-            # table_role was extended to grant access to the long_lat table/
+            # table_role was extended to grant access to the energy_usage table/
             perm_view = security_manager.find_permission_view_menu(
-                'datasource_access', long_lat_perm)
+                'datasource_access', energy_usage_perm)
             TEST_ROLE = security_manager.find_role(TEST_ROLE_NAME)
             self.assertIn(perm_view, TEST_ROLE.permissions)
 

tests/base_tests.py

@@ -28,12 +28,14 @@ class SupersetTestCase(unittest.TestCase):
     requires_examples = False
     examples_loaded = False
 
+    _multiprocess_shared_ = True
+
     def __init__(self, *args, **kwargs):
         if (
             self.requires_examples and
             not os.environ.get('examples_loaded')
         ):
-            logging.info('Loading examples')
+            logging.info('Loading examples in test case')
             cli.load_examples_run(load_test_data=True)
             logging.info('Done loading examples')
             security_manager.sync_role_definitions()

tests/cache_tests.py

@@ -13,6 +13,8 @@
 
 class CacheTests(SupersetTestCase):
 
+    _multiprocess_shared_ = True
+
     def __init__(self, *args, **kwargs):
         super(CacheTests, self).__init__(*args, **kwargs)
 

tests/celery_tests.py

@@ -37,6 +37,8 @@ class CeleryConfig(object):
 
 class UtilityFunctionTests(SupersetTestCase):
 
+    _multiprocess_shared_ = True
+
     # TODO(bkyryliuk): support more cases in CTA function.
     def test_create_table_as(self):
         q = SupersetQuery('SELECT * FROM outer_space;')

tests/core_tests.py

@@ -36,6 +36,7 @@
 class CoreTests(SupersetTestCase):
 
     requires_examples = True
+    _multiprocess_can_split_ = True
 
     def __init__(self, *args, **kwargs):
         super(CoreTests, self).__init__(*args, **kwargs)
@@ -378,7 +379,7 @@ def test_warm_up_cache(self):
 
         data = self.get_json_resp(
             '/superset/warm_up_cache?table_name=energy_usage&db_name=main')
-        assert len(data) == 4
+        assert len(data) == 3
 
     def test_shortner(self):
         self.login(username='admin')

tests/dashboard_tests.py

@@ -19,6 +19,7 @@
 class DashboardTests(SupersetTestCase):
 
     requires_examples = True
+    _multiprocess_can_split_ = True
 
     def __init__(self, *args, **kwargs):
         super(DashboardTests, self).__init__(*args, **kwargs)
@@ -211,7 +212,7 @@ def test_add_slices(self, username='admin'):
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         new_slice = db.session.query(models.Slice).filter_by(
-            slice_name='Mapbox Long/Lat').first()
+            slice_name='Energy Force Layout').first()
         existing_slice = db.session.query(models.Slice).filter_by(
             slice_name='Name Cloud').first()
         data = {
@@ -225,15 +226,15 @@ def test_add_slices(self, username='admin'):
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         new_slice = db.session.query(models.Slice).filter_by(
-            slice_name='Mapbox Long/Lat').first()
+            slice_name='Energy Force Layout').first()
         assert new_slice in dash.slices
         assert len(set(dash.slices)) == len(dash.slices)
 
         # cleaning up
         dash = db.session.query(models.Dashboard).filter_by(
             slug='births').first()
         dash.slices = [
-            o for o in dash.slices if o.slice_name != 'Mapbox Long/Lat']
+            o for o in dash.slices if o.slice_name != 'Energy Force Layout']
         db.session.commit()
 
     def test_remove_slices(self, username='admin'):

tests/dataframe_test.py

@@ -12,6 +12,8 @@
 
 
 class SupersetDataFrameTestCase(SupersetTestCase):
+    _multiprocess_can_split_ = True
+
     def test_dedup(self):
         self.assertEquals(
             dedup(['foo', 'bar']),

tests/datasource_tests.py

@@ -14,6 +14,7 @@
 class DatasourceTests(SupersetTestCase):
 
     requires_examples = True
+    _multiprocess_can_split_ = True
 
     def __init__(self, *args, **kwargs):
         super(DatasourceTests, self).__init__(*args, **kwargs)

tests/db_engine_specs_test.py

@@ -18,6 +18,8 @@
 
 
 class DbEngineSpecsTestCase(SupersetTestCase):
+    _multiprocess_can_split_ = True
+
     def test_0_progress(self):
         log = """
             17/02/07 18:26:27 INFO log.PerfLogger: <PERFLOG method=compile from=org.apache.hadoop.hive.ql.Driver>

tests/dict_import_export_tests.py

@@ -26,6 +26,8 @@
 class DictImportExportTests(SupersetTestCase):
     """Testing export import functionality for dashboards"""
 
+    _multiprocess_can_split_ = True
+
     def __init__(self, *args, **kwargs):
         super(DictImportExportTests, self).__init__(*args, **kwargs)
 

tests/druid_tests.py

@@ -82,6 +82,7 @@ def __reduce__(self):
 class DruidTests(SupersetTestCase):
 
     """Testing interactions with Druid"""
+    _multiprocess_shared_ = True
 
     def __init__(self, *args, **kwargs):
         super(DruidTests, self).__init__(*args, **kwargs)

tests/email_tests.py

@@ -19,6 +19,9 @@
 
 
 class EmailSmtpTest(unittest.TestCase):
+
+    #_multiprocess_shared_ = True
+
     def setUp(self):
         app.config['smtp_ssl'] = False
 

tests/model_tests.py

@@ -15,6 +15,7 @@
 
 
 class DatabaseModelTestCase(SupersetTestCase):
+    _multiprocess_shared_ = True
 
     def test_database_schema_presto(self):
         sqlalchemy_uri = 'presto://presto.airbnb.io:8080/hive/default'
@@ -79,7 +80,7 @@ def test_database_impersonate_user(self):
 
     def test_select_star(self):
         main_db = get_main_database(db.session)
-        table_name = 'bart_lines'
+        table_name = 'energy_usage'
         sql = main_db.select_star(
             table_name, show_cols=False, latest_partition=False)
         expected = textwrap.dedent("""\
@@ -91,11 +92,10 @@ def test_select_star(self):
         sql = main_db.select_star(
             table_name, show_cols=True, latest_partition=False)
         expected = textwrap.dedent("""\
-        SELECT color,
-               name,
-               path_json,
-               polyline
-        FROM bart_lines
+        SELECT source,
+               target,
+               value
+        FROM energy_usage
         LIMIT 100""".format(**locals()))
         assert sql.startswith(expected)
 
@@ -129,6 +129,8 @@ def test_multi_statement(self):
 
 
 class SqlaTableModelTestCase(SupersetTestCase):
+    _multiprocess_shared_ = True
+
 
     def test_get_timestamp_expression(self):
         tbl = self.get_table_by_name('birth_names')

tests/security_tests.py

@@ -18,6 +18,8 @@ def get_perm_tuples(role_name):
 class RolePermissionTests(SupersetTestCase):
     """Testing export import functionality for dashboards"""
 
+    _multiprocess_can_split_ = True
+
     def __init__(self, *args, **kwargs):
         super(RolePermissionTests, self).__init__(*args, **kwargs)
 

tests/sql_parse_tests.py

@@ -8,8 +8,11 @@
 
 from superset import sql_parse
 
+#can parallelize
+
 
 class SupersetTestCase(unittest.TestCase):
+    _multiprocess_can_split_ = True
 
     def extract_tables(self, query):
         sq = sql_parse.SupersetQuery(query)