Merge branch 'master' into john-bodley--fix-updating

.codecov.yml

@@ -16,7 +16,7 @@ coverage:
         target: auto
         threshold: 0%
       core-packages-ts:
-        target: 95%
+        target: 100%
         paths:
           - 'superset-frontend/packages'
           - '!superset-frontend/packages/**/*.jsx'

CONTRIBUTING.md

@@ -1339,9 +1339,9 @@ More information on the async query feature can be found in [SIP-39](https://git
 
 Chart parameters are stored as a JSON encoded string the `slices.params` column and are often referenced throughout the code as form-data. Currently the form-data is neither versioned nor typed as thus is somewhat free-formed. Note in the future there may be merit in using something like [JSON Schema](https://json-schema.org/) to both annotate and validate the JSON object in addition to using a Mypy `TypedDict` (introduced in Python 3.8) for typing the form-data in the backend. This section serves as a potential primer for that work.
 
-The following tables provide a non-exhausive list of the various fields which can be present in the JSON object grouped by the Explorer pane sections. These values were obtained by extracting the distinct fields from a legacy deployment consisting of tens of thousands of charts and thus some fields may be missing whilst others may be deprecated.
+The following tables provide a non-exhaustive list of the various fields which can be present in the JSON object grouped by the Explorer pane sections. These values were obtained by extracting the distinct fields from a legacy deployment consisting of tens of thousands of charts and thus some fields may be missing whilst others may be deprecated.
 
-Note not all fields are correctly categorized. The fields vary based on visualization type and may apprear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsuccessful and thus future work may be required to clean up the form-data.
+Note not all fields are correctly categorized. The fields vary based on visualization type and may appear in different sections depending on the type. Verified deprecated columns may indicate a missing migration and/or prior migrations which were unsuccessful and thus future work may be required to clean up the form-data.
 
 ### Datasource & Chart Type
 

Makefile

@@ -46,6 +46,9 @@ superset:
 	# Load some data to play with
 	superset load-examples
 
+	# Install node packages
+	cd superset-frontend; npm install
+
 update: update-py update-js
 
 update-py:

UPDATING.md

@@ -36,6 +36,10 @@ assists people when migrating to a new version.
 ### Potential Downtime
 
 - [16756](https://github.com/apache/incubator-superset/pull/16756): a change which renames the `dbs.allow_csv_upload` column to `dbs.allow_file_upload` via a (potentially locking) DDL operation.
+- [17539](https://github.com/apache/superset/pull/17539): all Superset CLI commands
+  (init, load_examples and etc) require setting the FLASK_APP environment variable
+  (which is set by default when .flaskenv is loaded)
+- [17360](https://github.com/apache/superset/pull/17360): changes the column type from `VARCHAR(32)` to `TEXT` in table `table_columns`, potentially requiring a table lock on MySQL dbs or taking some time to complete on large deployments.
 
 ### Deprecations
 

requirements/base.txt

@@ -77,7 +77,7 @@ flask==1.1.4
     #   flask-openid
     #   flask-sqlalchemy
     #   flask-wtf
-flask-appbuilder==3.4.1rc2
+flask-appbuilder==3.4.1
     # via apache-superset
 flask-babel==1.0.0
     # via flask-appbuilder

scripts/python_tests.sh

@@ -18,6 +18,11 @@
 #
 set -e
 
+# Temporary fix, probably related with https://bugs.launchpad.net/ubuntu/+source/opencv/+bug/1890170
+# MySQL was failling with:
+# from . import _mysql
+# ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
+export LD_PRELOAD=/lib/x86_64-linux-gnu/libstdc++.so.6
 export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.integration_tests.superset_test_config}
 export SUPERSET_TESTENV=true
 echo "Superset config module: $SUPERSET_CONFIG"

setup.py

@@ -75,7 +75,7 @@ def get_git_sha() -> str:
         "cryptography>=3.3.2",
         "deprecation>=2.1.0, <2.2.0",
         "flask>=1.1.0, <2.0.0",
-        "flask-appbuilder>=3.4.1rc2, <4.0.0",
+        "flask-appbuilder>=3.4.1, <4.0.0",
         "flask-caching>=1.10.0",
         "flask-compress",
         "flask-talisman",

superset-frontend/jest.config.js

@@ -56,9 +56,10 @@ module.exports = {
   coverageReporters: ['lcov', 'json-summary', 'html'],
   transform: {
     '^.+\\.jsx?$': 'babel-jest',
-    // ts-jest can't load plugin 'babel-plugin-typescript-to-proptypes'
-    'reactify\\.tsx$': 'babel-jest',
-    '^.+\\.tsx?$': 'ts-jest',
+    // ts-jest doesn't work with `--coverage`. @superset-ui/core should
+    // 100% coverage, so we use babel-jest in packages and plugins.
+    '(plugins|packages)\\/.+\\.tsx?$': 'babel-jest',
+    '(((?!(plugins|packages)).)*)\\/.+\\.tsx?$': 'ts-jest',
   },
   moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
   snapshotSerializers: ['@emotion/jest/enzyme-serializer'],

superset-frontend/packages/superset-ui-core/src/color/CategoricalColorNamespace.ts

@@ -40,12 +40,7 @@ export default class CategoricalColorNamespace {
   getScale(schemeId?: string) {
     const id = schemeId ?? getCategoricalSchemeRegistry().getDefaultKey() ?? '';
     const scheme = getCategoricalSchemeRegistry().get(id);
-    const newScale = new CategoricalColorScale(
-      scheme?.colors ?? [],
-      this.forcedItems,
-    );
-
-    return newScale;
+    return new CategoricalColorScale(scheme?.colors ?? [], this.forcedItems);
   }
 
   /**

superset-frontend/packages/superset-ui-core/src/connection/SupersetClientClass.ts

@@ -33,13 +33,6 @@ import {
 } from './types';
 import { DEFAULT_FETCH_RETRY_OPTIONS, DEFAULT_BASE_URL } from './constants';
 
-function redirectUnauthorized() {
-  // the next param will be picked by flask to redirect the user after the login
-  setTimeout(() => {
-    window.location.href = `/login?next=${window.location.href}`;
-  });
-}
-
 export default class SupersetClientClass {
   credentials: Credentials;
 
@@ -159,8 +152,8 @@ export default class SupersetClientClass {
       timeout: timeout ?? this.timeout,
       fetchRetryOptions: fetchRetryOptions ?? this.fetchRetryOptions,
     }).catch(res => {
-      if (res && res.status === 401) {
-        redirectUnauthorized();
+      if (res?.status === 401) {
+        this.redirectUnauthorized();
       }
       return Promise.reject(res);
     });
@@ -226,4 +219,8 @@ export default class SupersetClientClass {
       endpoint[0] === '/' ? endpoint.slice(1) : endpoint
     }`;
   }
+
+  redirectUnauthorized() {
+    window.location.href = `/login?next=${window.location.href}`;
+  }
 }

superset-frontend/packages/superset-ui-core/src/utils/random.ts

@@ -16,15 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import seedrandom from 'seedrandom';
-
-let random = seedrandom('superset-ui');
+import _seedrandom from 'seedrandom';
 
 export function seed(seed: string) {
-  random = seedrandom(seed);
-  return random;
+  return _seedrandom(seed);
 }
 
 export function seedRandom() {
-  return random();
+  return _seedrandom('superset-ui')();
 }

superset-frontend/packages/superset-ui-core/test/connection/SupersetClientClass.test.ts

@@ -17,10 +17,7 @@
  * under the License.
  */
 import fetchMock from 'fetch-mock';
-import {
-  SupersetClientClass,
-  ClientConfig,
-} from '@superset-ui/core/src/connection';
+import { SupersetClientClass, ClientConfig, CallApi } from '@superset-ui/core';
 import { LOGIN_GLOB } from './fixtures/constants';
 
 describe('SupersetClientClass', () => {
@@ -321,7 +318,7 @@ describe('SupersetClientClass', () => {
       await client.init();
       await client.get({ url: mockGetUrl });
 
-      const fetchRequest = fetchMock.calls(mockGetUrl)[0][1];
+      const fetchRequest = fetchMock.calls(mockGetUrl)[0][1] as CallApi;
       expect(fetchRequest.mode).toBe(clientConfig.mode);
       expect(fetchRequest.credentials).toBe(clientConfig.credentials);
       expect(fetchRequest.headers).toEqual(
@@ -378,7 +375,7 @@ describe('SupersetClientClass', () => {
         await client.init();
         await client.get({ url: mockGetUrl, ...overrideConfig });
 
-        const fetchRequest = fetchMock.calls(mockGetUrl)[0][1];
+        const fetchRequest = fetchMock.calls(mockGetUrl)[0][1] as CallApi;
         expect(fetchRequest.mode).toBe(overrideConfig.mode);
         expect(fetchRequest.credentials).toBe(overrideConfig.credentials);
         expect(fetchRequest.headers).toEqual(
@@ -423,7 +420,7 @@ describe('SupersetClientClass', () => {
         await client.init();
         await client.post({ url: mockPostUrl, ...overrideConfig });
 
-        const fetchRequest = fetchMock.calls(mockPostUrl)[0][1];
+        const fetchRequest = fetchMock.calls(mockPostUrl)[0][1] as CallApi;
 
         expect(fetchRequest.mode).toBe(overrideConfig.mode);
         expect(fetchRequest.credentials).toBe(overrideConfig.credentials);
@@ -454,7 +451,8 @@ describe('SupersetClientClass', () => {
         await client.init();
         await client.post({ url: mockPostUrl, postPayload });
 
-        const formData = fetchMock.calls(mockPostUrl)[0][1].body as FormData;
+        const fetchRequest = fetchMock.calls(mockPostUrl)[0][1] as CallApi;
+        const formData = fetchRequest.body as FormData;
 
         expect(fetchMock.calls(mockPostUrl)).toHaveLength(1);
         Object.entries(postPayload).forEach(([key, value]) => {
@@ -470,7 +468,8 @@ describe('SupersetClientClass', () => {
         await client.init();
         await client.post({ url: mockPostUrl, postPayload, stringify: false });
 
-        const formData = fetchMock.calls(mockPostUrl)[0][1].body as FormData;
+        const fetchRequest = fetchMock.calls(mockPostUrl)[0][1] as CallApi;
+        const formData = fetchRequest.body as FormData;
 
         expect(fetchMock.calls(mockPostUrl)).toHaveLength(1);
         Object.entries(postPayload).forEach(([key, value]) => {
@@ -479,4 +478,36 @@ describe('SupersetClientClass', () => {
       });
     });
   });
+
+  it('should redirect Unauthorized', async () => {
+    const mockRequestUrl = 'https://host/get/url';
+    const { location } = window;
+    // @ts-ignore
+    delete window.location;
+    // @ts-ignore
+    window.location = { href: mockRequestUrl };
+    const authSpy = jest
+      .spyOn(SupersetClientClass.prototype, 'ensureAuth')
+      .mockImplementation();
+    const rejectValue = { status: 401 };
+    fetchMock.get(mockRequestUrl, () => Promise.reject(rejectValue), {
+      overwriteRoutes: true,
+    });
+
+    const client = new SupersetClientClass({});
+
+    let error;
+    try {
+      await client.request({ url: mockRequestUrl, method: 'GET' });
+    } catch (err) {
+      error = err;
+    } finally {
+      const redirectURL = window.location.href;
+      expect(redirectURL).toBe(`/login?next=${mockRequestUrl}`);
+      expect(error.status).toBe(401);
+    }
+
+    authSpy.mockReset();
+    window.location = location;
+  });
 });

superset-frontend/packages/superset-ui-core/test/translation/Translator.test.ts

@@ -17,16 +17,16 @@
  * under the License.
  */
 
-import { logging } from '@superset-ui/core';
-import Translator from '@superset-ui/core/src/translation/Translator';
 import {
+  logging,
   configure,
   t,
   tn,
   addLocaleData,
   addTranslation,
   addTranslations,
-} from '@superset-ui/core/src/translation/TranslatorSingleton';
+} from '@superset-ui/core';
+import Translator from '../../src/translation/Translator';
 import languagePackZh from './languagePacks/zh';
 import languagePackEn from './languagePacks/en';
 

superset-frontend/packages/superset-ui-core/test/translation/TranslatorSingleton.test.ts

@@ -19,13 +19,8 @@
 
 /* eslint no-console: 0 */
 import mockConsole from 'jest-mock-console';
-import Translator from '@superset-ui/core/src/translation/Translator';
-import {
-  configure,
-  resetTranslation,
-  t,
-  tn,
-} from '@superset-ui/core/src/translation/TranslatorSingleton';
+import { configure, resetTranslation, t, tn } from '@superset-ui/core';
+import Translator from '../../src/translation/Translator';
 
 import languagePackEn from './languagePacks/en';
 import languagePackZh from './languagePacks/zh';
@@ -76,4 +71,16 @@ describe('TranslatorSingleton', () => {
       });
     });
   });
+  it('should be reset translation setting', () => {
+    configure();
+    expect(t('second')).toEqual('second');
+
+    resetTranslation();
+    const restoreConsole = mockConsole();
+    expect(t('second')).toEqual('second');
+    resetTranslation();
+    expect(t('second')).toEqual('second');
+    expect(console.warn).toBeCalledTimes(2);
+    restoreConsole();
+  });
 });

superset-frontend/packages/superset-ui-core/test/translation/index.test.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { configure, t, tn } from '@superset-ui/core/src/translation';
+import { configure, t, tn } from '@superset-ui/core';
 
 describe('index', () => {
   it('exports configure()', () => {

superset-frontend/packages/superset-ui-core/test/translation/languagePacks/en.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { LanguagePack } from '@superset-ui/core/src/translation';
+import { LanguagePack } from '@superset-ui/core';
 
 const languagePack: LanguagePack = {
   domain: 'superset',

superset-frontend/packages/superset-ui-core/test/translation/languagePacks/zh.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { LanguagePack } from '@superset-ui/core/src/translation';
+import { LanguagePack } from '@superset-ui/core';
 
 const languagePack: LanguagePack = {
   domain: 'superset',

superset-frontend/packages/superset-ui-core/test/utils/logging.test.ts

@@ -21,40 +21,41 @@
 describe('logging', () => {
   beforeEach(() => {
     jest.resetModules();
-    // Explicit is better than implicit
-    console.warn = console.error = function mockedConsole(message) {
-      throw new Error(message);
-    };
+    jest.resetAllMocks();
   });
+
   it('should pipe to `console` methods', () => {
-    const { logging } = require('@superset-ui/core/src');
+    const { logging } = require('@superset-ui/core');
 
+    jest.spyOn(logging, 'debug').mockImplementation();
+    jest.spyOn(logging, 'log').mockImplementation();
+    jest.spyOn(logging, 'info').mockImplementation();
     expect(() => {
       logging.debug();
       logging.log();
       logging.info();
     }).not.toThrow();
-    expect(() => {
-      logging.warn('warn');
-    }).toThrow('warn');
-    expect(() => {
-      logging.error('error');
-    }).toThrow('error');
 
-    // to support: npx jest --silent
-    const spy = jest.spyOn(logging, 'trace');
-    spy.mockImplementation(() => {
+    jest.spyOn(logging, 'warn').mockImplementation(() => {
+      throw new Error('warn');
+    });
+    expect(() => logging.warn()).toThrow('warn');
+
+    jest.spyOn(logging, 'error').mockImplementation(() => {
+      throw new Error('error');
+    });
+    expect(() => logging.error()).toThrow('error');
+
+    jest.spyOn(logging, 'trace').mockImplementation(() => {
       throw new Error('Trace:');
     });
-    expect(() => {
-      logging.trace();
-    }).toThrow('Trace:');
-    spy.mockRestore();
+    expect(() => logging.trace()).toThrow('Trace:');
   });
+
   it('should use noop functions when console unavailable', () => {
     const { console } = window;
     Object.assign(window, { console: undefined });
-    const { logging } = require('@superset-ui/core/src');
+    const { logging } = require('@superset-ui/core');
 
     afterAll(() => {
       Object.assign(window, { console });