Use manifest hooks for dev server proxy

CONTRIBUTING.md

@@ -75,7 +75,7 @@ little bit helps, and credit will always be given.
     - [Creating a new language dictionary](#creating-a-new-language-dictionary)
   - [Tips](#tips)
     - [Adding a new datasource](#adding-a-new-datasource)
-    - [Creating a new visualization type](#creating-a-new-visualization-type)
+    - [Improving visualizations](#improving-visualizations)
     - [Adding a DB migration](#adding-a-db-migration)
     - [Merging DB migrations](#merging-db-migrations)
     - [SQL Lab Async](#sql-lab-async)
@@ -389,7 +389,7 @@ Make sure your machine meets the [OS dependencies](https://superset.incubator.ap
 
 Developers should use a virtualenv.
 
-```
+```bash
 pip install virtualenv
 ```
 
@@ -726,7 +726,7 @@ In TypeScript/JavaScript, the technique is similar:
 we import `t` (simple translation), `tn` (translation containing a number).
 
 ```javascript
-import { t, tn } from "@superset-ui/translation";
+import { t, tn } from '@superset-ui/translation';
 ```
 
 ### Enabling language selection
@@ -803,11 +803,32 @@ Then, [extract strings for the new language](#extracting-new-strings-for-transla
 
    This means it'll register MyDatasource and MyOtherDatasource in superset.my_models module in the source registry.
 
-### Creating a new visualization type
+### Improving visualizations
+
+Superset is working towards a plugin system where new visualizations can be installed as optional npm packages. To achieve this goal,
+we are not accepting pull requests for new community-contributed visualization types at the moment. However, bugfixes for current visualizations are welcome. To edit the frontend code for visualizations, you will have to check out a copy of [apache-superset/superset-ui-plugins](https://github.com/apache-superset/superset-ui-plugins):
+
+```bash
+git clone https://github.com/apache-superset/superset-ui-plugins.git
+yarn && yarn build
+```
+
+Then use `npm link` to create a symlink of the source code in `superset-frontend/node_modules`:
+
+```bash
+cd incubator-superset/superset-frontend
+npm link ../../superset-ui-plugins/packages/superset-ui-[PLUGIN NAME]
+
+# Or to link all plugin packages:
+# npm link ../../superset-ui-plugins/packages/*
+
+# Start developing
+npm run dev-server
+```
+
+When plugin packages are linked with `npm link`, the dev server will automatically load files from the plugin's `/src` directory.
 
-Here's an example as a Github PR with comments that describe what the
-different sections of the code do:
-https://github.com/apache/incubator-superset/pull/3013
+Note that every time you do `npm install`, you will lose the symlink(s) and may have to run `npm link` again.
 
 ### Adding a DB migration
 
@@ -905,12 +926,14 @@ To do this, you'll need to:
 - Configure a results backend, here's a local `FileSystemCache` example,
   not recommended for production,
   but perfect for testing (stores cache in `/tmp`)
+
   ```python
   from werkzeug.contrib.cache import FileSystemCache
   RESULTS_BACKEND = FileSystemCache('/tmp/sqllab')
   ```
 
-* Start up a celery worker
+- Start up a celery worker
+
   ```shell script
   celery worker --app=superset.tasks.celery_app:app -Ofair
   ```

superset-frontend/package.json

@@ -231,10 +231,10 @@
     "typescript": "^3.8.3",
     "url-loader": "^1.0.1",
     "webpack": "^4.42.0",
-    "webpack-assets-manifest": "^3.1.1",
     "webpack-bundle-analyzer": "^3.6.1",
     "webpack-cli": "^3.3.11",
     "webpack-dev-server": "^3.10.3",
+    "webpack-manifest-plugin": "^2.2.0",
     "webpack-sources": "^1.4.3",
     "yargs": "12 - 15"
   },

superset-frontend/src/visualizations/presets/MainPreset.js

@@ -60,7 +60,7 @@ import {
   LineMultiChartPlugin,
   PieChartPlugin,
   TimePivotChartPlugin,
-} from '@superset-ui/legacy-preset-chart-nvd3/lib';
+} from '@superset-ui/legacy-preset-chart-nvd3';
 import { BoxPlotChartPlugin } from '@superset-ui/preset-chart-xy/esm/legacy';
 import { DeckGLChartPreset } from '@superset-ui/legacy-preset-chart-deckgl';
 

superset-frontend/webpack.config.js

@@ -20,17 +20,17 @@
 const fs = require('fs');
 const path = require('path');
 const webpack = require('webpack');
-const BundleAnalyzerPlugin = require('webpack-bundle-analyzer')
-  .BundleAnalyzerPlugin;
+const { BundleAnalyzerPlugin } = require('webpack-bundle-analyzer');
 const { CleanWebpackPlugin } = require('clean-webpack-plugin');
 const CopyPlugin = require('copy-webpack-plugin');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const OptimizeCSSAssetsPlugin = require('optimize-css-assets-webpack-plugin');
 const SpeedMeasurePlugin = require('speed-measure-webpack-plugin');
 const TerserPlugin = require('terser-webpack-plugin');
-const WebpackAssetsManifest = require('webpack-assets-manifest');
+const ManifestPlugin = require('webpack-manifest-plugin');
 const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin');
 const parsedArgs = require('yargs').argv;
+const getProxyConfig = require('./webpack.proxy-config');
 const packageConfig = require('./package.json');
 
 // input dir
@@ -60,14 +60,39 @@ if (isDevMode) {
 
 const plugins = [
   // creates a manifest.json mapping of name to hashed output used in template files
-  new WebpackAssetsManifest({
-    publicPath: true,
+  new ManifestPlugin({
+    publicPath: output.publicPath,
+    seed: { app: 'superset' },
     // This enables us to include all relevant files for an entry
-    entrypoints: true,
+    generate: (seed, files, entrypoints) => {
+      // Each entrypoint's chunk files in the format of
+      // {
+      //   entry: {
+      //     css: [],
+      //     js: []
+      //   }
+      // }
+      const entryFiles = {};
+      for (const [entry, chunks] of Object.entries(entrypoints)) {
+        entryFiles[entry] = {
+          css: chunks
+            .filter(x => x.endsWith('.css'))
+            .map(x => path.join(output.publicPath, x)),
+          js: chunks
+            .filter(x => x.endsWith('.js'))
+            .map(x => path.join(output.publicPath, x)),
+        };
+      }
+      return {
+        ...seed,
+        // files: filePaths,
+        entrypoints: entryFiles,
+      };
+    },
     // Also write to disk when using devServer
     // instead of only keeping manifest.json in memory
     // This is required to make devServer work with flask.
-    writeToDisk: isDevMode,
+    writeToFileEmit: isDevMode,
   }),
 
   // create fresh dist/ upon build
@@ -127,6 +152,8 @@ const babelLoader = {
   loader: 'babel-loader',
   options: {
     cacheDirectory: true,
+    // disable gzip compression for cache files
+    // faster when there are millions of small files
     cacheCompression: false,
   },
 };
@@ -205,7 +232,7 @@ const config = {
       {
         test: /\.jsx?$/,
         // include source code for plugins, but exclude node_modules within them
-        exclude: [/superset-ui.*\/node_modules\/.*/],
+        exclude: [/superset-ui.*\/node_modules\//],
         include: [new RegExp(`${APP_DIR}/src`), /superset-ui.*\/src/],
         use: [babelLoader],
       },
@@ -277,28 +304,17 @@ const config = {
   devtool: false,
 };
 
-let proxyConfig = {};
-const requireModule = module.require;
-
-function loadProxyConfig() {
-  try {
-    delete require.cache[require.resolve('./webpack.proxy-config')];
-    proxyConfig = requireModule('./webpack.proxy-config');
-  } catch (e) {
-    if (e.code !== 'ENOENT') {
-      console.error('\n>> Error loading proxy config:');
-      console.trace(e);
-    }
-  }
-}
+let proxyConfig = getProxyConfig();
 
 if (isDevMode) {
   config.devtool = 'eval-cheap-module-source-map';
   config.devServer = {
-    before() {
-      loadProxyConfig();
-      // hot reloading proxy config
-      fs.watch('./webpack.proxy-config.js', loadProxyConfig);
+    before(app, server, compiler) {
+      // load proxy config when manifest updates
+      const hook = compiler.hooks.webpackManifestPluginAfterEmit;
+      hook.tap('ManifestPlugin', manifest => {
+        proxyConfig = getProxyConfig(manifest);
+      });
     },
     historyApiFallback: true,
     hot: true,