feat: bake translations as part of the build processes (#28483)

.github/CODEOWNERS

@@ -30,7 +30,3 @@
 
 **/*.geojson @villebro @rusackas
 /superset-frontend/plugins/legacy-plugin-chart-country-map/ @villebro @rusackas
-
-# Translations are a finnicky contribution that we care about
-
-/superset/translations/ @villebro @rusackas

.github/workflows/superset-python-misc.yml

@@ -50,4 +50,4 @@ jobs:
         uses: ./.github/actions/setup-backend/
       - name: Test babel extraction
         if: steps.check.outputs.python
-        run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
+        run: scripts/translations/babel_update.sh

.github/workflows/superset-translations.yml

@@ -43,7 +43,7 @@ jobs:
         if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
-          npm run check-translation
+          npm run build-translation
 
   babel-extract:
     runs-on: ubuntu-20.04
@@ -64,4 +64,4 @@ jobs:
         uses: ./.github/actions/setup-backend/
       - name: Test babel extraction
         if: steps.check.outputs.python
-        run: ./scripts/babel_update.sh
+        run: ./scripts/translations/babel_update.sh

.gitignore

@@ -106,8 +106,11 @@ testCSV.csv
 apache-superset-*.tar.gz*
 release.json
 
-# Translation binaries
-messages.mo
+# Translation-related files
+# these json files are generated by ./scripts/po2json.sh
+superset/translations/**/messages.json
+# these mo binary files are generated by `pybabel compile`
+superset/translations/**/messages.mo
 
 docker/requirements-local.txt
 

Dockerfile

@@ -26,27 +26,37 @@ FROM --platform=${BUILDPLATFORM} node:18-bullseye-slim AS superset-node
 
 ARG NPM_BUILD_CMD="build"
 
+# Somehow we need python3 + build-essential on this side of the house to install node-gyp
 RUN apt-get update -qq \
-    && apt-get install -yqq --no-install-recommends \
+    && apt-get install \
+        -yqq --no-install-recommends \
         build-essential \
         python3
 
 ENV BUILD_CMD=${NPM_BUILD_CMD} \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 # NPM ci first, as to NOT invalidate previous steps except for when package.json changes
-WORKDIR /app/superset-frontend
 
 RUN --mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \
     /frontend-mem-nag.sh
 
+WORKDIR /app/superset-frontend
 RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json \
     --mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json \
     npm ci
 
-COPY ./superset-frontend ./
-# This seems to be the most expensive step
+# Runs the webpack build process
+COPY superset-frontend /app/superset-frontend
 RUN npm run ${BUILD_CMD}
 
+# This copies the .po files needed for translation
+RUN mkdir -p /app/superset/translations
+COPY superset/translations /app/superset/translations
+# Compiles .json files from the .po files, then deletes the .po files
+RUN npm run build-translation
+RUN rm /app/superset/translations/*/LC_MESSAGES/*.po
+RUN rm /app/superset/translations/messages.pot
+
 ######################################################################
 # Final lean image...
 ######################################################################
@@ -87,13 +97,23 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     && apt-get autoremove -yqq --purge build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+# Copy the compiled frontend assets
 COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
+
 ## Lastly, let's install superset itself
 COPY --chown=superset:superset superset superset
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install -e . \
-    && flask fab babel-compile --target superset/translations \
-    && chown -R superset:superset superset/translations
+    pip install -e .
+
+# Copy the .json translations from the frontend layer
+COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
+
+# Compile translations for the backend - this generates .mo files, then deletes the .po files
+COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
+RUN ./scripts/translations/generate_mo_files.sh \
+    && chown -R superset:superset superset/translations \
+    && rm superset/translations/messages.pot \
+    && rm superset/translations/*/LC_MESSAGES/*.po
 
 COPY --chmod=755 ./docker/run-server.sh /usr/bin/
 USER superset

RELEASING/README.md

@@ -445,8 +445,16 @@ Create the distribution
 ```bash
 cd superset-frontend/
 npm ci && npm run build
+# Compile translations for the frontend
+npm run build-translation
+
 cd ../
-flask fab babel-compile --target superset/translations
+
+
+# Compile translations for the backend
+./scripts/translations/generate_po_files.sh
+
+# build the python distribution
 python setup.py sdist
 ```
 

UPDATING.md

@@ -47,6 +47,10 @@ assists people when migrating to a new version.
   more clearly provides access to all databases, as specified in its name. Before it only allowed
   listing all databases in CRUD-view and dropdown and didn't provide access to data as it
   seemed the name would imply.
+- [28483](https://github.com/apache/superset/pull/28483) Starting with this version we bundle
+  translations inside the python package. This includes the .mo files needed by pybabel on the
+  backend, as well as the .json files used by the frontend. If you were doing anything before
+  as part of your bundling to expose translation packages, it's probably not needed anymore.
 
 ### Potential Downtime
 

docs/docs/contributing/development.mdx

@@ -737,97 +737,6 @@ npm run storybook
 
 When contributing new React components to Superset, please try to add a Story alongside the component's `jsx/tsx` file.
 
-## Translating
-
-We use [Flask-Babel](https://python-babel.github.io/flask-babel/) to translate Superset.
-In Python files, we import the magic `_` function using:
-
-```python
-from flask_babel import lazy_gettext as _
-```
-
-then wrap our translatable strings with it, e.g. `_('Translate me')`.
-During extraction, string literals passed to `_` will be added to the
-generated `.po` file for each language for later translation.
-
-At runtime, the `_` function will return the translation of the given
-string for the current language, or the given string itself
-if no translation is available.
-
-In TypeScript/JavaScript, the technique is similar:
-we import `t` (simple translation), `tn` (translation containing a number).
-
-```javascript
-import { t, tn } from "@superset-ui/translation";
-```
-
-### Enabling language selection
-
-Add the `LANGUAGES` variable to your `superset_config.py`. Having more than one
-option inside will add a language selection dropdown to the UI on the right side
-of the navigation bar.
-
-```python
-LANGUAGES = {
-    'en': {'flag': 'us', 'name': 'English'},
-    'fr': {'flag': 'fr', 'name': 'French'},
-    'zh': {'flag': 'cn', 'name': 'Chinese'},
-}
-```
-
-### Updating language files
-
-```bash
-./scripts/babel_update.sh
-```
-
-This script will
-
-1. update the template file `superset/translations/messages.pot` with current application strings.
-2. update language files with the new extracted strings.
-
-You can then translate the strings gathered in files located under
-`superset/translation`, where there's one per language. You can use [Poedit](https://poedit.net/features)
-to translate the `po` file more conveniently.
-There are some [tutorials in the wiki](https://wiki.lxde.org/en/Translate_*.po_files_with_Poedit).
-
-In the case of JS translation, we need to convert the PO file into a JSON file, and we need the global download of the npm package po2json.
-
-```bash
-npm install -g po2json
-```
-
-To convert all PO files to formatted JSON files you can use the `po2json.sh` script.
-
-```bash
-./scripts/po2json.sh
-```
-
-If you get errors running `po2json`, you might be running the Ubuntu package with the same
-name, rather than the Node.js package (they have a different format for the arguments). If
-there is a conflict, you may need to update your `PATH` environment variable or fully qualify
-the executable path (e.g. `/usr/local/bin/po2json` instead of `po2json`).
-If you get a lot of `[null,***]` in `messages.json`, just delete all the `null,`.
-For example, `"year":["年"]` is correct while `"year":[null,"年"]`is incorrect.
-
-For the translations to take effect we need to compile translation catalogs into binary MO files.
-
-```bash
-pybabel compile -d superset/translations
-```
-
-### Creating a new language dictionary
-
-To create a dictionary for a new language, run the following, where `LANGUAGE_CODE` is replaced with
-the language code for your target language, e.g. `es` (see [Flask AppBuilder i18n documentation](https://flask-appbuilder.readthedocs.io/en/latest/i18n.html) for more details):
-
-```bash
-pip install -r superset/translations/requirements.txt
-pybabel init -i superset/translations/messages.pot -d superset/translations -l LANGUAGE_CODE
-```
-
-Then, [Updating language files](#updating-language-files).
-
 ## Tips
 
 ### Adding a new datasource

docs/docs/contributing/howtos.mdx

@@ -459,7 +459,7 @@ npm run storybook
 
 When contributing new React components to Superset, please try to add a Story alongside the component's `jsx/tsx` file.
 
-## Contribute Translations
+## Contributing Translations
 
 We use [Flask-Babel](https://python-babel.github.io/flask-babel/) to translate Superset.
 In Python files, we use the following
@@ -538,18 +538,17 @@ pybabel init -i superset/translations/messages.pot -d superset/translations -l f
 
 ### Extracting new strings for translation
 
-This step needs to be done every time application strings change. This happens fairly
-frequently, so if you want to ensure that your translation has good coverage, this
-step needs to be run fairly frequently and the updated strings merged to the upstream
-codebase via PRs. To update the template file `superset/translations/messages.pot`
-with current application strings, run the following command:
+Periodically, when working on translations, we need to extract the strings from both the
+backend and the frontend to compile a list of all strings to be translated. It doesn't
+happen automatically and is a required step to gather the strings and get them into the
+`.po` files where they can be translated, so that they can then be compiled.
+
+This script does just that:
 
 ```bash
-pybabel extract -F superset/translations/babel.cfg -o superset/translations/messages.pot -k _ -k __ -k t -k tn -k tct .
+./scripts/translations/babel_update.sh
 ```
 
-Do not forget to update this file with the appropriate license information.
-
 ### Updating language files
 
 Run the following command to update the language files with the new extracted strings.
@@ -575,27 +574,15 @@ case of the Finnish translation, this would be `superset/translations/fi/LC_MESS
 ### Applying translations
 
 To make the translations available on the frontend, we need to convert the PO file into
-a JSON file. To do this, we need to globally install the npm package `po2json`.
+a collection of JSON files. To convert all PO files to formatted JSON files you can use
+the build-translation script
 
 ```bash
-npm install -g po2json
+npm run build-translation
 ```
 
-To convert all PO files to formatted JSON files you can use the `po2json.sh` script.
-
-```bash
-./scripts/po2json.sh
-```
-
-If you get errors running `po2json`, you might be running the Ubuntu package with the same
-name, rather than the Node.js package (they have a different format for the arguments). If
-there is a conflict, you may need to update your `PATH` environment variable or fully qualify
-the executable path (e.g. `/usr/local/bin/po2json` instead of `po2json`).
-If you get a lot of `[null,***]` in `messages.json`, just delete all the `null,`.
-For example, `"year":["年"]` is correct while `"year":[null,"年"]`is incorrect.
-
 Finally, for the translations to take effect we need to compile translation catalogs into
-binary MO files.
+binary MO files for the backend using pybabel.
 
 ```bash
 pybabel compile -d superset/translations

scripts/babel_update.sh → scripts/translations/babel_update.sh

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../.. && pwd )"
 LICENSE_TMP=$(mktemp)
 cat <<'EOF'> "$LICENSE_TMP"
 # Licensed to the Apache Software Foundation (ASF) under one

scripts/translations/generate_mo_files.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script generates .mo binary files from .po translation files
+# these .mo files are used by the backend to load translations
+
+flask fab babel-compile --target superset/translations