Build: use new Docker images from design document

Minimal implementation for POC of
readthedocs#8447

It uses a Feature flag for now as a way to select the new
`readthedocs/build:ubuntu20` image and install Python versions via `asdf`
(readthedocs/readthedocs-docker-images#166)

MinIO requires a new bucket called `languages` with a pre-compiled Python 3.9.6
version to work (*) (this version is hardcoded for now). However, if a different
version is selected it will be downloaded from official mirrors, installed and
used.

Build times on `latest` version for `test-build`:

* using the new image + cached Python version: 112s
* using the new image + non cached Python version: 288s
* using old image (current production): 87s

> Note that all the parsing of the Config File to support `build.os` and
> `build.languages` is not included in this PR on purpose. That work can be
> split as a separate work and done in parallel with the rest of work required
> here.

(*) to pre-compile a Python version:

```bash
docker run -it readthedocs/build:ubuntu20 /bin/bash

asdf install python 3.9.6
asdf global python 3.9.6
python -m pip install -U pip setuptools virtualenv
cd /home/docs/.asdf/installs/python
tar -cfvz ubuntu20-python-3.9.6.tar.gz 3.9.6

docker cp <container id>:/home/docs/.asdf/installs/python/ubuntu20-python-3.9.6.tar.gz .
```

and upload the .tar.gz file to MinIO `languages` bucket using the web interface

readthedocs/doc_builder/environments.py

@@ -784,6 +784,9 @@ def __init__(self, *args, **kwargs):
         # the image set by user or,
         if self.config and self.config.build.image:
             self.container_image = self.config.build.image
+        # the new Docker image structure or,
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            self.container_image = 'readthedocs/build:ubuntu20'
         # the image overridden by the project (manually set by an admin).
         if self.project.container_image:
             self.container_image = self.project.container_image

readthedocs/doc_builder/python_environments.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import shutil
+import tarfile
 
 import yaml
 from django.conf import settings
@@ -22,6 +23,7 @@
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.projects.constants import LOG_TEMPLATE
 from readthedocs.projects.models import Feature
+from readthedocs.storage import build_languages_storage
 
 log = logging.getLogger(__name__)
 
@@ -317,15 +319,117 @@ def setup_base(self):
         cli_args.append(
             self.venv_path(),
         )
+
+        # TODO: make ``self.config.python_interpreter`` return the correct value
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            python_interpreter = 'python'
+        else:
+            python_interpreter = self.config.python_interpreter
+
         self.build_env.run(
-            self.config.python_interpreter,
+            python_interpreter,
             *cli_args,
             # Don't use virtualenv bin that doesn't exist yet
             bin_path=None,
             # Don't use the project's root, some config files can interfere
             cwd=None,
         )
 
+    def install_languages(self):
+        if settings.RTD_DOCKER_COMPOSE:
+            # Create a symlink for ``root`` user to use the same ``.asdf``
+            # installation than ``docs`` user. Required for local building
+            # since everything is run as ``root`` when using Local Development
+            # instance
+            cmd = [
+                'ln',
+                '-s',
+                '/home/docs/.asdf',
+                '/root/.asdf',
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+        # TODO: do not use a Feature flag here, but check for ``build.os`` and
+        # ``build.languages`` instead
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            # TODO: iterate over ``build.languages`` and install all languages
+            # specified for this project
+            os = 'ubuntu20'
+            language = 'python'
+            version = '3.9.6'
+
+            # TODO: generate the correct path for the Python version
+            # language_path = f'{os}/{language}/2021-08-30/{version}.tar.gz'
+            language_path = f'{os}-{language}-{version}.tar.gz'
+            language_version_cached = build_languages_storage.exists(language_path)
+            if language_version_cached:
+                remote_fd = build_languages_storage.open(language_path, mode='rb')
+                with tarfile.open(fileobj=remote_fd) as tar:
+                    # Extract it on the shared path between host and Docker container
+                    extract_path = os.path.join(self.project.doc_path, 'languages')
+                    tar.extractall(extra_path)
+
+                    # Move the extracted content to the ``asdf`` installation
+                    cmd = [
+                        'mv',
+                        f'{extract_path}/{version}',
+                        f'/home/docs/.asdf/installs/{language}/{version}',
+                    ]
+                    self.build_env.run(
+                        *cmd,
+                    )
+            else:
+                # If the language version selected is not available from the
+                # cache we compile it at build time
+                cmd = [
+                    'asdf',
+                    'install',
+                    language,
+                    version,
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
+            # Make the language version chosen by the user the default one
+            cmd = [
+                'asdf',
+                'global',
+                language,
+                version,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            # Recreate shims for this language to make the new version
+            # installed available
+            cmd = [
+                'asdf',
+                'reshim',
+                language,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            if language == 'python' and not language_version_cached:
+                # Install our own requirements if the version is compiled
+                cmd = [
+                    'python',
+                    '-m'
+                    'pip',
+                    'install',
+                    '-U',
+                    'virtualenv',
+                    'setuptools',
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
     def install_core_requirements(self):
         """Install basic Read the Docs requirements into the virtualenv."""
         pip_install_cmd = [

readthedocs/projects/models.py

@@ -1669,6 +1669,7 @@ def add_features(sender, **kwargs):
     USE_SPHINX_BUILDERS = 'use_sphinx_builders'
     DEDUPLICATE_BUILDS = 'deduplicate_builds'
     DONT_CREATE_INDEX = 'dont_create_index'
+    USE_NEW_DOCKER_IMAGES_STRUCTURE = 'use_new_docker_images_structure'
 
     FEATURES = (
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
@@ -1690,6 +1691,10 @@ def add_features(sender, **kwargs):
             USE_TESTING_BUILD_IMAGE,
             _('Use Docker image labelled as `testing` to build the docs'),
         ),
+        (
+            USE_NEW_DOCKER_IMAGES_STRUCTURE,
+            _('Use new Docker images that install languages at build time'),
+        ),
         (
             API_LARGE_DATA,
             _('Try alternative method of posting large data'),

readthedocs/projects/tasks.py

@@ -1144,6 +1144,7 @@ def update_app_instances(
 
     def setup_build(self):
         self.install_system_dependencies()
+        self.python_env.install_languages()
         self.setup_python_environment()
 
     def setup_python_environment(self):

readthedocs/settings/base.py

@@ -310,6 +310,7 @@ def USE_PROMOS(self):  # noqa
     # https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
+    RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
 
     @property

readthedocs/settings/docker_compose.py

@@ -134,6 +134,8 @@ def show_debug_toolbar(request):
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.s3_storage.S3BuildMediaStorage'
     # Storage backend for build cached environments
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.s3_storage.S3BuildEnvironmentStorage'
+    # Storage backend for build languages
+    RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.storage.s3_storage.S3BuildLanguagesStorage'
     # Storage for static files (those collected with `collectstatic`)
     STATICFILES_STORAGE = 'readthedocs.storage.s3_storage.S3StaticStorage'
 
@@ -142,6 +144,7 @@ def show_debug_toolbar(request):
     S3_MEDIA_STORAGE_BUCKET = 'media'
     S3_BUILD_COMMANDS_STORAGE_BUCKET = 'builds'
     S3_BUILD_ENVIRONMENT_STORAGE_BUCKET = 'envs'
+    S3_BUILD_LANGUAGES_STORAGE_BUCKET = 'languages'
     S3_STATIC_STORAGE_BUCKET = 'static'
     S3_STATIC_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'
     S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'

readthedocs/storage/__init__.py

@@ -26,6 +26,12 @@ def _setup(self):
         self._wrapped = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)()
 
 
+class ConfiguredBuildLanguagesStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_storage_class(settings.RTD_BUILD_LANGUAGES_STORAGE)()
+
+
 build_media_storage = ConfiguredBuildMediaStorage()
 build_environment_storage = ConfiguredBuildEnvironmentStorage()
 build_commands_storage = ConfiguredBuildCommandsStorage()
+build_languages_storage = ConfiguredBuildLanguagesStorage()

readthedocs/storage/s3_storage.py

@@ -89,3 +89,17 @@ def __init__(self, *args, **kwargs):
                 'AWS S3 not configured correctly. '
                 'Ensure S3_BUILD_ENVIRONMENT_STORAGE_BUCKET is defined.',
             )
+
+
+class S3BuildLanguagesStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage):
+
+    bucket_name = getattr(settings, 'S3_BUILD_LANGUAGES_STORAGE_BUCKET', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_BUILD_LANGUAGES_STORAGE_BUCKET is defined.',
+            )