diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index 69d88d30640..a01c3f2add1 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -784,6 +784,9 @@ def __init__(self, *args, **kwargs): # the image set by user or, if self.config and self.config.build.image: self.container_image = self.config.build.image + # the new Docker image structure or, + if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE): + self.container_image = 'readthedocs/build:ubuntu20' # the image overridden by the project (manually set by an admin). if self.project.container_image: self.container_image = self.project.container_image diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 97fe0355dc8..c0f323a7e3c 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -8,6 +8,7 @@ import logging import os import shutil +import tarfile import yaml from django.conf import settings @@ -22,6 +23,7 @@ from readthedocs.doc_builder.loader import get_builder_class from readthedocs.projects.constants import LOG_TEMPLATE from readthedocs.projects.models import Feature +from readthedocs.storage import build_languages_storage log = logging.getLogger(__name__) @@ -317,8 +319,15 @@ def setup_base(self): cli_args.append( self.venv_path(), ) + + # TODO: make ``self.config.python_interpreter`` return the correct value + if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE): + python_interpreter = 'python' + else: + python_interpreter = self.config.python_interpreter + self.build_env.run( - self.config.python_interpreter, + python_interpreter, *cli_args, # Don't use virtualenv bin that doesn't exist yet bin_path=None, @@ -326,6 +335,101 @@ def setup_base(self): cwd=None, ) + def install_languages(self): + if settings.RTD_DOCKER_COMPOSE: + # Create a symlink for ``root`` user to use the same ``.asdf`` + # installation than ``docs`` user. Required for local building + # since everything is run as ``root`` when using Local Development + # instance + cmd = [ + 'ln', + '-s', + '/home/docs/.asdf', + '/root/.asdf', + ] + self.build_env.run( + *cmd, + ) + + # TODO: do not use a Feature flag here, but check for ``build.os`` and + # ``build.languages`` instead + if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE): + # TODO: iterate over ``build.languages`` and install all languages + # specified for this project + os = 'ubuntu20' + language = 'python' + version = '3.9.6' + + # TODO: generate the correct path for the Python version + # language_path = f'{os}/{language}/2021-08-30/{version}.tar.gz' + language_path = f'{os}-{language}-{version}.tar.gz' + language_version_cached = build_languages_storage.exists(language_path) + if language_version_cached: + remote_fd = build_languages_storage.open(language_path, mode='rb') + with tarfile.open(fileobj=remote_fd) as tar: + # Extract it on the shared path between host and Docker container + extract_path = os.path.join(self.project.doc_path, 'languages') + tar.extractall(extra_path) + + # Move the extracted content to the ``asdf`` installation + cmd = [ + 'mv', + f'{extract_path}/{version}', + f'/home/docs/.asdf/installs/{language}/{version}', + ] + self.build_env.run( + *cmd, + ) + else: + # If the language version selected is not available from the + # cache we compile it at build time + cmd = [ + 'asdf', + 'install', + language, + version, + ] + self.build_env.run( + *cmd, + ) + + # Make the language version chosen by the user the default one + cmd = [ + 'asdf', + 'global', + language, + version, + ] + self.build_env.run( + *cmd, + ) + + # Recreate shims for this language to make the new version + # installed available + cmd = [ + 'asdf', + 'reshim', + language, + ] + self.build_env.run( + *cmd, + ) + + if language == 'python' and not language_version_cached: + # Install our own requirements if the version is compiled + cmd = [ + 'python', + '-m' + 'pip', + 'install', + '-U', + 'virtualenv', + 'setuptools', + ] + self.build_env.run( + *cmd, + ) + def install_core_requirements(self): """Install basic Read the Docs requirements into the virtualenv.""" pip_install_cmd = [ diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index ba02c035551..255ac1114ea 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1669,6 +1669,7 @@ def add_features(sender, **kwargs): USE_SPHINX_BUILDERS = 'use_sphinx_builders' DEDUPLICATE_BUILDS = 'deduplicate_builds' DONT_CREATE_INDEX = 'dont_create_index' + USE_NEW_DOCKER_IMAGES_STRUCTURE = 'use_new_docker_images_structure' FEATURES = ( (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')), @@ -1690,6 +1691,10 @@ def add_features(sender, **kwargs): USE_TESTING_BUILD_IMAGE, _('Use Docker image labelled as `testing` to build the docs'), ), + ( + USE_NEW_DOCKER_IMAGES_STRUCTURE, + _('Use new Docker images that install languages at build time'), + ), ( API_LARGE_DATA, _('Try alternative method of posting large data'), diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index e3a2c98ad25..11ccd869de9 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1144,6 +1144,7 @@ def update_app_instances( def setup_build(self): self.install_system_dependencies() + self.python_env.install_languages() self.setup_python_environment() def setup_python_environment(self): diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 85cf6f2cbb4..e167915aec9 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -310,6 +310,7 @@ def USE_PROMOS(self): # noqa # https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage' RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage' + RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage' RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage' @property diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index 4f598a91b7f..0e6d3c232c3 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -134,6 +134,8 @@ def show_debug_toolbar(request): RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.s3_storage.S3BuildMediaStorage' # Storage backend for build cached environments RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.s3_storage.S3BuildEnvironmentStorage' + # Storage backend for build languages + RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.storage.s3_storage.S3BuildLanguagesStorage' # Storage for static files (those collected with `collectstatic`) STATICFILES_STORAGE = 'readthedocs.storage.s3_storage.S3StaticStorage' @@ -142,6 +144,7 @@ def show_debug_toolbar(request): S3_MEDIA_STORAGE_BUCKET = 'media' S3_BUILD_COMMANDS_STORAGE_BUCKET = 'builds' S3_BUILD_ENVIRONMENT_STORAGE_BUCKET = 'envs' + S3_BUILD_LANGUAGES_STORAGE_BUCKET = 'languages' S3_STATIC_STORAGE_BUCKET = 'static' S3_STATIC_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io' S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io' diff --git a/readthedocs/storage/__init__.py b/readthedocs/storage/__init__.py index 644fb55e7c2..801764e02b6 100644 --- a/readthedocs/storage/__init__.py +++ b/readthedocs/storage/__init__.py @@ -26,6 +26,12 @@ def _setup(self): self._wrapped = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)() +class ConfiguredBuildLanguagesStorage(LazyObject): + def _setup(self): + self._wrapped = get_storage_class(settings.RTD_BUILD_LANGUAGES_STORAGE)() + + build_media_storage = ConfiguredBuildMediaStorage() build_environment_storage = ConfiguredBuildEnvironmentStorage() build_commands_storage = ConfiguredBuildCommandsStorage() +build_languages_storage = ConfiguredBuildLanguagesStorage() diff --git a/readthedocs/storage/s3_storage.py b/readthedocs/storage/s3_storage.py index 92ff276fd89..a1f1f537357 100644 --- a/readthedocs/storage/s3_storage.py +++ b/readthedocs/storage/s3_storage.py @@ -89,3 +89,17 @@ def __init__(self, *args, **kwargs): 'AWS S3 not configured correctly. ' 'Ensure S3_BUILD_ENVIRONMENT_STORAGE_BUCKET is defined.', ) + + +class S3BuildLanguagesStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage): + + bucket_name = getattr(settings, 'S3_BUILD_LANGUAGES_STORAGE_BUCKET', None) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if not self.bucket_name: + raise ImproperlyConfigured( + 'AWS S3 not configured correctly. ' + 'Ensure S3_BUILD_LANGUAGES_STORAGE_BUCKET is defined.', + )