Remove upload to docker registry code snippets (#1017)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: soapy1

conda-store-server/conda_store_server/_internal/orm.py

@@ -395,18 +395,11 @@ def conda_env_export_key(self):
     def conda_pack_key(self):
         return f"archive/{self.build_key}.tar.gz"
 
-    @property
-    def docker_manifest_key(self):
-        return f"docker/manifest/{self.build_key}"
-
     @property
     def constructor_installer_key(self):
         ext = "exe" if sys.platform == "win32" else "sh"
         return f"installer/{self.build_key}.{ext}"
 
-    def docker_blob_key(self, blob_hash):
-        return f"docker/blobs/{blob_hash}"
-
     @hybrid_property
     def has_lockfile(self):
         return any(

conda-store-server/conda_store_server/_internal/schema.py

@@ -154,10 +154,9 @@ class BuildArtifactType(str, enum.Enum):
     LOGS = "LOGS"
     YAML = "YAML"
     CONDA_PACK = "CONDA_PACK"
-    DOCKER_BLOB = "DOCKER_BLOB"
     DOCKER_MANIFEST = "DOCKER_MANIFEST"
-    CONTAINER_REGISTRY = "CONTAINER_REGISTRY"
     CONSTRUCTOR_INSTALLER = "CONSTRUCTOR_INSTALLER"
+    _ = "CONTAINER_REGISTRY"
 
 
 class BuildStatus(enum.Enum):
@@ -267,9 +266,6 @@ class Settings(BaseModel):
             BuildArtifactType.LOGS,
             BuildArtifactType.LOCKFILE,
             BuildArtifactType.YAML,
-            # no possible way to delete these artifacts
-            # in most container registries via api
-            BuildArtifactType.CONTAINER_REGISTRY,
         ],
         description="artifacts to keep on build deletion",
         metadata={"global": True},
@@ -446,80 +442,6 @@ def __str__(self):
         return str(self.dict())
 
 
-###############################
-#  Docker Registry Schema
-###############################
-
-
-def _docker_datetime_factory():
-    """Utcnow datetime + timezone as string"""
-    return datetime.datetime.utcnow().astimezone().isoformat()
-
-
-class DockerManifestLayer(BaseModel):
-    mediaType: str = "application/vnd.docker.image.rootfs.diff.tar.gzip"
-    size: int
-    digest: str
-
-
-class DockerManifestConfig(BaseModel):
-    mediaType: str = "application/vnd.docker.container.image.v1+json"
-    size: int
-    digest: str
-
-
-class DockerManifest(BaseModel):
-    schemaVersion: int = 2
-    mediaType: str = "application/vnd.docker.distribution.manifest.v2+json"
-    config: DockerManifestConfig
-    layers: List[DockerManifestLayer] = []
-
-
-class DockerConfigConfig(BaseModel):
-    Hostname: str = ""
-    Domainname: str = ""
-    User: str = ""
-    AttachStdin: bool = False
-    AttachStdout: bool = False
-    AttachStderr: bool = False
-    Tty: bool = False
-    OpenStdin: bool = False
-    StdinOnce: bool = False
-    Env: List[str] = [
-        "PATH=/opt/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-    ]
-    Cmd: List[str] = ["/bin/sh"]
-    ArgsEscaped: bool = True
-    Image: Optional[str] = None
-    Volumes: Optional[List[str]] = None
-    WorkingDir: str = ""
-    Entrypoint: Optional[str] = None
-    OnBuild: Optional[str] = None
-    Labels: Optional[Dict[str, str]] = {"conda_store": "0.0.1"}
-
-
-class DockerConfigRootFS(BaseModel):
-    type: str = "layers"
-    diff_ids: List[str] = []
-
-
-class DockerConfigHistory(BaseModel):
-    created: str = Field(default_factory=_docker_datetime_factory)
-    created_by: str = ""
-
-
-class DockerConfig(BaseModel):
-    architecture: str = "amd64"
-    os: str = "linux"
-    config: DockerConfigConfig
-    container: str
-    container_config: DockerConfigConfig
-    created: str = Field(default_factory=_docker_datetime_factory)
-    docker_version: str = "18.09.7"
-    history: List[DockerConfigHistory] = []
-    rootfs: DockerConfigRootFS
-
-
 # https://docs.docker.com/registry/spec/api/#errors-2
 class DockerRegistryError(enum.Enum):
     NAME_UNKNOWN = {

conda-store-server/conda_store_server/_internal/worker/tasks.py

@@ -274,11 +274,6 @@ def delete_build_artifact(db: Session, conda_store, build_artifact):
         ):
             shutil.rmtree(conda_prefix)
             db.delete(build_artifact)
-    elif build_artifact.artifact_type == schema.BuildArtifactType.CONTAINER_REGISTRY:
-        pass
-        # # container registry tag deletion is not generally implemented
-        # # the underlying library `python_docker` is already capable
-        # conda_store.container_registry.delete_image(build_artifact.key)
     else:
         conda_store.log.info(f"deleting {build_artifact.key}")
         conda_store.storage.delete(db, build_artifact.build.id, build_artifact.key)

conda-store-server/conda_store_server/app.py

@@ -25,7 +25,7 @@
 )
 from traitlets.config import LoggingConfigurable
 
-from conda_store_server import CONDA_STORE_DIR, BuildKey, api, registry, storage
+from conda_store_server import CONDA_STORE_DIR, BuildKey, api, storage
 from conda_store_server._internal import conda_utils, environment, orm, schema, utils
 from conda_store_server.plugins import hookspec, plugin_manager
 from conda_store_server.plugins.types import lock
@@ -78,12 +78,7 @@ class CondaStore(LoggingConfigurable):
         config=True,
     )
 
-    container_registry_class = Type(
-        default_value=registry.ContainerRegistry,
-        klass=registry.ContainerRegistry,
-        allow_none=False,
-        config=True,
-    )
+    container_registry_class = Type(allow_none=True, help="(deprecated)")
 
     store_directory = Unicode(
         str(CONDA_STORE_DIR / "state"),
@@ -282,9 +277,6 @@ def _check_redis(self, proposal):
             schema.BuildArtifactType.LOGS,
             schema.BuildArtifactType.LOCKFILE,
             schema.BuildArtifactType.YAML,
-            # no possible way to delete these artifacts
-            # in most container registries via api
-            schema.BuildArtifactType.CONTAINER_REGISTRY,
         ],
         help="artifacts to keep on build deletion",
         config=True,
@@ -410,15 +402,6 @@ def storage(self):
 
         return self._storage
 
-    @property
-    def container_registry(self):
-        if hasattr(self, "_container_registry"):
-            return self._container_registry
-        self._container_registry = self.container_registry_class(
-            parent=self, log=self.log
-        )
-        return self._container_registry
-
     @property
     def celery_config(self):
         return {

conda-store-server/conda_store_server/registry.py

@@ -2,192 +2,23 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-import gzip
-import hashlib
-import urllib.parse
-
-from python_docker.registry import Image, Registry
-from sqlalchemy.orm import Session
-from traitlets import Callable, Dict, default
+from traitlets import Callable, Dict
 from traitlets.config import LoggingConfigurable
 
-from conda_store_server._internal import orm, schema, utils
-
 
 class ContainerRegistry(LoggingConfigurable):
     container_registries = Dict(
         {},
-        help="Registries url to upload built container images with callable function to configure registry instance with credentials",
+        help="(deprecated) Registries url to upload built container images with callable function to configure registry instance with credentials",
         config=True,
     )
 
     container_registry_image_name = Callable(
-        help="Image name to assign to docker image pushed for particular registry",
+        help="(deprecated) Image name to assign to docker image pushed for particular registry",
         config=True,
     )
 
-    @default("container_registry_image_name")
-    def _default_container_registry_image_name(self):
-        def _container_registry_image_name(registry: Registry, build: orm.Build):
-            return f"{registry.username}/{build.environment.namespace.name}-{build.environment.name}"
-
-        return _container_registry_image_name
-
     container_registry_image_tag = Callable(
-        help="Image name and tag to assign to docker image pushed for particular registry",
+        help="(deprecated) Image name and tag to assign to docker image pushed for particular registry",
         config=True,
     )
-
-    @default("container_registry_image_tag")
-    def _default_container_registry_image_tag(self):
-        def _container_registry_image_tag(registry: Registry, build: orm.Build):
-            return build.key
-
-        return _container_registry_image_tag
-
-    def store_image(self, db: Session, conda_store, build: orm.Build, image: Image):
-        self.log.info("storing container image locally")
-        with utils.timer(self.log, "storing container image locally"):
-            # https://docs.docker.com/registry/spec/manifest-v2-2/#example-image-manifest
-            docker_manifest = schema.DockerManifest.construct()
-            docker_config = schema.DockerConfig.construct(
-                config=schema.DockerConfigConfig(),
-                container_config=schema.DockerConfigConfig(),
-                rootfs=schema.DockerConfigRootFS(),
-            )
-
-            for layer in image.layers:
-                # https://github.com/google/nixery/pull/64#issuecomment-541019077
-                # docker manifest expects compressed hash while configuration file
-                # expects uncompressed hash -- good luck finding this detail in docs :)
-                content_uncompressed_hash = hashlib.sha256(layer.content).hexdigest()
-                content_compressed = gzip.compress(layer.content)
-                content_compressed_hash = hashlib.sha256(content_compressed).hexdigest()
-                conda_store.storage.set(
-                    db,
-                    build.id,
-                    build.docker_blob_key(content_compressed_hash),
-                    content_compressed,
-                    content_type="application/gzip",
-                    artifact_type=schema.BuildArtifactType.DOCKER_BLOB,
-                )
-
-                docker_layer = schema.DockerManifestLayer(
-                    size=len(content_compressed),
-                    digest=f"sha256:{content_compressed_hash}",
-                )
-                docker_manifest.layers.append(docker_layer)
-
-                docker_config_history = schema.DockerConfigHistory()
-                docker_config.history.append(docker_config_history)
-
-                docker_config.rootfs.diff_ids.append(
-                    f"sha256:{content_uncompressed_hash}"
-                )
-
-            docker_config_content = docker_config.json().encode("utf-8")
-            docker_config_hash = hashlib.sha256(docker_config_content).hexdigest()
-            docker_manifest.config = schema.DockerManifestConfig(
-                size=len(docker_config_content), digest=f"sha256:{docker_config_hash}"
-            )
-            docker_manifest_content = docker_manifest.json().encode("utf-8")
-            docker_manifest_hash = hashlib.sha256(docker_manifest_content).hexdigest()
-
-            conda_store.storage.set(
-                db,
-                build.id,
-                build.docker_blob_key(docker_config_hash),
-                docker_config_content,
-                content_type="application/vnd.docker.container.image.v1+json",
-                artifact_type=schema.BuildArtifactType.DOCKER_BLOB,
-            )
-
-            # docker likes to have a sha256 key version of the manifest this
-            # is sort of hack to avoid having to figure out which sha256
-            # refers to which manifest.
-            conda_store.storage.set(
-                db,
-                build.id,
-                f"docker/manifest/sha256:{docker_manifest_hash}",
-                docker_manifest_content,
-                content_type="application/vnd.docker.distribution.manifest.v2+json",
-                artifact_type=schema.BuildArtifactType.DOCKER_BLOB,
-            )
-
-            conda_store.storage.set(
-                db,
-                build.id,
-                build.docker_manifest_key,
-                docker_manifest_content,
-                content_type="application/vnd.docker.distribution.manifest.v2+json",
-                artifact_type=schema.BuildArtifactType.DOCKER_MANIFEST,
-            )
-
-            conda_store.log.info(
-                f"built docker image: {image.name}:{image.tag} layers={len(image.layers)}"
-            )
-
-    @staticmethod
-    def parse_image_uri(image_name: str):
-        """Must be in fully specified format [<scheme>://]<registry_url>/<image_name>:<tag_name>"""
-        if not image_name.startswith("http"):
-            image_name = f"https://{image_name}"
-
-        parsed_url = urllib.parse.urlparse(image_name)
-        registry_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
-        image_name, tag_name = parsed_url.path.split(":", 1)
-        image_name = image_name[1:]  # remove beginning "/"
-        return registry_url, image_name, tag_name
-
-    def pull_image(self, image_name: str) -> Image:
-        """Must be in fully specified format [<scheme>://]<registry_url>/<image_name>:<tag_name>
-
-        Docker is the only weird registry where you must use:
-          - `https://registry-1.docker.io`
-        """
-        registry_url, name, tag = self.parse_image_uri(image_name)
-
-        for url in self.container_registries:
-            if registry_url in url:
-                registry = self.container_registries[registry_url](url)
-                break
-        else:
-            self.log.warning(
-                f"registry {registry_url} not configured using registry without authentication"
-            )
-            registry = Registry(hostname=registry_url)
-
-        return registry.pull_image(name, tag)
-
-    def push_image(self, db, build, image: Image):
-        for registry_url, configure_registry in self.container_registries.items():
-            self.log.info(f"beginning upload of image to registry {registry_url}")
-            with utils.timer(self.log, f"uploading image to registry {registry_url}"):
-                registry = configure_registry(registry_url)
-                image.name = self.container_registry_image_name(registry, build)
-                image.tag = self.container_registry_image_tag(registry, build)
-                registry.push_image(image)
-
-                registry_build_artifact = orm.BuildArtifact(
-                    build_id=build.id,
-                    artifact_type=schema.BuildArtifactType.CONTAINER_REGISTRY,
-                    key=f"{registry_url}/{image.name}:{image.tag}",
-                )
-                db.add(registry_build_artifact)
-                db.commit()
-
-    def delete_image(self, image_name: str):
-        registry_url, name, tag = self.parse_image_uri(image_name)
-
-        for url in self.container_registries:
-            if registry_url in url:
-                registry = self.container_registries[registry_url](url)
-                break
-        else:
-            self.log.warning(
-                f"registry {registry_url} not configured using registry without authentication"
-            )
-            registry = Registry(hostname=registry_url)
-
-        self.log.info(f"deleting container image {image_name}")
-        registry.delete_image(name, tag)