feat(releases): Add an endpoint for artifact bundles (#13448)

Creates an assemble-like endpoint for uploading artifact bundles

Author: jan-auer

src/sentry/api/endpoints/chunk.py

@@ -25,6 +25,11 @@
 MAX_CONCURRENCY = settings.DEBUG and 1 or 8
 HASH_ALGORITHM = 'sha1'
 
+CHUNK_UPLOAD_ACCEPT = (
+    'debug_files',  # DIF assemble
+    'release_files',  # Artifacts assemble
+)
+
 
 class GzipChunk(BytesIO):
     def __init__(self, file):
@@ -61,6 +66,7 @@ def get(self, request, organization):
                 'concurrency': MAX_CONCURRENCY,
                 'hashAlgorithm': HASH_ALGORITHM,
                 'compression': ['gzip'],
+                'accept': CHUNK_UPLOAD_ACCEPT,
             }
         )
 

src/sentry/api/endpoints/debug_files.py

@@ -18,8 +18,9 @@
 from sentry.api.paginator import OffsetPaginator
 from sentry.api.serializers import serialize
 from sentry.constants import KNOWN_DIF_FORMATS
-from sentry.models import ChunkFileState, FileBlobOwner, ProjectDebugFile, \
-    create_files_from_dif_zip, get_assemble_status, set_assemble_status
+from sentry.models import FileBlobOwner, ProjectDebugFile, create_files_from_dif_zip
+from sentry.tasks.assemble import get_assemble_status, set_assemble_status, \
+    AssembleTask, ChunkFileState
 from sentry.utils import json
 
 try:
@@ -252,7 +253,10 @@ def post(self, request, project):
                         "name": {"type": "string"},
                         "chunks": {
                             "type": "array",
-                            "items": {"type": "string"}
+                            "items": {
+                                "type": "string",
+                                "pattern": "^[0-9a-f]{40}$",
+                            }
                         }
                     },
                     "additionalProperties": False
@@ -273,15 +277,14 @@ def post(self, request, project):
 
         file_response = {}
 
-        from sentry.tasks.assemble import assemble_dif
         for checksum, file_to_assemble in six.iteritems(files):
             name = file_to_assemble.get('name', None)
             chunks = file_to_assemble.get('chunks', [])
 
             # First, check the cached assemble status. During assembling, a
             # ProjectDebugFile will be created and we need to prevent a race
             # condition.
-            state, detail = get_assemble_status(project, checksum)
+            state, detail = get_assemble_status(AssembleTask.DIF, project.id, checksum)
             if state == ChunkFileState.OK:
                 file_response[checksum] = {
                     'state': state,
@@ -337,7 +340,10 @@ def post(self, request, project):
 
             # We don't have a state yet, this means we can now start
             # an assemble job in the background.
-            set_assemble_status(project, checksum, ChunkFileState.CREATED)
+            set_assemble_status(AssembleTask.DIF, project.id, checksum,
+                                ChunkFileState.CREATED)
+
+            from sentry.tasks.assemble import assemble_dif
             assemble_dif.apply_async(
                 kwargs={
                     'project_id': project.id,

src/sentry/api/endpoints/organization_release_assemble.py

@@ -0,0 +1,105 @@
+from __future__ import absolute_import
+
+import jsonschema
+from rest_framework.response import Response
+
+from sentry.api.bases.organization import OrganizationReleasesBaseEndpoint
+from sentry.api.exceptions import ResourceDoesNotExist
+from sentry.models import Release
+from sentry.tasks.assemble import get_assemble_status, set_assemble_status, \
+    AssembleTask, ChunkFileState
+from sentry.utils import json
+
+
+class OrganizationReleaseAssembleEndpoint(OrganizationReleasesBaseEndpoint):
+    def post(self, request, organization, version):
+        """
+        Handle an artifact bundle and merge it into the release
+        ```````````````````````````````````````````````````````
+
+        :auth: required
+        """
+
+        try:
+            release = Release.objects.get(
+                organization_id=organization.id,
+                version=version,
+            )
+        except Release.DoesNotExist:
+            raise ResourceDoesNotExist
+
+        if not self.has_release_permission(request, organization, release):
+            raise ResourceDoesNotExist
+
+        schema = {
+            "type": "object",
+            "properties": {
+                "checksum": {
+                    "type": "string",
+                    "pattern": "^[0-9a-f]{40}$",
+                },
+                "chunks": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "pattern": "^[0-9a-f]{40}$",
+                    }
+                }
+            },
+            "required": ["checksum", "chunks"],
+            "additionalProperties": False,
+        }
+
+        try:
+            data = json.loads(request.body)
+            jsonschema.validate(data, schema)
+        except jsonschema.ValidationError as e:
+            return Response({'error': str(e).splitlines()[0]},
+                            status=400)
+        except BaseException as e:
+            return Response({'error': 'Invalid json body'},
+                            status=400)
+
+        checksum = data.get('checksum', None)
+        chunks = data.get('chunks', [])
+
+        state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum)
+        if state == ChunkFileState.OK:
+            return Response({
+                'state': state,
+                'detail': None,
+                'missingChunks': [],
+            }, status=200)
+        elif state is not None:
+            return Response({
+                'state': state,
+                'detail': detail,
+                'missingChunks': [],
+            })
+
+        # There is neither a known file nor a cached state, so we will
+        # have to create a new file.  Assure that there are checksums.
+        # If not, we assume this is a poll and report NOT_FOUND
+        if not chunks:
+            return Response({
+                'state': ChunkFileState.NOT_FOUND,
+                'missingChunks': [],
+            }, status=200)
+
+        set_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum,
+                            ChunkFileState.CREATED)
+
+        from sentry.tasks.assemble import assemble_artifacts
+        assemble_artifacts.apply_async(
+            kwargs={
+                'org_id': organization.id,
+                'version': version,
+                'checksum': checksum,
+                'chunks': chunks,
+            }
+        )
+
+        return Response({
+            'state': ChunkFileState.CREATED,
+            'missingChunks': [],
+        }, status=200)

src/sentry/api/urls.py

@@ -105,6 +105,7 @@
 from .endpoints.organization_recent_searches import OrganizationRecentSearchesEndpoint
 from .endpoints.organization_releases import OrganizationReleasesEndpoint
 from .endpoints.organization_release_details import OrganizationReleaseDetailsEndpoint
+from .endpoints.organization_release_assemble import OrganizationReleaseAssembleEndpoint
 from .endpoints.organization_release_files import OrganizationReleaseFilesEndpoint
 from .endpoints.organization_release_file_details import OrganizationReleaseFileDetailsEndpoint
 from .endpoints.organization_release_commits import OrganizationReleaseCommitsEndpoint
@@ -748,6 +749,11 @@
         OrganizationReleaseDetailsEndpoint.as_view(),
         name='sentry-api-0-organization-release-details'
     ),
+    url(
+        r'^organizations/(?P<organization_slug>[^\/]+)/releases/(?P<version>[^/]+)/assemble/$',
+        OrganizationReleaseAssembleEndpoint.as_view(),
+        name='sentry-api-0-organization-release-assemble'
+    ),
     url(
         r'^organizations/(?P<organization_slug>[^\/]+)/releases/(?P<version>[^/]+)/files/$',
         OrganizationReleaseFilesEndpoint.as_view(),

src/sentry/models/debugfile.py

@@ -24,7 +24,6 @@
 from symbolic import Archive, SymbolicError, ObjectErrorUnsupportedObject
 
 from sentry import options
-from sentry.cache import default_cache
 from sentry.constants import KNOWN_DIF_FORMATS
 from sentry.db.models import FlexibleForeignKey, Model, sane_repr, BaseManager, JSONField
 from sentry.models.file import File
@@ -47,37 +46,6 @@
 _proguard_file_re = re.compile(r'/proguard/(?:mapping-)?(.*?)\.txt$')
 
 
-def _get_idempotency_id(project, checksum):
-    """For some operations an idempotency ID is needed."""
-    return hashlib.sha1(b'%s|%s|project.dsym' % (
-        str(project.id).encode('ascii'),
-        checksum.encode('ascii'),
-    )).hexdigest()
-
-
-def get_assemble_status(project, checksum):
-    """For a given file it checks what the current status of the assembling is.
-    Returns a tuple in the form ``(status, details)`` where details is either
-    `None` or a string identifying an error condition or notice.
-    """
-    cache_key = 'assemble-status:%s' % _get_idempotency_id(
-        project, checksum)
-    rv = default_cache.get(cache_key)
-    if rv is None:
-        return None, None
-    return tuple(rv)
-
-
-def set_assemble_status(project, checksum, state, detail=None):
-    cache_key = 'assemble-status:%s' % _get_idempotency_id(
-        project, checksum)
-
-    # NB: Also cache successfully created debug files to avoid races between
-    # multiple DIFs with the same identifier. On the downside, this blocks
-    # re-uploads for 10 minutes.
-    default_cache.set(cache_key, (state, detail), 600)
-
-
 class BadDif(Exception):
     pass
 

src/sentry/models/file.py

@@ -57,19 +57,6 @@ class nooplogger(object):
     exception = staticmethod(lambda *a, **kw: None)
 
 
-def enum(**named_values):
-    return type('Enum', (), named_values)
-
-
-ChunkFileState = enum(
-    OK='ok',  # File in database
-    NOT_FOUND='not_found',  # File not found in database
-    CREATED='created',  # File was created in the request and send to the worker for assembling
-    ASSEMBLING='assembling',  # File still being processed by worker
-    ERROR='error'  # Error happened during assembling
-)
-
-
 def _get_size_and_checksum(fileobj, logger=nooplogger):
     logger.info('_get_size_and_checksum.start')
     size = 0