Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 41 additions & 33 deletions src/sentry/debug_files/artifact_bundles.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import orjson
import sentry_sdk
from django.conf import settings
from django.db import router
from django.db import IntegrityError, router
from django.db.models import Count, Exists, OuterRef
from django.utils import timezone
from sentry_redis_tools.clients import RedisCluster
Expand Down Expand Up @@ -186,38 +186,46 @@ def index_urls_in_bundle(

# We want to start a transaction for each bundle, so that in case of failures we keep consistency at the
# bundle level, and we also have to retry only the failed bundle in the future and not all the bundles.
with atomic_transaction(
using=(
router.db_for_write(ArtifactBundle),
router.db_for_write(ArtifactBundleIndex),
)
):
# Since we use read committed isolation, the value we read here can change after query execution,
# but we have this check in place for analytics purposes.
bundle_was_indexed = ArtifactBundle.objects.filter(
id=artifact_bundle.id,
indexing_state=ArtifactBundleIndexingState.WAS_INDEXED.value,
).exists()
# If the bundle was already indexed, we will skip insertion into the database.
if bundle_was_indexed:
metrics.incr("artifact_bundle_indexing.bundle_was_already_indexed")
return

# Insert the index
# NOTE: The django ORM by default tries to batch *all* the inserts into a single query,
# which is not quite that efficient. We want to have a fixed batch size,
# which will result in a fixed number of unique `INSERT` queries.
ArtifactBundleIndex.objects.bulk_create(urls_to_index, batch_size=50)

# Mark the bundle as indexed
ArtifactBundle.objects.filter(id=artifact_bundle.id).update(
indexing_state=ArtifactBundleIndexingState.WAS_INDEXED.value
)

metrics.incr("artifact_bundle_indexing.bundles_indexed")
metrics.incr("artifact_bundle_indexing.urls_indexed", len(urls_to_index))
if skipped_urls:
metrics.incr("artifact_bundle_indexing.urls_skipped", skipped_urls)
try:
with atomic_transaction(
using=(
router.db_for_write(ArtifactBundle),
router.db_for_write(ArtifactBundleIndex),
)
):
# Since we use read committed isolation, the value we read here can change after query execution,
# but we have this check in place for analytics purposes.
bundle_was_indexed = ArtifactBundle.objects.filter(
id=artifact_bundle.id,
indexing_state=ArtifactBundleIndexingState.WAS_INDEXED.value,
).exists()
# If the bundle was already indexed, we will skip insertion into the database.
if bundle_was_indexed:
metrics.incr("artifact_bundle_indexing.bundle_was_already_indexed")
return

# Insert the index
# NOTE: The django ORM by default tries to batch *all* the inserts into a single query,
# which is not quite that efficient. We want to have a fixed batch size,
# which will result in a fixed number of unique `INSERT` queries.
ArtifactBundleIndex.objects.bulk_create(urls_to_index, batch_size=50)

# Mark the bundle as indexed
ArtifactBundle.objects.filter(id=artifact_bundle.id).update(
indexing_state=ArtifactBundleIndexingState.WAS_INDEXED.value
)
except IntegrityError:
# An upload of the same bundle can delete this one as a duplicate. The foreign key is
# deferred, so that only raises on commit.
if ArtifactBundle.objects.filter(id=artifact_bundle.id).exists():
raise
metrics.incr("artifact_bundle_indexing.bundle_deleted_during_indexing")
return

metrics.incr("artifact_bundle_indexing.bundles_indexed")
metrics.incr("artifact_bundle_indexing.urls_indexed", len(urls_to_index))
if skipped_urls:
metrics.incr("artifact_bundle_indexing.urls_skipped", skipped_urls)


# ===== Renewal of Artifact Bundles =====
Expand Down
28 changes: 26 additions & 2 deletions tests/sentry/debug_files/test_artifact_bundles.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from hashlib import sha1
from io import BytesIO
from typing import Literal, TypedDict
from unittest.mock import patch
from unittest.mock import MagicMock, patch

from django.core.files.base import ContentFile
from django.db import connections, router
Expand Down Expand Up @@ -33,7 +33,7 @@
from sentry.models.files.fileblob import FileBlob
from sentry.models.project import Project
from sentry.tasks.assemble import assemble_artifacts
from sentry.testutils.cases import TestCase
from sentry.testutils.cases import TestCase, TransactionTestCase
from sentry.testutils.helpers.options import override_options
from sentry.utils import json

Expand Down Expand Up @@ -448,6 +448,30 @@ def test_skips_files_named_after_their_debug_id(self) -> None:
]


class IndexUrlsInDeletedBundleTest(TransactionTestCase):
# `TestCase` never commits, so it would never check the deferred foreign key.
@patch("sentry.debug_files.artifact_bundles.metrics.incr")
def test_bundle_deleted_before_commit(self, mock_incr: MagicMock) -> None:
artifact_bundle = self.create_artifact_bundle(artifact_count=1)
files: dict[str, _ArtifactFile] = {
"files/_/_/app.js": {
"url": "~/static/js/app.js",
"type": "minified_source",
"content": b"app",
},
}
with ArtifactBundleArchive(BytesIO(make_compressed_zip_file(files))) as archive:
# Simulate an upload of the same bundle deleting this one as a duplicate.
ArtifactBundle.objects.filter(id=artifact_bundle.id).delete()

index_urls_in_bundle(self.organization.id, artifact_bundle, archive)

mock_incr.assert_any_call("artifact_bundle_indexing.bundle_deleted_during_indexing")
assert not ArtifactBundleIndex.objects.filter(
artifact_bundle_id=artifact_bundle.id
).exists()


class GetArtifactBundlesContainingUrlTest(TestCase):
def setUp(self) -> None:
self.release_name = "1.0.0"
Expand Down
Loading