Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/sentry/options/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,13 @@
default={"default": {"hosts": {0: {"host": "127.0.0.1", "port": 6379}}}},
flags=FLAG_NOSTORE | FLAG_IMMUTABLE,
)
# Kill switch for retrying Redis Cluster client creation when no startup node is reachable.
register(
"redis.cluster.create-retry.enabled",
type=Bool,
default=True,
flags=FLAG_AUTOMATOR_MODIFIABLE,
)

# Processing worker caches
register(
Expand Down
39 changes: 35 additions & 4 deletions src/sentry/utils/redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
import importlib.resources
import inspect
import logging
from collections.abc import Generator
from collections.abc import Callable, Generator
from copy import deepcopy
from typing import Any, Literal, TypeVar, cast, overload

from django.utils.functional import SimpleLazyObject
from redis.client import Script, StrictRedis
from rediscluster.exceptions import RedisClusterException
from sentry_redis_tools.clients import RedisCluster
from sentry_redis_tools.failover_redis import FailoverRedis
from sentry_redis_tools.retrying_cluster import RetryingRedisCluster
Expand Down Expand Up @@ -37,6 +38,34 @@
"max_connections": 16,
}

# Short network blips can leave every startup node briefly unreachable.
# Building a client only reads cluster topology, so retrying it cannot double-apply a write.
_UNREACHABLE_CLUSTER_MESSAGE = "Redis Cluster cannot be connected"


def _create_cluster_with_retry(
create: Callable[[], RetryingRedisCluster],
) -> RetryingRedisCluster:
"""
Call `create`, retrying once when no startup node is reachable. Other
cluster errors raise immediately.

There is no sleep between attempts: a failed attempt has usually already
waited out the socket timeout on each unreachable startup node.
"""
try:
return create()
except RedisClusterException as e:
if not str(e).startswith(_UNREACHABLE_CLUSTER_MESSAGE):
raise
if not options.get("redis.cluster.create-retry.enabled"):
raise
# Deferred: this module is imported before Django settings, which metrics reads.
from sentry.utils import metrics

metrics.incr("redis.cluster.create.retry")
return create()


class RedisClusterManager:
def __init__(self, options_manager: OptionsManager) -> None:
Expand Down Expand Up @@ -126,8 +155,9 @@ def cluster_factory() -> (
RedisCluster[bytes] | StrictRedis[bytes] | RedisCluster[str] | StrictRedis[str]
):
if is_redis_cluster:
cluster = _add_transaction_checks(
RetryingRedisCluster(

def create() -> RetryingRedisCluster:
return RetryingRedisCluster(
# Intentionally copy hosts here because redis-cluster-py
# mutates the inner dicts and this closure can be run
# concurrently, as SimpleLazyObject is not threadsafe. This
Expand All @@ -142,7 +172,8 @@ def cluster_factory() -> (
readonly_mode=readonly_mode,
**{**_REDIS_CLUSTER_DEFAULT_CLIENT_ARGS, **client_args},
)
)

cluster = _add_transaction_checks(_create_cluster_with_retry(create))
if key_prefix is not None:
return _add_key_prefix(cluster, key_prefix)
return cluster
Expand Down
59 changes: 59 additions & 0 deletions tests/sentry/utils/test_redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from django.db import transaction
from redis.client import Script, StrictRedis
from redis.exceptions import ResponseError
from rediscluster.exceptions import RedisClusterException
from sentry_redis_tools.clients import RedisCluster
from sentry_redis_tools.failover_redis import FailoverRedis

Expand Down Expand Up @@ -51,6 +52,11 @@ def _options_manager() -> OptionsManager:
)


_UNREACHABLE = RedisClusterException(
"Redis Cluster cannot be connected. Please provide at least one reachable node."
)


class ClusterManagerTestCase(TestCase):
def setUp(self) -> None:
imports._cache.clear()
Expand Down Expand Up @@ -107,6 +113,59 @@ def test_cluster_max_connections_from_client_args(
manager.get("raised")._setupfunc() # type: ignore[union-attr]
assert RetryingRedisCluster.call_args.kwargs["max_connections"] == 24

@mock.patch("sentry.utils.redis._add_transaction_checks", side_effect=lambda client: client)
@mock.patch("sentry.utils.redis.RetryingRedisCluster")
def test_cluster_creation_retries_once_when_unreachable(
self,
RetryingRedisCluster: mock.MagicMock,
_add_transaction_checks: mock.MagicMock,
) -> None:
RetryingRedisCluster.side_effect = [_UNREACHABLE, mock.sentinel.cluster]

manager = RedisClusterManager(_options_manager())

with override_options({"redis.cluster.create-retry.enabled": True}):
assert manager.get("baz")._setupfunc() is mock.sentinel.cluster # type: ignore[union-attr]
assert RetryingRedisCluster.call_count == 2

@mock.patch("sentry.utils.redis.RetryingRedisCluster")
def test_cluster_creation_gives_up_when_still_unreachable(
self, RetryingRedisCluster: mock.MagicMock
) -> None:
RetryingRedisCluster.side_effect = _UNREACHABLE

manager = RedisClusterManager(_options_manager())

with override_options({"redis.cluster.create-retry.enabled": True}):
with pytest.raises(RedisClusterException, match="cannot be connected"):
manager.get("baz")._setupfunc() # type: ignore[union-attr]
assert RetryingRedisCluster.call_count == 2

@mock.patch("sentry.utils.redis.RetryingRedisCluster")
def test_cluster_creation_does_not_retry_other_errors(
self, RetryingRedisCluster: mock.MagicMock
) -> None:
RetryingRedisCluster.side_effect = RedisClusterException(
"startup_nodes could not agree on a valid slots cache."
)

manager = RedisClusterManager(_options_manager())

with pytest.raises(RedisClusterException, match="could not agree"):
manager.get("baz")._setupfunc() # type: ignore[union-attr]
assert RetryingRedisCluster.call_count == 1

@mock.patch("sentry.utils.redis.RetryingRedisCluster")
def test_cluster_creation_retry_kill_switch(self, RetryingRedisCluster: mock.MagicMock) -> None:
RetryingRedisCluster.side_effect = _UNREACHABLE

manager = RedisClusterManager(_options_manager())

with override_options({"redis.cluster.create-retry.enabled": False}):
with pytest.raises(RedisClusterException, match="cannot be connected"):
manager.get("baz")._setupfunc() # type: ignore[union-attr]
assert RetryingRedisCluster.call_count == 1

@mock.patch("sentry.utils.redis.RetryingRedisCluster")
def test_multiple_retrieval_do_not_setup_lazy_object(
self, RetryingRedisCluster: mock.MagicMock
Expand Down
Loading