Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 38 additions & 31 deletions broadcastclient/broadcastclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
"math/rand/v2"
"net"
"net/http"
"strconv"
Expand Down Expand Up @@ -141,7 +142,6 @@ type BroadcastClient struct {

retrying bool
shuttingDown bool
firstReconnectAttempt bool
confirmedSequenceNumberListener chan arbutil.MessageIndex
txStreamer TransactionStreamerInterface
fatalErrChan chan error
Expand Down Expand Up @@ -178,7 +178,6 @@ func NewBroadcastClient(
fatalErrChan: fatalErrChan,
sigVerifier: sigVerifier,
adjustCount: adjustCount,
firstReconnectAttempt: true,
}, err
}

Expand Down Expand Up @@ -362,7 +361,6 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa
bc.connMutex.Lock()
bc.conn = conn
bc.compression = compressionNegotiated
bc.firstReconnectAttempt = true
bc.connMutex.Unlock()
log.Info("Feed connected", "feedServerVersion", feedServerVersion, "chainId", chainId, "requestedSeqNum", nextSeqNum)

Expand Down Expand Up @@ -415,24 +413,7 @@ func (bc *BroadcastClient) startBackgroundReader(earlyFrameData io.Reader) {
}
_ = bc.conn.Close()

// Skip backoff for first reconnection attempt
if bc.firstReconnectAttempt {
bc.firstReconnectAttempt = false
log.Info("First reconnection attempt, skipping backoff", "url", bc.websocketUrl)
earlyFrameData = bc.retryConnect(ctx)
continue
}
timer := time.NewTimer(backoffDuration)
if backoffDuration < bc.config().ReconnectMaximumBackoff {
backoffDuration *= 2
}
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
earlyFrameData = bc.retryConnect(ctx)
earlyFrameData, backoffDuration = bc.retryConnect(ctx, backoffDuration)
continue
}
backoffDuration = bc.config().ReconnectInitialBackoff
Expand All @@ -450,7 +431,6 @@ func (bc *BroadcastClient) startBackgroundReader(earlyFrameData io.Reader) {
sourcesDisconnectedGauge.Dec(1)
sourcesConnectedGauge.Inc(1)
bc.adjustCount(1)
bc.firstReconnectAttempt = true
}
if len(res.Messages) > 0 {
log.Debug("received batch item", "count", len(res.Messages), "first seq", res.Messages[0].SequenceNumber)
Expand Down Expand Up @@ -512,32 +492,59 @@ func (bc *BroadcastClient) isShuttingDown() bool {
return bc.shuttingDown
}

func (bc *BroadcastClient) retryConnect(ctx context.Context) io.Reader {
maxWaitDuration := 15 * time.Second
var waitDuration time.Duration = 0 // Don't wait for first reconnect
func (bc *BroadcastClient) retryConnect(ctx context.Context, backoffDuration time.Duration) (io.Reader, time.Duration) {
bc.retrying = true

for !bc.isShuttingDown() {
config := bc.config()
backoffDuration = clampReconnectBackoff(backoffDuration, config.ReconnectMaximumBackoff)
waitDuration := reconnectFullJitter(backoffDuration, rand.Int64N)
timer := time.NewTimer(waitDuration)
select {
case <-ctx.Done():
timer.Stop()
return nil
return nil, backoffDuration
case <-timer.C:
}

bc.retryCount.Add(1)
earlyFrameData, err := bc.connect(ctx, bc.nextSeqNum)
nextBackoff := nextReconnectBackoff(backoffDuration, config.ReconnectMaximumBackoff)
if err == nil {
bc.retrying = false
return earlyFrameData
return earlyFrameData, nextBackoff
}
log.Debug("failed reconnect to sequencer broadcast", "url", bc.websocketUrl, "err", err, "retryBackoff", backoffDuration)
backoffDuration = nextBackoff
}
return nil, backoffDuration
}

if waitDuration < maxWaitDuration {
waitDuration += 500 * time.Millisecond
}
func clampReconnectBackoff(backoffDuration, maximumBackoff time.Duration) time.Duration {
if maximumBackoff > 0 && backoffDuration > maximumBackoff {
return maximumBackoff
}
return backoffDuration
}

func nextReconnectBackoff(backoffDuration, maximumBackoff time.Duration) time.Duration {
if backoffDuration <= 0 || maximumBackoff <= 0 {
return backoffDuration
}
if backoffDuration >= maximumBackoff {
return maximumBackoff
}
if backoffDuration > maximumBackoff/2 {
return maximumBackoff
}
return backoffDuration * 2
}

func reconnectFullJitter(backoffDuration time.Duration, randomInt64N func(int64) int64) time.Duration {
if backoffDuration <= 0 {
return 0
}
return nil
return time.Duration(randomInt64N(int64(backoffDuration)))
}

func (bc *BroadcastClient) StopAndWait() {
Expand Down
43 changes: 43 additions & 0 deletions broadcastclient/broadcastclient_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,49 @@ import (
"github.com/offchainlabs/nitro/wsbroadcastserver"
)

func TestReconnectBackoffSequence(t *testing.T) {
initial := time.Second
maximum := 8 * time.Second
backoff := initial

for _, expected := range []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 8 * time.Second} {
backoff = nextReconnectBackoff(backoff, maximum)
if backoff != expected {
t.Fatalf("expected backoff %v, got %v", expected, backoff)
}
}

if got := clampReconnectBackoff(16*time.Second, maximum); got != maximum {
t.Fatalf("expected hot-reloaded maximum %v, got %v", maximum, got)
}
if got := nextReconnectBackoff(0, maximum); got != 0 {
t.Fatalf("zero backoff should remain disabled, got %v", got)
}
if got := nextReconnectBackoff(time.Second, 0); got != time.Second {
t.Fatalf("zero maximum should preserve the configured backoff, got %v", got)
}
}

func TestReconnectFullJitter(t *testing.T) {
backoff := 10 * time.Second
got := reconnectFullJitter(backoff, func(n int64) int64 {
if n != int64(backoff) {
t.Fatalf("expected jitter bound %v, got %v", backoff, time.Duration(n))
}
return n / 2
})
if got != backoff/2 {
t.Fatalf("expected deterministic jitter %v, got %v", backoff/2, got)
}

if got := reconnectFullJitter(0, func(int64) int64 {
t.Fatal("random source must not be called for zero backoff")
return 0
}); got != 0 {
t.Fatalf("expected zero jitter for disabled backoff, got %v", got)
}
}

func feedMessage(t *testing.T, b *broadcaster.Broadcaster, seqNum arbutil.MessageIndex) []*message.BroadcastFeedMessage {
msg := arbostypes.MessageWithMetadataAndBlockInfo{
MessageWithMeta: arbostypes.EmptyTestMessageWithMetadata,
Expand Down
Loading