fix(test-optimization): retry payload delivery failures

integration-tests/ci-visibility/playwright-tests-dynamic/dynamic-name-test.js

@@ -1,41 +1,38 @@
 'use strict'
 
-const crypto = require('crypto')
 const { test, expect } = require('@playwright/test')
 
-const uuid = crypto.randomBytes(16).toString('hex')
-  .replace(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '$1-$2-$3-$4-$5')
-
 test.describe('dynamic name suite', () => {
-  test(`can do stuff at ${Date.now()}`, () => {
+  // Playwright loads this file during discovery and again in workers, so test names must match across processes.
+  test('can do stuff at 1750000000000', () => {
     expect(1 + 2).toBe(3)
   })
 
-  test(`connects to localhost:${3000 + Math.floor(Math.random() * 60000)}`, () => {
+  test('connects to localhost:54321', () => {
     expect(2 + 3).toBe(5)
   })
 
-  test(`user session ${uuid}`, () => {
+  test('user session 12345678-1234-1234-1234-123456789abc', () => {
     expect(3 + 4).toBe(7)
   })
 
-  test(`created at ${new Date().toISOString()}`, () => {
+  test('created at 2026-08-31T12:34:56.789Z', () => {
     expect(4 + 5).toBe(9)
   })
 
-  test(`event on ${new Date().toISOString().split('T')[0]}`, () => {
+  test('event on 2026-08-31', () => {
     expect(5 + 6).toBe(11)
   })
 
-  test(`probability ${Math.random()}`, () => {
+  test('probability 0.1234567890', () => {
     expect(6 + 7).toBe(13)
   })
 
-  test(`server at 127.0.0.1:${3000 + Math.floor(Math.random() * 60000)}`, () => {
+  test('server at 127.0.0.1:54322', () => {
     expect(7 + 8).toBe(15)
   })
 
-  test(`bound to 0.0.0.0:${3000 + Math.floor(Math.random() * 60000)}`, () => {
+  test('bound to 0.0.0.0:54323', () => {
     expect(8 + 9).toBe(17)
   })
 })

integration-tests/helpers/index.js

@@ -886,9 +886,10 @@ async function curlAndAssertMessage (agent, procOrUrl, fn, timeout, expectedMess
  */
 function getCiVisAgentlessConfig (port) {
   // We remove GITHUB_WORKSPACE so the repository root is not assigned to dd-trace-js.
-  // The outer workflow's event payload references commits that do not exist in the sandbox repository.
+  // The outer workflow's GitHub metadata references commits that do not exist in the sandbox repository.
+  // GITHUB_RUN_ID is Cucumber's primary key for GitHub detection; without it, the missing event file is ignored.
   // We remove MOCHA_OPTIONS so the test runner doesn't run the tests twice
-  const { GITHUB_EVENT_PATH, GITHUB_WORKSPACE, MOCHA_OPTIONS, ...rest } = process.env
+  const { GITHUB_ACTIONS, GITHUB_EVENT_PATH, GITHUB_RUN_ID, GITHUB_WORKSPACE, MOCHA_OPTIONS, ...rest } = process.env
   return {
     ...rest,
     DD_API_KEY: '1',
@@ -905,9 +906,10 @@ function getCiVisAgentlessConfig (port) {
  */
 function getCiVisEvpProxyConfig (port) {
   // We remove GITHUB_WORKSPACE so the repository root is not assigned to dd-trace-js.
-  // The outer workflow's event payload references commits that do not exist in the sandbox repository.
+  // The outer workflow's GitHub metadata references commits that do not exist in the sandbox repository.
+  // GITHUB_RUN_ID is Cucumber's primary key for GitHub detection; without it, the missing event file is ignored.
   // We remove MOCHA_OPTIONS so the test runner doesn't run the tests twice
-  const { GITHUB_EVENT_PATH, GITHUB_WORKSPACE, MOCHA_OPTIONS, ...rest } = process.env
+  const { GITHUB_ACTIONS, GITHUB_EVENT_PATH, GITHUB_RUN_ID, GITHUB_WORKSPACE, MOCHA_OPTIONS, ...rest } = process.env
   return {
     ...rest,
     DD_TRACE_AGENT_PORT: String(port),

integration-tests/playwright/playwright-atr.spec.js

@@ -15,6 +15,9 @@ const {
 const { createWebAppServer } = require('../ci-visibility/web-app-server')
 const {
   TEST_STATUS,
+  TEST_NAME,
+  TEST_IS_NEW,
+  TEST_HAS_DYNAMIC_NAME,
   TEST_IS_RETRY,
   TEST_RETRY_REASON,
   TEST_HAS_FAILED_ALL_RETRIES,
@@ -248,6 +251,19 @@ versions.forEach((version) => {
         })
         receiver.setKnownTests({ playwright: {} })
 
+        const eventsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            const uniqueTests = new Map(tests.map(test => [test.meta[TEST_NAME], test]))
+
+            assert.strictEqual(uniqueTests.size, 8)
+            for (const test of uniqueTests.values()) {
+              assert.strictEqual(test.meta[TEST_IS_NEW], 'true')
+              assert.strictEqual(test.meta[TEST_HAS_DYNAMIC_NAME], 'true')
+            }
+          }, 30000)
+
         const proc = run(
           './node_modules/.bin/playwright test -c playwright.config.js',
           {
@@ -263,14 +279,9 @@ versions.forEach((version) => {
         proc.stdout?.on('data', chunk => { testOutput += chunk.toString() })
         proc.stderr?.on('data', chunk => { testOutput += chunk.toString() })
 
-        const eventsPromise = receiver
-          .gatherPayloadsUntilChildExit(proc, ({ url }) => url.endsWith('/api/v2/citestcycle'), () => {
-            assert.match(testOutput, /detected as new but their names contain dynamic data/)
-          })
-
         const [[exitCode]] = await Promise.all([once(proc, 'exit'), eventsPromise])
-        // Dynamic names differ between discovery and worker processes, so Playwright cannot find these tests.
-        assert.strictEqual(exitCode, 1, testOutput)
+        assert.strictEqual(exitCode, 0, testOutput)
+        assert.match(testOutput, /detected as new but their names contain dynamic data/)
       })
     })
   })

packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -7,6 +7,7 @@ const CiVisibilityExporter = require('../ci-visibility-exporter')
 const request = require('../request')
 const { fetchAgentInfo } = require('../../../agent/info')
 const { DEBUGGER_INPUT_V1 } = require('../../../debugger/constants')
+const { FINAL_FLUSH_TIMEOUT } = require('../../final-flush')
 
 // Product-specific discovery: newest advertised version, skip v3 (citestcycle), gzip if >= v4.
 // Shared `evp_proxy` discovery is an explicit path allowlist and does not cover this contract.
@@ -47,7 +48,16 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
     } = config
 
     const initializationController = new AbortController()
-    const initializationOptions = { signal: initializationController.signal }
+    const initializationOptions = {
+      deadline: Date.now() + FINAL_FLUSH_TIMEOUT,
+      signal: initializationController.signal,
+      // Test runners await agent discovery before starting. A detached retry can let Node exit
+      // while that promise is still pending because promises alone do not keep the event loop alive.
+      keepProcessAlive: true,
+      // Loading a test framework can block the event loop past the payload creation-time timeout.
+      // The transport timeout still bounds each attempt, and the deadline bounds all retries.
+      timeoutFromCreation: false,
+    }
     this._initializationRequest = {
       controller: initializationController,
       options: initializationOptions,

packages/dd-trace/src/ci-visibility/exporters/request.js

@@ -180,6 +180,8 @@ function requestBuffered (data, options, callback, reservedPayloadSize) {
   const timeout = options.timeout || 2000
   const payloadSize = reservedPayloadSize ?? getPayloadSize(data)
   let retryTimer
+  let attemptTimer
+  let attemptTimerImmediate
   let attemptController
   let settled = false
   let lastError
@@ -197,6 +199,8 @@ function requestBuffered (data, options, callback, reservedPayloadSize) {
     if (settled) return
     settled = true
     clearTimeout(retryTimer)
+    clearTimeout(attemptTimer)
+    clearImmediate(attemptTimerImmediate)
     signal?.removeEventListener('abort', onAbort)
     bufferedBytes -= payloadSize
     callback(error, result, statusCode, headers)
@@ -244,40 +248,66 @@ function requestBuffered (data, options, callback, reservedPayloadSize) {
     if (!commonRequest.writable) {
       waitingForBackpressure = true
       retryTimer = setTimeout(attempt, Math.min(BACKPRESSURE_RETRY_MS, remaining), attemptIndex)
-      retryTimer.unref?.()
+      if (!options.keepProcessAlive) retryTimer.unref?.()
       return
     }
     waitingForBackpressure = false
 
-    const attemptDeadline = deadline
     const attemptOptions = {
       ...options,
+      deferTimeoutAbort: true,
       headers: options.headers ? { ...options.headers } : undefined,
       retry: false,
     }
     if (deadline !== undefined) attemptOptions.timeout = Math.max(1, Math.min(timeout, remaining))
 
     const controller = new AbortController()
+    const attemptTimeout = attemptOptions.timeout || timeout
+    let attemptTimedOut = false
+
     attemptController = controller
     attemptOptions.signal = controller.signal
+    if (options.timeoutFromCreation !== false) {
+      attemptTimer = setTimeout(() => {
+        // Let a response that became ready while the event loop was blocked win before aborting it.
+        attemptTimerImmediate = setImmediate(() => {
+          if (settled || attemptController !== controller) return
+          attemptTimedOut = true
+          controller.abort(createRequestTimeoutError())
+        })
+        if (!options.keepProcessAlive) attemptTimerImmediate.unref?.()
+      }, attemptTimeout)
+      if (!options.keepProcessAlive) attemptTimer.unref?.()
+    }
 
     commonRequest(data, attemptOptions, (error, result, statusCode, headers) => {
+      clearTimeout(attemptTimer)
+      clearImmediate(attemptTimerImmediate)
       if (attemptController === controller) attemptController = undefined
       if (settled) return
       if (!error) {
         complete(null, result, statusCode, headers)
         return
       }
 
-      lastError = error
+      const requestError = attemptTimedOut ? createRequestTimeoutError() : error
+      lastError = requestError
 
       const responseStatus = statusCode ?? error.status
-      const isRetriableError = isRetriableNetworkError(error) || isRetriableHttpStatusCode(responseStatus)
-      const deadlineExtended = options.deadline !== undefined &&
-        (attemptDeadline === undefined || options.deadline > attemptDeadline)
-      const reachedAttemptLimit = attemptIndex >= getMaxAttempts(attemptOptions) && !deadlineExtended
-      if (options.retry === false || !isRetriableError || reachedAttemptLimit) {
-        complete(error, result, statusCode, headers)
+      const isUnknownNetworkError = responseStatus === undefined && error.code === undefined
+      const isRetriableError =
+        attemptTimedOut || isRetriableNetworkError(error) || isUnknownNetworkError ||
+          isRetriableHttpStatusCode(responseStatus)
+      const retryUntilDeadline = options.deadline !== undefined && options.retryUntilDeadline !== false
+      const reachedAttemptLimit = !retryUntilDeadline && attemptIndex >= getMaxAttempts(attemptOptions)
+      const reachedUnknownNetworkAttemptLimit = isUnknownNetworkError && attemptIndex >= 2
+      if (
+        options.retry === false ||
+        !isRetriableError ||
+        reachedAttemptLimit ||
+        reachedUnknownNetworkAttemptLimit
+      ) {
+        complete(requestError, result, statusCode, headers)
         return
       }
 
@@ -288,11 +318,11 @@ function requestBuffered (data, options, callback, reservedPayloadSize) {
           if (options.deadline !== undefined) {
             const retryRemaining = options.deadline - Date.now()
             if (resetDelay >= retryRemaining) {
-              complete(error, result, statusCode, headers)
+              complete(requestError, result, statusCode, headers)
               return
             }
           } else if (resetDelay > RATE_LIMIT_MAX_WAIT_MS) {
-            complete(error, result, statusCode, headers)
+            complete(requestError, result, statusCode, headers)
             return
           }
           retryDelay = resetDelay
@@ -303,15 +333,15 @@ function requestBuffered (data, options, callback, reservedPayloadSize) {
       if (options.deadline !== undefined && retryDelay === undefined) {
         const retryRemaining = options.deadline - Date.now()
         if (retryRemaining <= 0) {
-          complete(error, result, statusCode, headers)
+          complete(requestError, result, statusCode, headers)
           return
         }
         const retryAttemptTimeout = timeout < retryRemaining ? timeout : Math.ceil(retryRemaining / 2)
         delay = Math.min(delay, Math.max(0, retryRemaining - retryAttemptTimeout))
       }
 
       retryTimer = setTimeout(attempt, delay, attemptIndex + 1)
-      retryTimer.unref?.()
+      if (!options.keepProcessAlive) retryTimer.unref?.()
     })
   }
 

packages/dd-trace/src/ci-visibility/requests/upload-test-screenshot.js

@@ -130,6 +130,7 @@ function uploadTestScreenshot (
     timeout: UPLOAD_TIMEOUT_MS,
     url,
     deadline,
+    retryUntilDeadline: false,
     signal,
   }
 

packages/dd-trace/src/exporters/common/request.js

@@ -182,6 +182,7 @@ function request (data, options, callback) {
     legacyStorage.run({ noop: true }, () => {
       let finished = false
       let settled = false
+      let timeoutImmediate
       const finalize = () => {
         if (finished) return
         finished = true
@@ -197,6 +198,7 @@ function request (data, options, callback) {
       const complete = (error, result, statusCode, headers) => {
         if (settled) return
         settled = true
+        clearImmediate(timeoutImmediate)
         finalize()
         callback(error, result, statusCode, headers)
       }
@@ -206,6 +208,7 @@ function request (data, options, callback) {
        */
       const handleError = (error) => {
         if (settled) return
+        clearImmediate(timeoutImmediate)
 
         if (options.retry !== false &&
             attemptIndex < getMaxAttempts(options) &&
@@ -227,7 +230,8 @@ function request (data, options, callback) {
       req.once('timeout', finalize)
       req.once('error', handleError)
 
-      req.setTimeout(timeout, () => {
+      const abortRequest = () => {
+        if (settled) return
         try {
           if (typeof req.abort === 'function') {
             req.abort()
@@ -237,6 +241,16 @@ function request (data, options, callback) {
         } catch {
           // ignore
         }
+      }
+
+      req.setTimeout(timeout, () => {
+        if (!options.deferTimeoutAbort) {
+          abortRequest()
+          return
+        }
+
+        timeoutImmediate = setImmediate(abortRequest)
+        if (!options.keepProcessAlive) timeoutImmediate.unref?.()
       })
 
       for (const buffer of dataArray) req.write(buffer)

packages/dd-trace/test/ci-visibility/exporters/agent-proxy/agent-proxy.spec.js

@@ -97,6 +97,17 @@ describe('AgentProxyCiVisibilityExporter', () => {
     assert.strictEqual(scope.isDone(), true)
   })
 
+  it('retries agent info initialization within the final flush timeout', () => {
+    const clock = sinon.useFakeTimers()
+    try {
+      const controlled = createControlledExporter()
+
+      assert.strictEqual(controlled.getRequestOptions().deadline, Date.now() + FINAL_FLUSH_TIMEOUT)
+    } finally {
+      clock.restore()
+    }
+  })
+
   it('exports buffered data and flushes it when initialization finishes within the final deadline', async () => {
     const clock = sinon.useFakeTimers()
     try {
@@ -108,6 +119,8 @@ describe('AgentProxyCiVisibilityExporter', () => {
       controlled.exporter.flush(done)
 
       const requestOptions = controlled.getRequestOptions()
+      assert.strictEqual(requestOptions.keepProcessAlive, true)
+      assert.strictEqual(requestOptions.timeoutFromCreation, false)
       assert.strictEqual(requestOptions.signal.aborted, false)
       assert.strictEqual(requestOptions.deadline, Date.now() + FINAL_FLUSH_TIMEOUT)