diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index 4411eff0a3..5895b78a1b 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -14,7 +14,8 @@ concurrency: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 15 + # larger than individual steps so logs can be collected + timeout-minutes: 20 env: RUN_EXTRA_TESTS: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'true' || 'false' }} steps: @@ -33,8 +34,34 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v6 - name: Run tests - run: ./gradlew test + timeout-minutes: 15 + working-directory: ./backend + run: | + mkdir -p build/diagnostics + # Thread dumps before the step cap, so a hang shows where it is stuck. + # `jcmd Gradle` matches every JVM whose main class contains "Gradle": the daemon and the test executor. + ( sleep 660; jcmd Gradle Thread.print; sleep 120; jcmd Gradle Thread.print ) \ + > build/diagnostics/threaddumps.txt 2>&1 & + watchdog=$! + rc=0 + ./gradlew test || rc=$? + kill "$watchdog" 2>/dev/null || true + exit $rc + - name: Show where the test run stopped + if: failure() working-directory: ./backend + # the suite runs one test at a time, so the last line is the test a hang was stuck in + run: tail -n 5 build/diagnostics/test-progress.log || echo "No progress log, so no test ever started." + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v7 + with: + name: backend-test-results + path: | + backend/build/test-results/test/*.xml + backend/build/diagnostics/** + backend/build/reports/tests/test/index.html + if-no-files-found: warn lint: runs-on: ubuntu-latest diff --git a/backend/build.gradle b/backend/build.gradle index 0ff1f8937a..1194324cf9 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -1,3 +1,6 @@ +import org.gradle.api.tasks.testing.TestDescriptor +import org.gradle.api.tasks.testing.TestListener +import org.gradle.api.tasks.testing.TestResult import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.gradle.api.tasks.testing.logging.TestLogEvent @@ -129,6 +132,26 @@ tasks.named('test') { exceptionFormat = TestExceptionFormat.FULL showExceptions = true } + // Records each test's start and finish so a hung run names the test it was stuck in. + def progressLog = layout.buildDirectory.file('diagnostics/test-progress.log') + doFirst { + def file = progressLog.get().asFile + file.parentFile.mkdirs() + file.text = '' + } + addTestListener(new TestListener() { + void beforeSuite(TestDescriptor suite) {} + + void afterSuite(TestDescriptor suite, TestResult result) {} + + void beforeTest(TestDescriptor descriptor) { + progressLog.get().asFile << "${java.time.Instant.now()} STARTED ${descriptor.className}.${descriptor.name}\n" + } + + void afterTest(TestDescriptor descriptor, TestResult result) { + progressLog.get().asFile << "${java.time.Instant.now()} ${result.resultType} ${descriptor.className}.${descriptor.name}\n" + } + }) } tasks.named('bootBuildImage') {