feat(e2e): add two-chain anvil environment

.github/workflows/test-e2e-multi-chains.yml

@@ -0,0 +1,240 @@
+name: Aggkit E2E - Multi Chain
+
+on:
+  workflow_call:
+    inputs:
+      aggsender-find-imported-bridge-artifact:
+        description: "Artifact containing the aggsender imported-bridge helper"
+        required: true
+        type: string
+      kurtosis-cdk-ref:
+        description: "The kurtosis-cdk revision to test"
+        required: true
+        type: string
+      agglayer-e2e-ref:
+        description: "The agglayer/e2e revision containing the Bats tests"
+        required: true
+        type: string
+      kurtosis-cdk-args-1:
+        description: "Kurtosis arguments for the first chain"
+        required: true
+        type: string
+      kurtosis-cdk-args-2:
+        description: "Kurtosis arguments for the second chain"
+        required: true
+        type: string
+      kurtosis-cdk-args-3:
+        description: "Kurtosis arguments for the optional third chain"
+        required: false
+        type: string
+        default: "{}"
+      kurtosis-cdk-enclave-name:
+        description: "Kurtosis enclave name"
+        required: true
+        type: string
+      docker-image-override:
+        description: "Kurtosis image key to override"
+        required: false
+        type: string
+        default: ""
+      docker-tag:
+        description: "Local Docker tag used by Kurtosis"
+        required: false
+        type: string
+        default: "local"
+      docker-artifact-name:
+        description: "Artifact containing the Docker image"
+        required: false
+        type: string
+        default: "aggkit"
+      number-of-chains:
+        description: "Number of chains to deploy"
+        required: false
+        type: number
+        default: 2
+      aggkit-config-artifact:
+        description: "Artifact containing aggkit-owned Kurtosis config templates"
+        required: false
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+  packages: write
+  id-token: write
+
+jobs:
+  test-multi-aggkit-e2e:
+    name: Multi chains E2E test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout agglayer-e2e
+        uses: actions/checkout@v6
+        with:
+          repository: agglayer/e2e
+          ref: ${{ inputs.agglayer-e2e-ref }}
+          path: agglayer-e2e
+
+      - name: Checkout kurtosis-cdk
+        uses: actions/checkout@v6
+        with:
+          repository: 0xPolygon/kurtosis-cdk
+          ref: ${{ inputs.kurtosis-cdk-ref }}
+          path: kurtosis-cdk
+
+      - name: Download binary
+        if: inputs.aggsender-find-imported-bridge-artifact != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.aggsender-find-imported-bridge-artifact }}
+          path: /tmp
+
+      - name: Download artifact
+        if: inputs.docker-artifact-name != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.docker-artifact-name }}
+          path: /tmp
+
+      - name: Load image
+        run: |
+          if [[ -e /tmp/${{ inputs.docker-artifact-name }}.tar ]]; then
+            docker load --input /tmp/${{ inputs.docker-artifact-name }}.tar
+            docker images
+          fi
+
+      # foundry-toolchain downloads foundryup from foundry.paradigm.xyz. The
+      # endpoint intermittently returns a non-shell executable on hosted
+      # runners, causing every retry to fail before the Bats tests start.
+      - name: Install Foundry
+        env:
+          FOUNDRY_VERSION: v1.7.0
+          FOUNDRY_SHA256: 88501301c43e2cb3231009e68bd76af17cc0f7e9981f9d37ceabc6b857febb2f
+        run: |
+          archive="foundry_${FOUNDRY_VERSION}_linux_amd64.tar.gz"
+          install_dir="${RUNNER_TEMP}/foundry/bin"
+          mkdir -p "${install_dir}"
+          curl -fsSLO "https://github.com/foundry-rs/foundry/releases/download/${FOUNDRY_VERSION}/${archive}"
+          echo "${FOUNDRY_SHA256}  ${archive}" | sha256sum --check
+          tar -xzf "${archive}" -C "${install_dir}"
+          "${install_dir}/cast" --version
+          echo "${install_dir}" >> "${GITHUB_PATH}"
+
+      - name: Install bats
+        uses: bats-core/bats-action@3.0.0
+
+      - name: Install kurtosis cli and jq
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli jq
+          kurtosis analytics disable
+
+      - name: Write args input to a file
+        run: |
+          if [[ '${{ inputs.kurtosis-cdk-args-3 }}' == '{}' && ${{ inputs.number-of-chains }} -eq 3 ]]; then
+            echo "Error: kurtosis-cdk-args-3 is empty but number-of-chains is set to 3"
+            exit 1
+          fi
+          docker_tag='${{ inputs.docker-tag }}'
+          if [[ "${docker_tag}" == "" ]]; then
+            echo '${{ inputs.kurtosis-cdk-args-1 }}' | tee /tmp/kurtosis-args-1.json
+            echo '${{ inputs.kurtosis-cdk-args-2 }}' | tee /tmp/kurtosis-args-2.json
+            if [[ ${{ inputs.number-of-chains }} -eq 3 ]]; then
+              echo '${{ inputs.kurtosis-cdk-args-3 }}' | tee /tmp/kurtosis-args-3.json
+            fi
+            exit
+          fi
+          echo '${{ inputs.kurtosis-cdk-args-1 }}' | jq --arg img '${{ inputs.docker-image-override }}' \
+            --arg tag '${{ inputs.docker-tag }}' '.args[$img] = $tag' | tee /tmp/kurtosis-args-1.json
+          echo '${{ inputs.kurtosis-cdk-args-2 }}' | jq --arg img '${{ inputs.docker-image-override }}' \
+            --arg tag '${{ inputs.docker-tag }}' '.args[$img] = $tag' | tee /tmp/kurtosis-args-2.json
+          if [[ ${{ inputs.number-of-chains }} -eq 3 ]]; then
+            echo '${{ inputs.kurtosis-cdk-args-3 }}' | jq --arg img '${{ inputs.docker-image-override }}' \
+              --arg tag '${{ inputs.docker-tag }}' '.args[$img] = $tag' | tee /tmp/kurtosis-args-3.json
+          fi
+
+      - name: Download aggkit config override
+        if: inputs.aggkit-config-artifact != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.aggkit-config-artifact }}
+          path: /tmp/aggkit-config
+
+      - name: Apply aggkit config override
+        if: inputs.aggkit-config-artifact != ''
+        run: |
+          set -eo pipefail
+          dst=kurtosis-cdk/static_files/chain/shared/aggkit
+          cp /tmp/aggkit-config/aggkit-config.template.toml "${dst}/config.toml"
+          if [[ -f /tmp/aggkit-config/aggkit-cdk-config.template.toml ]]; then
+            cp /tmp/aggkit-config/aggkit-cdk-config.template.toml "${dst}/cdk-config.toml"
+          fi
+          echo "Applied aggkit-owned config templates over kurtosis-cdk static_files:"
+          ls -l "${dst}"
+
+      - name: Startup the kurtosis-cdk package
+        run: |
+          pushd kurtosis-cdk || exit 1
+          kurtosis run --enclave '${{ inputs.kurtosis-cdk-enclave-name }}' --args-file /tmp/kurtosis-args-1.json .
+          kurtosis run --enclave '${{ inputs.kurtosis-cdk-enclave-name }}' --args-file /tmp/kurtosis-args-2.json .
+          if [[ ${{ inputs.number-of-chains }} -eq 3 ]]; then
+            kurtosis run --enclave '${{ inputs.kurtosis-cdk-enclave-name }}' --args-file /tmp/kurtosis-args-3.json .
+          fi
+          popd
+
+      - name: Run e2e tests
+        run: |
+          pushd agglayer-e2e || exit 1
+          set -a
+          source ./tests/.env
+          set +a
+          export BATS_LIB_PATH="${PWD}/core/helpers/lib"
+          export PROJECT_ROOT="${PWD}"
+          export ENCLAVE_NAME='${{ inputs.kurtosis-cdk-enclave-name }}'
+          chmod +x "/tmp/${{ inputs.aggsender-find-imported-bridge-artifact }}"
+          export AGGSENDER_IMPORTED_BRIDGE_PATH="/tmp/${{ inputs.aggsender-find-imported-bridge-artifact }}"
+          if [[ ${{ inputs.number-of-chains }} -eq 3 ]]; then
+            bats ./tests/aggkit/bridge-e2e-3-chains.bats
+          else
+            bats ./tests/aggkit/bridge-e2e-2-chains.bats
+          fi
+          popd
+
+      - name: Dump enclave logs
+        if: ${{ failure() }}
+        run: kurtosis dump ./dump
+
+      - name: Generate archive name
+        if: ${{ failure() }}
+        run: |
+          archive_name="dump_run_with_args_${{ inputs.number-of-chains }}_${{ github.run_id }}"
+          echo "ARCHIVE_NAME=${archive_name}" >> "${GITHUB_ENV}"
+          echo "Generated archive name: ${archive_name}"
+
+      - name: Upload logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARCHIVE_NAME }}
+          path: ./dump
+
+      - name: Clean up kurtosis-cdk enclave
+        if: ${{ always() }}
+        run: |
+          kurtosis enclave stop '${{ inputs.kurtosis-cdk-enclave-name }}'
+          kurtosis clean
+
+      - name: Record test run in Datadog
+        if: always()
+        continue-on-error: true
+        uses: agglayer/gha-record-e2e-test-run@v1
+        with:
+          enable_ai_analysis: "true"
+          host: github-actions-runners
+          status: ${{ job.status }}
+          ref_kurtosis: ${{ inputs.kurtosis-cdk-ref }}
+          ref_e2e: ${{ inputs.agglayer-e2e-ref }}
+        env:
+          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

.github/workflows/test-e2e.yml

@@ -273,7 +273,7 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@e0dcceca73cc66c1dd577da2b491887586c0d4b8
+    uses: ./.github/workflows/test-e2e-multi-chains.yml
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}
@@ -301,7 +301,7 @@ jobs:
       - build-tools
       - read-aggkit-args
       - get-kurtosis-cdk-commit
-    uses: agglayer/e2e/.github/workflows/aggkit-e2e-multi-chains.yml@e0dcceca73cc66c1dd577da2b491887586c0d4b8
+    uses: ./.github/workflows/test-e2e-multi-chains.yml
     secrets: inherit
     with:
       kurtosis-cdk-ref: ${{ needs.get-kurtosis-cdk-commit.outputs.kurtosis-commit }}

.github/workflows/test-go-e2e.yml

@@ -43,22 +43,16 @@ jobs:
 
       - name: Pull Docker images from compose files
         run: |
-          # op-pp env
-          cd test/e2e/envs/op-pp
-          docker compose pull geth beacon validator agglayer op-geth-001 op-node-001
-          cd ../op-pp-2chains
-          # kurtosis-cdk snapshot branch: feat/op-pp-2chains-snapshot @ 627340aa
-          # This commit generated the op-pp-2chains env snapshot
-          docker compose pull geth beacon validator agglayer \
-            op-reth-001 op-node-001 op-reth-002 op-node-002
+          cd test/e2e/envs/anvil-2chains
+          # Digest-pinned kurtosis-cdk anvil snapshot images. The aggkit services
+          # deliberately use the locally built aggkit:local image and are excluded.
+          docker compose pull anvil-001 l2-anvil-001 l2-anvil-002 agglayer
 
       - name: Save pulled Docker images
         run: |
-          imgs=""
-          for d in op-pp op-pp-2chains; do
-            imgs="$imgs $(cd test/e2e/envs/$d && docker compose config --images | grep -v 'aggkit:local')"
-          done
-          docker save $(echo "$imgs" | tr ' ' '\n' | sort -u) -o /tmp/docker-images.tar
+          mapfile -t imgs < <(cd test/e2e/envs/anvil-2chains && \
+            docker compose config --images | grep -v 'aggkit:local' | sort -u)
+          docker save "${imgs[@]}" -o /tmp/docker-images.tar
 
       - name: Upload Docker images artifact
         uses: actions/upload-artifact@v4
@@ -76,29 +70,32 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Default op-pp group: everything except the remove-GER tests, which run in their own
-          # groups below (each matrix entry is a separate `go test` invocation, so listing a test in
-          # more than one entry's regex would run it twice). Enumerated as an explicit positive,
-          # anchored regex because Go's -run has no negation syntax.
-          - env: op-pp
-            group: default
-            run: "^(TestJustBridge|TestAutoClaimL1ToL2AllowAll|TestAutoClaimL1ToL2APIApprove|TestAutoClaimL1ToL2BasicFilter|TestAutoClaimL2ToL1AllowAll|TestAutoClaimL2ToL2AllowAll|TestBridgeL2ToL2|TestBackwardForwardLET_NoDivergence|TestBackwardForwardLET_Case1|TestBackwardForwardLET_Case2|TestBackwardForwardLET_Case3|TestBackwardForwardLET_Case4|TestBackwardForwardLET_AggsenderAPIFallback)$"
-          # Remove-GER groups (op-pp): split across 3 CI matrix entries under the 20-min-per-group
-          # budget. Measured passing-path wall-clock (S8/S9 reports) is well under this budget for
-          # every group (all five tests combined finish in under 6 minutes single-run), so no further
-          # regrouping is needed.
-          - env: op-pp
+          # Keep stateful suites in separate stacks. The AutoClaim and backward/forward LET tests
+          # restart services and mutate chain/AggLayer state, so sharing one go-test process makes
+          # a primary failure cascade into otherwise unrelated bridge tests.
+          - env: anvil-2chains
+            group: autoclaim
+            run: "^(TestAutoClaimL1ToL2AllowAll|TestAutoClaimL1ToL2APIApprove|TestAutoClaimL1ToL2BasicFilter|TestAutoClaimL2ToL1AllowAll|TestAutoClaimL2ToL2AllowAll)$"
+          - env: anvil-2chains
+            group: bridge
+            run: "^(TestJustBridge|TestBridgeL2ToL2|TestBridgeTrackerL1ToL2)$"
+          - env: anvil-2chains
+            group: backward-forward-let
+            run: "^(TestBackwardForwardLET_NoDivergence|TestBackwardForwardLET_Case1|TestBackwardForwardLET_Case2|TestBackwardForwardLET_Case3|TestBackwardForwardLET_Case4|TestBackwardForwardLET_AggsenderAPIFallback)$"
+          # Remove-GER scenarios mutate claim bitmaps and emergency state. Category A has a fixed
+          # deposit-count-2 proof and must not share a snapshot with a preceding bridge test.
+          - env: anvil-2chains
             group: removeger-fast
-            run: TestRemoveGER_NoProblematicClaims|TestRemoveGER_CategoryA|TestGenerateInvalidGER
-          - env: op-pp
+            run: "^(TestRemoveGER_NoProblematicClaims|TestGenerateInvalidGER)$"
+          - env: anvil-2chains
+            group: removeger-category-a
+            run: "^TestRemoveGER_CategoryA$"
+          - env: anvil-2chains
             group: removeger-b1
             run: TestRemoveGER_CategoryB1
-          - env: op-pp
+          - env: anvil-2chains
             group: removeger-b2
             run: TestRemoveGER_CategoryB2
-          - env: op-pp-2chains
-            group: default
-            run: TestBridgeL2ToL2|TestAutoClaimL2ToL2AllowAll|TestBridgeTrackerL1ToL2
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -113,6 +110,21 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y docker-compose
 
+      - name: Install cast
+        if: matrix.group == 'removeger-fast'
+        env:
+          FOUNDRY_VERSION: v1.8.1
+        run: |
+          archive="foundry_${FOUNDRY_VERSION}_linux_amd64.tar.gz"
+          install_dir="${RUNNER_TEMP}/foundry/bin"
+          mkdir -p "${install_dir}"
+          curl -fsSLO "https://github.com/foundry-rs/foundry/releases/download/${FOUNDRY_VERSION}/${archive}"
+          curl -fsSLO "https://github.com/foundry-rs/foundry/releases/download/${FOUNDRY_VERSION}/${archive%.tar.gz}.sha256"
+          sha256sum --check "${archive%.tar.gz}.sha256"
+          tar -xzf "${archive}" -C "${install_dir}" cast
+          "${install_dir}/cast" --version
+          echo "${install_dir}" >> "${GITHUB_PATH}"
+
       - name: Download aggkit Docker image
         uses: actions/download-artifact@v4
         with:
@@ -139,7 +151,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          # Include matrix.group: multiple matrix entries now share the same env (op-pp), and
+          # Include matrix.group because multiple matrix entries share the same Anvil env, and
           # actions/upload-artifact@v4 requires unique artifact names within a workflow run.
           name: e2e-test-results-${{ matrix.env }}-${{ matrix.group }}
           path: |
@@ -184,6 +196,8 @@ jobs:
           docker load -i /tmp/docker-images.tar
 
       - name: Run force_ger_update E2E test
+        env:
+          AGGKIT_E2E_ENV: anvil-2chains
         run: make test-e2e-force_ger_update
 
       - name: Upload test results

Makefile

@@ -154,14 +154,13 @@ test-unit: ## Runs the unit tests
 TEST_RUN ?=
 .PHONY: test-e2e
 test-e2e: ## Runs the e2e tests
-	# 60m: covers the CI matrix's remove-GER groups (test-go-e2e.yml), whose combined per-test context
-	# budgets can exceed the previous 45m; kept in step with job timeout-minutes: 60 there. Provisional
-	# -- S8/S9 tighten per-test timeouts and may lower this once real durations are measured.
+	# Keep this aligned with test-go-e2e.yml's job timeout; remove-GER scenarios have long retry budgets.
 	go test -v -timeout 60m $(if $(TEST_RUN),-run "$(TEST_RUN)") ./test/e2e/...
 
 .PHONY: test-e2e-force_ger_update
 test-e2e-force_ger_update: ## Runs the isolated force_ger_update e2e test (dedicated CI job/runner only)
-	RUN_FORCE_GER_UPDATE_E2E=true E2E_SKIP_POSTTEST_BRIDGE_CHECK=true go test -v -timeout 30m -run TestForceGERUpdateE2E ./test/e2e/...
+	AGGKIT_E2E_ENV=anvil-2chains RUN_FORCE_GER_UPDATE_E2E=true E2E_SKIP_POSTTEST_BRIDGE_CHECK=true \
+		go test -v -timeout 30m -run TestForceGERUpdateE2E ./test/e2e/...
 
 .PHONY: lint
 lint: ## Runs the linter