Add UI tests for the Jetpack app, with a fixtures backend that runs in CI

.buildkite/commands/boot-simulator.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -eu
+
+# Starts booting a Simulator, and returns without waiting for it.
+#
+# A test job spends most of a minute booting its Simulator once its tests are ready to run. Started
+# from the top of the job instead, the Simulator boots while the job downloads its build and
+# installs its gems.
+#
+# DEVICE is a name, such as "iPhone 18 Pro", or a name and an iOS version, such as
+# "iPhone 17 Pro (26.5)". With no version it's the newest iOS that has the device, which is the one
+# fastlane picks to run the tests on.
+
+DEVICE=${1:?Usage $0 DEVICE}
+
+UDID=$(xcrun simctl list devices available --json | ruby -rjson -e '
+  name, version = ARGV[0].match(/\A(.*?)(?: \(([\d.]+)\))?\z/).captures
+  candidates = JSON.parse($stdin.read)["devices"].flat_map do |runtime, devices|
+    os = runtime[/\.iOS-([\d-]+)\z/, 1]
+    next [] if os.nil?
+
+    devices.select { |device| device["name"] == name }.map { |device| [Gem::Version.new(os.tr("-", ".")), device["udid"]] }
+  end
+  candidates.select! { |os, _| os == Gem::Version.new(version) } unless version.nil?
+  puts candidates.max_by(&:first)&.last
+' "$DEVICE")
+
+if [[ -z "$UDID" ]]; then
+  echo "Found no Simulator for '$DEVICE' to boot ahead of the tests."
+  exit 0
+fi
+
+echo "Booting $DEVICE ($UDID) while the job sets up"
+xcrun simctl boot "$UDID" > /dev/null 2>&1 &

.buildkite/commands/build-for-testing.sh

@@ -18,8 +18,9 @@ fi
 
 "$(dirname "${BASH_SOURCE[0]}")/install-secrets.sh"
 
+# Anything after the app's name is an option for the lane, such as `http_fixtures:true`.
 echo "--- :hammer_and_wrench: Building"
-bundle exec fastlane "build_${APP}_for_testing"
+bundle exec fastlane "build_${APP}_for_testing" "${@:2}"
 
 echo "--- :arrow_up: Upload Build Products"
 tar -cf "build-products-${APP}.tar" DerivedData/Build/Products/

.buildkite/commands/run-swift-package-tests.sh

@@ -4,5 +4,6 @@ if "$(dirname "${BASH_SOURCE[0]}")/should-skip-job.sh" --job-type validation; th
   exit 0
 fi
 
+# The HTTPFixtures module and its tests are compiled out unless this condition is set.
 echo "--- :swift: Running cross-platform Swift package tests"
-swift test
+swift test -Xswiftc -DUI_TEST_HTTP_FIXTURES

.buildkite/commands/run-ui-tests.sh

@@ -0,0 +1,84 @@
+#!/bin/bash -eu
+
+if "$(dirname "${BASH_SOURCE[0]}")/should-skip-job.sh" --job-type validation; then
+  exit 0
+fi
+
+DEVICE=${1:?Usage $0 DEVICE AREA}
+AREA=${2:?Usage $0 DEVICE AREA}
+
+"$(dirname "${BASH_SOURCE[0]}")/boot-simulator.sh" "$DEVICE"
+
+# The suites are grouped by area of the app, one folder each, and each area runs in a job of its own.
+TESTS_DIR=Tests/JetpackUITests/Tests
+
+# Only the suites that run against the fixtures in Tests/JetpackUITests/Fixtures. The others sign
+# in to a real WordPress.com account, which CI doesn't have a token for.
+FIXTURE_SUITE='class var backend: Backend \{ \.fixtures \}'
+
+# A suite outside the folders the pipeline has a job for would never run here, so fail instead.
+for suite in $(grep -lE "$FIXTURE_SUITE" "$TESTS_DIR"/*.swift "$TESTS_DIR"/*/*.swift 2>/dev/null); do
+  area=$(basename "$(dirname "$suite")")
+  if ! grep -qE "^ +- \"$area\"\$" .buildkite/pipeline.yml; then
+    echo "Error: no UI Tests job runs $suite. Move it into an area's folder under $TESTS_DIR, or add its folder to the step's matrix in .buildkite/pipeline.yml."
+    exit 1
+  fi
+done
+
+SUITE_FILES=$(grep -lE "$FIXTURE_SUITE" "$TESTS_DIR/$AREA"/*.swift || true)
+
+if [[ -z "$SUITE_FILES" ]]; then
+  echo "Error: found no UI test suite that runs against the fixtures in $TESTS_DIR/$AREA"
+  exit 1
+fi
+
+SUITES=$(echo "$SUITE_FILES" \
+  | xargs -n1 basename \
+  | sed -e 's/\.swift$//' -e 's/^/JetpackUITests\//' \
+  | paste -sd, -)
+
+echo "Running the $AREA UI tests on $DEVICE. The iOS version will be the latest available in the CI host."
+echo "Suites: $SUITES"
+
+echo "--- 📦 Downloading Build Artifacts"
+download_artifact build-products-jetpack.tar
+tar -xf build-products-jetpack.tar
+
+# Only the gems are needed here, given we run the tests on a pre-built binary
+echo "--- :rubygems: Setting up Gems"
+install_gems
+
+echo "--- 🔬 Testing"
+set +e
+bundle exec fastlane test_without_building name:JetpackUITests device:"$DEVICE" only_testing:"$SUITES" reset_simulator:false
+TESTS_EXIT_STATUS=$?
+set -e
+
+if [[ $TESTS_EXIT_STATUS -ne 0 ]]; then
+  # Keep the (otherwise collapsed) current "Testing" section open in Buildkite logs on error. See https://buildkite.com/docs/pipelines/managing-log-output#collapsing-output
+  echo "^^^ +++"
+  echo "UI Tests failed!"
+fi
+
+echo "--- 📦 Zipping test results"
+cd build/results/ && zip -rq Jetpack.xcresult.zip Jetpack.xcresult && cd -
+
+echo "--- 💥 Collecting Crash reports"
+mkdir -p build/results/crashes
+find ~/Library/Logs/DiagnosticReports -name 'Jetpack*.ips' -exec cp "{}" "build/results/crashes/" \;
+
+echo "--- 🚦 Report Tests Status"
+if [[ $TESTS_EXIT_STATUS -eq 0 ]]; then
+  echo "UI Tests seems to have passed (exit code 0). All good 👍"
+else
+  echo "The UI Tests, ran during the '🔬 Testing' step above, have failed."
+  echo "For more details about the failed tests, check the Buildkite annotation, the logs under the '🔬 Testing' section and the \`.xcresult\` and test reports in Buildkite artifacts."
+fi
+
+if [[ $BUILDKITE_BRANCH == trunk ]] || [[ $BUILDKITE_BRANCH == release/* ]]; then
+    annotate_test_failures "build/results/report.junit" --slack "build-and-ship"
+else
+    annotate_test_failures "build/results/report.junit"
+fi
+
+exit $TESTS_EXIT_STATUS

.buildkite/commands/run-unit-tests.sh

@@ -4,6 +4,10 @@ if "$(dirname "${BASH_SOURCE[0]}")/should-skip-job.sh" --job-type validation; th
   exit 0
 fi
 
+DEVICE=${1:?Usage $0 DEVICE}
+
+"$(dirname "${BASH_SOURCE[0]}")/boot-simulator.sh" "$DEVICE"
+
 echo "--- 📦 Downloading Build Artifacts"
 download_artifact build-products-wordpress.tar
 tar -xf build-products-wordpress.tar
@@ -14,7 +18,8 @@ install_gems
 
 echo "--- 🔬 Testing"
 set +e
-bundle exec fastlane test_without_building name:WordPressUnitTests
+# Naming the Simulator is what lets the lane skip resolving the Swift packages and reading the build settings.
+bundle exec fastlane test_without_building name:WordPressUnitTests device:"$DEVICE" reset_simulator:false
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/pipeline.yml

@@ -66,9 +66,10 @@ steps:
           - github_commit_status:
               context: "WordPress Build for Testing"
 
+      # The UI tests run against this build, so it has the HTTP fixtures compiled in.
       - label: "🛠 :jetpack: Build for Testing"
         key: "build_jetpack"
-        command: ".buildkite/commands/build-for-testing.sh jetpack"
+        command: ".buildkite/commands/build-for-testing.sh jetpack http_fixtures:true"
         plugins: [$CI_TOOLKIT_PLUGIN]
         notify:
           - github_commit_status:
@@ -88,7 +89,7 @@ steps:
               context: "Swift Package Tests"
 
       - label: "🔬 :wordpress: Unit Tests"
-        command: ".buildkite/commands/run-unit-tests.sh"
+        command: ".buildkite/commands/run-unit-tests.sh 'iPhone 17 Pro (26.5)'"
         depends_on: "build_wordpress"
         plugins:
           - $CI_TOOLKIT_PLUGIN
@@ -124,6 +125,35 @@ steps:
       #     - github_commit_status:
       #         context: "Unit Tests Keystone"
 
+  #################
+  # Run UI Tests
+  #################
+  - group: "🔬 UI Tests"
+    key: ui_tests_group
+    steps:
+      - label: "🔬 :jetpack: UI Tests ({{matrix}})"
+        command: ".buildkite/commands/run-ui-tests.sh 'iPhone 18 Pro' {{matrix}}"
+        depends_on: "build_jetpack"
+        plugins: [$CI_TOOLKIT_PLUGIN]
+        # One job per area of the app: the folders in Tests/JetpackUITests/Tests.
+        matrix:
+          - "Me"
+          - "MySite"
+          - "Notifications"
+          - "Reader"
+          - "Stats"
+        artifact_paths:
+          - "build/results/*"
+          - "build/results/crashes/*"
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1
+        # One status for all the jobs: GitHub gets the context as written, without the matrix value.
+        notify:
+          - github_commit_status:
+              context: "UI Tests"
+
   #################
   # Linters
   #################

.swiftlint.yml

@@ -202,6 +202,13 @@ custom_rules:
     severity: error
     excluded: '.*Widgets/.*'
 
+  http_fixtures_build_flag:
+    name: "HTTP Fixtures Build Flag"
+    regex: '\A(?!#if UI_TEST_HTTP_FIXTURES\n).'
+    included: 'Modules/Sources/HTTPFixtures/.*\.swift'
+    message: "Wrap the whole file in `#if UI_TEST_HTTP_FIXTURES`. This module has to compile to nothing unless the build sets that condition, so that the app that ships has none of it. See docs/ui-tests.md."
+    severity: error
+
   full_range_attributed_string_attribute:
     name: "Full-Range Attributed String Attribute"
     regex: '\.addAttributes?\([^\n]*range:\s*NS(MakeRange\(0,|Range\(location:\s*0,\s*length:)'

AGENTS.md

@@ -41,7 +41,8 @@ WordPress-iOS uses a modular architecture with the main app and separate Swift p
 - The WordPress scheme uses `WordPressUnitTests.xctestplan` for the full unit test suite, including tests in the `Modules` Swift package.
 - Add every unit test target to `WordPressUnitTests.xctestplan`.
 - Run the full suite with `xcodebuild -workspace WordPress.xcworkspace -scheme WordPress -testPlan WordPressUnitTests test`. Do not use `swift test`.
-- To verify changes end-to-end on an iOS simulator, follow @docs/simulator-sign-in.md to sign in to the app.
+- UI tests live in `Tests/JetpackUITests` and run with the Jetpack scheme's `JetpackUITests` test plan. Most suites sign in to a real WordPress.com account, so they need a token; a suite can instead run against the fixtures in `Tests/JetpackUITests/Fixtures`, which needs an app built with the `UI_TEST_HTTP_FIXTURES` compilation condition. See `docs/ui-tests.md`.
+- To sign a Simulator into WordPress.com, run `make sim-login` (it targets the running simulator; the WordPress.com token comes from `~/.wpcom-token` or `WPCOM_TOKEN`, and it prompts if none is set). See @docs/simulator-sign-in.md for options and self-hosted sign-in.
 
 ### Important Considerations
 - **Multi-site Support**: Code must handle both WordPress.com and self-hosted sites

Makefile

@@ -1,7 +1,10 @@
-.PHONY: help dependencies
+.PHONY: help dependencies sim-login
 
 help: ## Show available targets
 	@grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | awk -F ':.*## ' '{printf "  %-20s %s\n", $$1, $$2}'
 
 dependencies: ## Download and cache Gutenberg XCFrameworks
 	./Scripts/download-gutenberg-xcframeworks.sh
+
+sim-login: ## Sign an iOS Simulator into WordPress.com (vars: DEVICE, APP, RESET=1; token from ~/.wpcom-token)
+	./Scripts/sim-signin.sh $(if $(APP),--app $(APP)) $(if $(DEVICE),--device $(DEVICE)) $(if $(RESET),--reset) $(ARGS)

Modules/Package.swift

@@ -13,6 +13,7 @@ let package = Package(
         .library(name: "DesignSystem", targets: ["DesignSystem"]),
         .library(name: "FormattableContentKit", targets: ["FormattableContentKit"]),
         .library(name: "GutenbergProcessors", targets: ["GutenbergProcessors"]),
+        .library(name: "HTTPFixtures", targets: ["HTTPFixtures"]),
         .library(name: "JetpackStats", targets: ["JetpackStats"]),
         .library(name: "JetpackSocial", targets: ["JetpackSocial"]),
         .library(name: "JetpackStatsWidgetsCore", targets: ["JetpackStatsWidgetsCore"]),
@@ -116,6 +117,10 @@ let package = Package(
             // Set to v5 to avoid @Sendable warnings and errors
             swiftSettings: [.swiftLanguageMode(.v5)]
         ),
+        // Answers the app's requests from fixtures, for the UI tests. Every file is wrapped in
+        // `#if UI_TEST_HTTP_FIXTURES`, so the module is empty unless the build sets that
+        // condition, and the app that ships has none of it. See docs/ui-tests.md.
+        .target(name: "HTTPFixtures"),
         .target(
             name: "JetpackStats",
             dependencies: [
@@ -370,6 +375,7 @@ let package = Package(
             ],
             resources: [.process("Resources")]
         ),
+        .testTarget(name: "HTTPFixturesTests", dependencies: ["HTTPFixtures"]),
         .testTarget(name: "JetpackStatsTests", dependencies: ["JetpackStats"]),
         .testTarget(
             name: "JetpackStatsWidgetsCoreTests",
@@ -525,6 +531,7 @@ enum XcodeSupport {
             "BuildSettingsKit",
             "FormattableContentKit",
             "GutenbergProcessors",
+            "HTTPFixtures",
             "JetpackSocial",
             "JetpackStats",
             "JetpackStatsWidgetsCore",