diff --git a/release/README.md b/release/README.md
index a6bdd6f88..2f19e56a1 100644
--- a/release/README.md
+++ b/release/README.md
@@ -34,7 +34,6 @@ unset GOOGLE_CLOUD_PROJECT
* `export ZONE_NAME=(google zone name)`
* `export AWS_PROFILE=...` (if needed)
-
#### Deploy clusters
* `./release/scripts/spawn-clusters.sh`
@@ -81,3 +80,29 @@ Cleanup resources by:
* Applying terraform destroy to release clusters
No manual cleaning operations are needed after this step.
+
+#### Compatibility matrix
+* `./release/scripts/run-compat-matrix.sh`
+
+Test which upgrade paths are possible between released DSS versions, so that the compatibility matrix of the documentation can be updated.
+
+The list of versions to test is the `VERSIONS` array at the top of the script, ordered from oldest to newest. The version being released must be added to it.
+
+Any local DSS instance must be stopped first with `make stop-locally`, otherwise it interferes with the stack started by the script.
+
+For each pair of versions A (older) to B (newer), the script starts a local stack with a single datastore shared by two `core-service` instances, one running version A and the other version B. The datastore is migrated to the latest schema of version B. Pairs where B is older than A are not tested and reported as not evaluated.
+
+Each pair is then validated by running the prober against both instances and the USS qualifier against the pool. A pair passes only if the three runs pass.
+
+The prober and the qualifier run from `MONITORING_IMAGE`, except for pairs involving v0.20.2 which use `V0_20_2_MONITORING_IMAGE`: those results are flagged with a footnote in the generated table.
+
+A local `dummy-oauth` image is built if it is not already present.
+
+The docker compose stack and the qualifier configuration are in `release/compat`.
+
+After this step, the script prints:
+
+* A summary matrix in the terminal
+* A Markdown compatibility matrix to copy into the documentation / GitHub release notes
+
+Container logs are available in `release/logs` and qualifier reports in `release/uss_qualifier_output/compat`.
diff --git a/release/compat/docker-compose.yaml b/release/compat/docker-compose.yaml
new file mode 100644
index 000000000..bbb2779d6
--- /dev/null
+++ b/release/compat/docker-compose.yaml
@@ -0,0 +1,101 @@
+name: dss-compat
+
+networks:
+ default:
+ name: dss-compat
+
+services:
+
+ crdb:
+ image: cockroachdb/cockroach:v24.1.3
+ command: start-single-node --insecure
+ healthcheck:
+ test: curl -f 'http://localhost:8080/health?ready=1' || exit 1
+ interval: 1s
+ timeout: 1s
+ retries: 10
+
+ rid-bootstrapper:
+ image: ${DSS_IMAGE_B}
+ command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/rid --db_version latest ${DATASTORE_FLAG_B} crdb
+ depends_on:
+ crdb:
+ condition: service_healthy
+
+ scd-bootstrapper:
+ image: ${DSS_IMAGE_B}
+ command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/scd --db_version latest ${DATASTORE_FLAG_B} crdb
+ depends_on:
+ crdb:
+ condition: service_healthy
+
+ aux-bootstrapper:
+ image: ${DSS_IMAGE_B}
+ command: /usr/bin/db-manager migrate --schemas_dir=/db-schemas/aux_ --db_version latest ${DATASTORE_FLAG_B} crdb
+ profiles: ["with-aux"]
+ depends_on:
+ crdb:
+ condition: service_healthy
+
+ core-service-a:
+ image: ${DSS_IMAGE_A}
+ volumes:
+ - ../../build/test-certs:/var/test-certs:ro
+ command: >
+ /usr/bin/core-service ${DATASTORE_FLAG_A} crdb ${PUBLIC_ENDPOINT_A}
+ -public_key_files /var/test-certs/auth2.pem
+ -log_format console
+ -addr :8082
+ -accepted_jwt_audiences localhost,core-service-a,core-service-b
+ -enable_scd
+ -allow_http_base_urls
+ -locality dss-compat-a
+ depends_on:
+ rid-bootstrapper:
+ condition: service_completed_successfully
+ scd-bootstrapper:
+ condition: service_completed_successfully
+ aux-bootstrapper:
+ condition: service_completed_successfully
+ required: false
+ healthcheck:
+ test: wget -O - 'http://localhost:8082/healthy' || exit 1
+ interval: 1s
+ timeout: 1s
+ retries: 10
+
+ core-service-b:
+ image: ${DSS_IMAGE_B}
+ volumes:
+ - ../../build/test-certs:/var/test-certs:ro
+ command: >
+ /usr/bin/core-service ${DATASTORE_FLAG_B} crdb ${PUBLIC_ENDPOINT_B}
+ -public_key_files /var/test-certs/auth2.pem
+ -log_format console
+ -addr :8082
+ -accepted_jwt_audiences localhost,core-service-a,core-service-b
+ -enable_scd
+ -allow_http_base_urls
+ -locality dss-compat-b
+ depends_on:
+ rid-bootstrapper:
+ condition: service_completed_successfully
+ scd-bootstrapper:
+ condition: service_completed_successfully
+ aux-bootstrapper:
+ condition: service_completed_successfully
+ required: false
+ healthcheck:
+ test: wget -O - 'http://localhost:8082/healthy' || exit 1
+ interval: 1s
+ timeout: 1s
+ retries: 10
+
+ oauth:
+ image: interuss-local/dummy-oauth
+ command: -private_key_file /var/test-certs/auth2.key
+ healthcheck:
+ test: wget -O - 'http://localhost:8085/token?intended_audience=-&scope=-' || exit 1
+ interval: 5s
+ timeout: 5s
+ retries: 20
diff --git a/release/compat/qualifier_config.yaml b/release/compat/qualifier_config.yaml
new file mode 100644
index 000000000..c4286d5d5
--- /dev/null
+++ b/release/compat/qualifier_config.yaml
@@ -0,0 +1,254 @@
+v1:
+ artifacts:
+ output_path: output/compat
+ raw_report:
+ indent: null
+ redact_access_tokens: true
+ tested_requirements:
+ - participant_requirements:
+ uss1: all_astm_dss_requirements
+ uss2: all_astm_dss_requirements
+ report_name: requirements
+ requirement_collections:
+ all_astm_dss_requirements:
+ requirement_collections:
+ - requirement_sets:
+ - astm.f3411.v22a.dss_provider
+ - astm.f3411.v19.dss_provider
+ - astm.f3548.v21.dss_provider
+ test_run:
+ action:
+ on_failure: Continue
+ test_suite:
+ resources:
+ f3411v19_dss_instances: netrid_dss_instances_v19
+ f3411v22a_dss_instances: netrid_dss_instances_v22a
+ f3548v21_dss_instances: scd_dss_instances
+ flight_intents: che_non_conflicting_flights
+ id_generator: id_generator
+ planning_area: kentland_planning_area
+ planning_area_volume: kentland_planning_area_volume
+ problematically_big_area: kentland_problematically_big_area
+ second_utm_auth: second_utm_auth
+ service_area: kentland_service_area
+ test_exclusions: test_exclusions
+ utm_client_identity: utm_client_identity
+ suite_type: suites.interuss.dss.all_tests
+ execution:
+ include_action_when: null
+ skip_action_when: null
+ stop_fast: false
+ stop_when_resource_not_created: false
+ non_baseline_inputs:
+ - v1.test_run.resources.resource_declarations.utm_auth
+ - v1.test_run.resources.resource_declarations.second_utm_auth
+ - v1.test_run.resources.resource_declarations.scd_dss_instances
+ - v1.test_run.resources.resource_declarations.netrid_dss_instances_v22a
+ - v1.test_run.resources.resource_declarations.netrid_dss_instances_v19
+ resources:
+ resource_declarations:
+ che_non_conflicting_flights:
+ dependencies: {}
+ resource_type: resources.flight_planning.FlightIntentsResource
+ specification:
+ file:
+ path: test_data.flight_intents.standard.non_conflicting
+ transformations:
+ - relative_translation:
+ degrees_east: 7.4774
+ degrees_north: 46.9748
+ meters_up: 605
+ id_generator:
+ dependencies:
+ client_identity: utm_client_identity
+ resource_type: resources.interuss.IDGeneratorResource
+ specification: {}
+ kentland_planning_area_volume:
+ resource_type: resources.VolumeResource
+ specification:
+ template:
+ outline_polygon:
+ vertices:
+ - lat: 37.1853
+ lng: -80.6140
+ - lat: 37.2148
+ lng: -80.6140
+ - lat: 37.2148
+ lng: -80.5440
+ - lat: 37.1853
+ lng: -80.5440
+ altitude_lower:
+ value: 0
+ reference: W84
+ units: M
+ altitude_upper:
+ value: 3048
+ reference: W84
+ units: M
+ kentland_planning_area:
+ resource_type: resources.PlanningAreaResource
+ dependencies:
+ volume: kentland_planning_area_volume
+ specification:
+ base_url: https://uss_qualifier.test.utm/interuss/dss/build/dev/dss_probing_qualifier_config/kentland_planning_area
+ kentland_problematically_big_area:
+ resource_type: resources.VolumeResource
+ specification:
+ template:
+ outline_polygon:
+ vertices:
+ - lat: 38
+ lng: -81
+ - lat: 37
+ lng: -81
+ - lat: 37
+ lng: -80
+ - lat: 38
+ lng: -80
+ kentland_service_area_volume:
+ resource_type: resources.VolumeResource
+ specification:
+ template:
+ outline_polygon:
+ vertices:
+ - lat: 37.1853
+ lng: -80.6140
+ - lat: 37.2148
+ lng: -80.6140
+ - lat: 37.2148
+ lng: -80.5440
+ - lat: 37.1853
+ lng: -80.5440
+ altitude_lower:
+ value: 0
+ reference: W84
+ units: M
+ altitude_upper:
+ value: 3048
+ reference: W84
+ units: M
+ start_time:
+ offset_from:
+ starting_from:
+ time_during_test: TimeOfEvaluation
+ offset: 1s
+ end_time:
+ offset_from:
+ starting_from:
+ time_during_test: TimeOfEvaluation
+ offset: 1h0m1s
+ kentland_service_area:
+ resource_type: resources.netrid.ServiceAreaResource
+ dependencies:
+ volume: kentland_service_area_volume
+ specification:
+ base_url: https://uss_qualifier.test.utm/interuss/dss/build/dev/dss_probing_qualifier_config/kentland_service_area
+ netrid_dss_instances_v19:
+ dependencies:
+ auth_adapter: utm_auth
+ resource_type: resources.astm.f3411.DSSInstancesResource
+ specification:
+ dss_instances:
+ - base_url: http://core-service-a:8082
+ has_private_address: true
+ participant_id: uss1
+ rid_version: F3411-19
+ - base_url: http://core-service-b:8082
+ has_private_address: true
+ participant_id: uss2
+ rid_version: F3411-19
+ netrid_dss_instances_v22a:
+ dependencies:
+ auth_adapter: utm_auth
+ resource_type: resources.astm.f3411.DSSInstancesResource
+ specification:
+ dss_instances:
+ - base_url: http://core-service-a:8082/rid/v2
+ has_private_address: true
+ participant_id: uss1
+ rid_version: F3411-22a
+ - base_url: http://core-service-b:8082/rid/v2
+ has_private_address: true
+ participant_id: uss2
+ rid_version: F3411-22a
+ scd_dss_instances:
+ dependencies:
+ auth_adapter: utm_auth
+ resource_type: resources.astm.f3548.v21.DSSInstancesResource
+ specification:
+ dss_instances:
+ - base_url: http://core-service-a:8082
+ has_private_address: true
+ participant_id: uss1
+ supports_ovn_request: true
+ user_participant_ids:
+ - mock_uss
+ - base_url: http://core-service-b:8082
+ has_private_address: true
+ participant_id: uss2
+ supports_ovn_request: true
+ user_participant_ids:
+ - mock_uss
+ second_utm_auth:
+ dependencies: {}
+ resource_type: resources.communications.AuthAdapterResource
+ specification:
+ environment_variable_containing_auth_spec: AUTH_SPEC_2
+ scopes_authorized:
+ - utm.strategic_coordination
+ test_exclusions:
+ dependencies: {}
+ resource_type: resources.dev.TestExclusionsResource
+ specification:
+ allow_cleartext_queries: true
+ allow_private_addresses: true
+ utm_auth:
+ dependencies: {}
+ resource_type: resources.communications.AuthAdapterResource
+ specification:
+ environment_variable_containing_auth_spec: AUTH_SPEC
+ scopes_authorized:
+ - rid.inject_test_data
+ - dss.read.identification_service_areas
+ - rid.service_provider
+ - rid.display_provider
+ - dss.write.identification_service_areas
+ - dss.read.identification_service_areas
+ - interuss.flight_planning.direct_automated_test
+ - interuss.flight_planning.plan
+ - utm.inject_test_data
+ - utm.strategic_coordination
+ - utm.conformance_monitoring_sa
+ - utm.availability_arbitration
+ - utm.constraint_management
+ - interuss.versioning.read_system_versions
+ - interuss.geospatial_map.query
+ - ''
+ utm_client_identity:
+ dependencies:
+ auth_adapter: utm_auth
+ resource_type: resources.communications.ClientIdentityResource
+ specification:
+ whoami_audience: localhost
+ whoami_scope: rid.display_provider
+ validation:
+ criteria:
+ - applicability:
+ test_scenarios: {}
+ pass_condition:
+ each_element:
+ has_execution_error: false
+ - applicability:
+ failed_checks:
+ has_severity:
+ higher_than: Low
+ pass_condition:
+ elements:
+ count:
+ equal_to: 0.0
+ - applicability:
+ skipped_actions: {}
+ pass_condition:
+ elements:
+ count:
+ equal_to: 6.0
diff --git a/release/scripts/run-compat-matrix.sh b/release/scripts/run-compat-matrix.sh
new file mode 100755
index 000000000..6c1b6ad66
--- /dev/null
+++ b/release/scripts/run-compat-matrix.sh
@@ -0,0 +1,277 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+VERSIONS=(v0.20.2 v0.21.1 v0.22.0 v0.23.0)
+
+MONITORING_IMAGE="interuss/monitoring:v0.34.0"
+
+# This should be removed when we stop testing v0.20.2.
+# Improvements landed in v0.21.x are exercised by monitoring images newer than
+# v0.24.0, so an older image is needed to check compatibility with v0.20.2.
+V0_20_2_MONITORING_IMAGE="interuss/monitoring:v0.24.0"
+
+###############################################################################
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=release/scripts/common.sh
+source "$SCRIPT_DIR/common.sh"
+
+
+declare -A VIDX
+for i in "${!VERSIONS[@]}"; do VIDX["${VERSIONS[$i]}"]="$i"; done
+
+DSS_REGISTRY="docker.io/interuss/dss"
+OAUTH_IMAGE="interuss-local/dummy-oauth"
+CRDB_IMAGE="cockroachdb/cockroach:v24.1.3"
+
+NET="dss-compat"
+COMPOSE=(docker compose -f "$SCRIPT_DIR/../compat/docker-compose.yaml")
+
+QUAL_OUT="$RELEASE_DIR/uss_qualifier_output/compat"
+mkdir -p "$LOG_DIR" "$QUAL_OUT"
+
+declare -A VERDICT DETAIL
+
+datastore_flag() {
+ case "$1" in
+ v0.20.2) echo "--cockroach_host" ;;
+ *) echo "--datastore_host" ;;
+ esac
+}
+
+public_endpoint_flag() {
+ case "$1" in
+ v0.20.2) echo "" ;;
+ *) echo "-public_endpoint http://$2:8082" ;;
+ esac
+}
+
+has_aux() {
+ [[ "$1" != "v0.20.2" ]]
+}
+
+wait_healthy() {
+ local svc="$1" cid state
+ cid=$("${COMPOSE[@]}" ps -q "$svc" 2>/dev/null) || return 1
+ [[ -n "$cid" ]] || return 1
+ for _ in $(seq 1 90); do
+ state=$(docker inspect -f '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$cid" 2>/dev/null || echo gone:)
+ case "$state" in
+ running:healthy) return 0 ;;
+ running:unhealthy) return 1 ;;
+ exited:*|dead:*|gone:*) return 1 ;;
+ esac
+ sleep 2
+ done
+ return 1
+}
+
+teardown() {
+ "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+}
+on_interrupt() {
+ trap - EXIT INT TERM
+ docker rm -f dss-compat-prober dss-compat-qualifier >/dev/null 2>&1 || true
+ teardown
+ exit 130
+}
+trap teardown EXIT
+trap on_interrupt INT TERM
+
+run_prober() {
+ local endpoint="$1" log="$2" image="$3"
+ docker run --rm \
+ --name dss-compat-prober \
+ --network "$NET" \
+ -w /app/monitoring/prober \
+ "$image" \
+ pytest . -rsx \
+ --dss-endpoint "http://$endpoint:8082" \
+ --rid-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \
+ --rid-v2-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \
+ --scd-auth1 "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \
+ --scd-auth2 "DummyOAuth(http://oauth:8085/token,sub=fake_uss2)" \
+ --scd-api-version 1.0.0 \
+ >"$log" 2>&1
+}
+
+run_qualifier() {
+ local out="$1" log="$2" image="$3"
+ rm -rf "$out"
+ mkdir -p "$out"
+ docker run --rm \
+ --name dss-compat-qualifier \
+ --network "$NET" \
+ -u "$(id -u):$(id -g)" \
+ -e AUTH_SPEC='DummyOAuth(http://oauth:8085/token,uss_qualifier)' \
+ -e AUTH_SPEC_2='DummyOAuth(http://oauth:8085/token,uss_qualifier_2)' \
+ -v "$SCRIPT_DIR/../compat/qualifier_config.yaml:/app/monitoring/uss_qualifier/compat_config.yaml:ro" \
+ -v "$out:/app/monitoring/uss_qualifier/output" \
+ -w /app/monitoring/uss_qualifier \
+ "$image" \
+ python main.py --config compat_config \
+ >"$log" 2>&1
+}
+
+
+run_combo() {
+ local a="$1" b="$2" key="$1|$2"
+ local slug="${a}_${b}"
+
+ export DSS_IMAGE_A="$DSS_REGISTRY:$a"
+ export DSS_IMAGE_B="$DSS_REGISTRY:$b"
+ export DATASTORE_FLAG_A DATASTORE_FLAG_B PUBLIC_ENDPOINT_A PUBLIC_ENDPOINT_B COMPOSE_PROFILES
+ DATASTORE_FLAG_A="$(datastore_flag "$a")"
+ DATASTORE_FLAG_B="$(datastore_flag "$b")"
+ PUBLIC_ENDPOINT_A="$(public_endpoint_flag "$a" core-service-a)"
+ PUBLIC_ENDPOINT_B="$(public_endpoint_flag "$b" core-service-b)"
+ COMPOSE_PROFILES=""
+ if has_aux "$b"; then
+ COMPOSE_PROFILES="with-aux"
+ fi
+
+ "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
+ if ! "${COMPOSE[@]}" up -d >"$LOG_DIR/$slug.compose.log" 2>&1 \
+ || ! wait_healthy oauth || ! wait_healthy core-service-a || ! wait_healthy core-service-b; then
+ "${COMPOSE[@]}" logs >>"$LOG_DIR/$slug.compose.log" 2>&1 || true
+ VERDICT[$key]="INFRA"
+ DETAIL[$key]="stack did not come up"
+ return
+ fi
+
+ local pa=0 pb=0 q=0 image="$MONITORING_IMAGE"
+ if [[ "$a" == "v0.20.2" || "$b" == "v0.20.2" ]]; then
+ image="$V0_20_2_MONITORING_IMAGE"
+ fi
+ run_prober core-service-a "$LOG_DIR/$slug.prober-a.log" "$image" || pa=1
+ run_prober core-service-b "$LOG_DIR/$slug.prober-b.log" "$image" || pb=1
+
+ run_qualifier "$QUAL_OUT/$slug" "$LOG_DIR/$slug.qualifier.log" "$image" || q=1
+
+ DETAIL[$key]="prober-a=$( ((pa)) && echo fail || echo ok ) prober-b=$( ((pb)) && echo fail || echo ok ) qualifier=$( ((q)) && echo fail || echo ok )"
+
+ if (( pa )) || (( pb )) || (( q )); then
+ VERDICT[$key]="FAIL"
+ else
+ VERDICT[$key]="PASS"
+ fi
+}
+
+symbol() {
+ case "$1" in
+ PASS) printf '%s✓%s' "$GREEN" "$RESET" ;;
+ FAIL) printf '%s✗%s' "$RED" "$RESET" ;;
+ INFRA) printf '%s⨯%s' "$RED" "$RESET" ;;
+ *) printf ' ' ;;
+ esac
+}
+
+emoji() {
+ case "$1" in
+ PASS) echo '✅' ;;
+ FAIL) echo '❌' ;;
+ INFRA) echo '❌' ;;
+ *) echo '⚪' ;;
+ esac
+}
+
+release_link() {
+ printf '[%s](https://github.com/interuss/dss/releases/tag/interuss%%2Fdss%%2F%s)' "$1" "$1"
+}
+
+md_table() {
+ local a b cell footnote=0
+ printf 'The following matrix shows what is possible when a user wants to upgrade a pool on version A (rows) to\n'
+ printf 'version B (columns).\n'
+ printf 'The table always assumes a migration to the latest schema of the target version B prior to DSS version upgrade per "Rolling upgrade procedure" below. Where this cannot be accomplished (e.g., DSS version X cannot function with the latest schema of DSS version X+1), the transition will be indicated as incompatible.\n'
+ printf '| A \\ B '
+ for b in "${VERSIONS[@]}"; do printf '| %s ' "$(release_link "$b")"; done
+ printf '|\n|---'
+ for _ in "${VERSIONS[@]}"; do printf '|---'; done
+ printf '|\n'
+ for a in "${VERSIONS[@]}"; do
+ printf '| **%s** ' "$(release_link "$a")"
+ for b in "${VERSIONS[@]}"; do
+ if (( ${VIDX[$b]} < ${VIDX[$a]} )); then
+ cell='⚪'
+ elif [[ "$a" == "$b" ]]; then
+ cell='✅'
+ else
+ cell="$(emoji "${VERDICT["$a|$b"]:-}")"
+ if [[ "$a" == "v0.20.2" || "$b" == "v0.20.2" ]]; then
+ cell="$cell1"
+ footnote=1
+ fi
+ fi
+ printf '| %s ' "$cell"
+ done
+ printf '|\n'
+ done
+ printf '✅ compatible · ⚠️ degraded, see explanation below · ❌ incompatible · ⚪ not evaluated\n'
+ if (( footnote )); then
+ printf '1Some tests in a multi-version pool may fail due to improvements in the test suite and DSS behavior.\n'
+ fi
+}
+
+section "Build dummy-oauth image (if missing)"
+if docker image inspect "$OAUTH_IMAGE" >/dev/null 2>&1; then
+ ok "$OAUTH_IMAGE already present"
+else
+ ( cd "$REPO_ROOT" && docker build -f cmds/dummy-oauth/Dockerfile -t "$OAUTH_IMAGE" . ) \
+ >"$LOG_DIR/dummy-oauth-build.log" 2>&1
+ ok "built $OAUTH_IMAGE"
+fi
+
+section "Pull images"
+for v in "${VERSIONS[@]}"; do
+ docker pull -q "$DSS_REGISTRY:$v" >/dev/null
+ info "$DSS_REGISTRY:$v"
+done
+docker pull -q "$MONITORING_IMAGE" >/dev/null
+docker pull -q "$V0_20_2_MONITORING_IMAGE" >/dev/null
+docker pull -q "$CRDB_IMAGE" >/dev/null
+
+TOTAL=$(( ${#VERSIONS[@]} * (${#VERSIONS[@]} - 1) / 2 ))
+N=0
+START=$(date +%s)
+for a in "${VERSIONS[@]}"; do
+ for b in "${VERSIONS[@]}"; do
+ (( ${VIDX[$b]} <= ${VIDX[$a]} )) && continue
+ N=$((N + 1))
+ section "[$N/$TOTAL] migrations $a · A=$a B=$b"
+ t0=$(date +%s)
+ run_combo "$a" "$b"
+ printf ' %s %s %s(%s)%s\n' "$(symbol "${VERDICT["$a|$b"]}")" "${VERDICT["$a|$b"]}" \
+ "$DIM" "${DETAIL["$a|$b"]}" "$RESET"
+ info "took $(fmt_dur "$(( $(date +%s) - t0 ))")"
+ done
+done
+teardown
+
+section "Matrix (rows = A; columns = B, migrations applied by B)"
+printf ' %-14s' ''
+for b in "${VERSIONS[@]}"; do printf '%-13s' "$b"; done
+printf '\n'
+for a in "${VERSIONS[@]}"; do
+ printf ' %-14s' "$a"
+ for b in "${VERSIONS[@]}"; do
+ if (( ${VIDX[$b]} < ${VIDX[$a]} )); then
+ printf '%s·%s' "$DIM" "$RESET"
+ elif [[ "$a" == "$b" ]]; then
+ printf '%s✓%s' "$GREEN" "$RESET"
+ else
+ printf '%s' "$(symbol "${VERDICT["$a|$b"]:-}")"
+ fi
+ printf '%-12s' ''
+ done
+ printf '\n'
+done
+printf '\n %s✓%s pass %s✗%s fail %s⨯%s stack did not start\n' \
+ "$GREEN" "$RESET" "$RED" "$RESET" "$RED" "$RESET"
+printf ' %stotal: %s logs: %s reports: %s%s\n' \
+ "$DIM" "$(fmt_dur "$(( $(date +%s) - START ))")" "$LOG_DIR" "$QUAL_OUT" "$RESET"
+
+section "Markdown"
+echo
+md_table