|
| 1 | +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | + |
| 4 | +import json |
| 5 | +import os |
| 6 | +import shutil |
| 7 | +import subprocess |
| 8 | +import sys |
| 9 | +from pathlib import Path |
| 10 | +from unittest.mock import patch |
| 11 | + |
| 12 | +import pytest |
| 13 | + |
| 14 | +sys.path.insert(0, str(Path(__file__).resolve().parent)) |
| 15 | + |
| 16 | +import check_proto_compatibility as compatibility |
| 17 | + |
| 18 | +TAG = "v0.1.6-pre.7" |
| 19 | +REASON = "Release captain accepted the protobuf changes." |
| 20 | + |
| 21 | + |
| 22 | +@pytest.mark.parametrize( |
| 23 | + ("ref", "exception_tag", "reason", "expected"), |
| 24 | + [ |
| 25 | + (f"refs/tags/{TAG}", TAG, REASON, True), |
| 26 | + ("refs/tags/v0.1.6-pre.8", "v0.1.6-pre.8", REASON, True), |
| 27 | + ("refs/tags/v0.1.6-pre.12", "v0.1.6-pre.12", REASON, True), |
| 28 | + ("refs/tags/v0.1.6-pre.8", "", "", False), |
| 29 | + ("refs/tags/v0.1.7-pre.1", "", "", False), |
| 30 | + ("refs/tags/v0.2.0-pre.1", "", "", False), |
| 31 | + (f"refs/tags/{TAG}", "", "", False), |
| 32 | + ("refs/tags/v0.1.6-pre.8", TAG, REASON, ValueError), |
| 33 | + ("refs/tags/v0.1.6", "v0.1.6", REASON, ValueError), |
| 34 | + ("refs/heads/main", TAG, REASON, ValueError), |
| 35 | + (f"refs/tags/{TAG}", TAG, " ", ValueError), |
| 36 | + ], |
| 37 | +) |
| 38 | +def test_exception_scope(monkeypatch, ref, exception_tag, reason, expected): |
| 39 | + monkeypatch.setenv("PROTO_COMPATIBILITY_EXCEPTION_TAG", exception_tag) |
| 40 | + monkeypatch.setenv("PROTO_COMPATIBILITY_EXCEPTION_REASON", reason) |
| 41 | + monkeypatch.setattr(sys, "argv", ["check", ref]) |
| 42 | + with ( |
| 43 | + patch.object(compatibility, "git", side_effect=[ref, "candidate-sha"]), |
| 44 | + patch.object( |
| 45 | + compatibility, "train_policy", return_value=("v0.1.5", "v0.1.6", False) |
| 46 | + ), |
| 47 | + patch.object(compatibility, "compare", return_value=0) as compare, |
| 48 | + ): |
| 49 | + if expected is ValueError: |
| 50 | + with pytest.raises(ValueError): |
| 51 | + compatibility.main() |
| 52 | + compare.assert_not_called() |
| 53 | + else: |
| 54 | + assert compatibility.main() == 0 |
| 55 | + assert compare.call_args.args[2] is expected |
| 56 | + |
| 57 | + |
| 58 | +@pytest.mark.parametrize( |
| 59 | + ("allows_breaks", "exit_code", "expected"), |
| 60 | + [(False, 100, 100), (True, 100, 0), (True, 1, 1), (True, 0, 0)], |
| 61 | +) |
| 62 | +def test_exception_only_accepts_breaking_diagnostics( |
| 63 | + allows_breaks, exit_code, expected |
| 64 | +): |
| 65 | + with ( |
| 66 | + patch.object(compatibility, "export_proto"), |
| 67 | + patch.object( |
| 68 | + compatibility.subprocess, |
| 69 | + "run", |
| 70 | + side_effect=[ |
| 71 | + subprocess.CompletedProcess([], 0), |
| 72 | + subprocess.CompletedProcess([], 0), |
| 73 | + subprocess.CompletedProcess([], exit_code, stdout="", stderr=""), |
| 74 | + ], |
| 75 | + ) as run, |
| 76 | + ): |
| 77 | + assert compatibility.compare("candidate", "baseline", allows_breaks) == expected |
| 78 | + assert [call.args[0][1] for call in run.call_args_list] == [ |
| 79 | + "build", |
| 80 | + "build", |
| 81 | + "breaking", |
| 82 | + ] |
| 83 | + |
| 84 | + |
| 85 | +@pytest.mark.parametrize("failed_snapshot", [0, 1]) |
| 86 | +def test_exception_does_not_accept_invalid_protobufs(failed_snapshot): |
| 87 | + results = [subprocess.CompletedProcess([], 0)] * failed_snapshot |
| 88 | + results.append(subprocess.CalledProcessError(100, ["buf", "build"])) |
| 89 | + with ( |
| 90 | + patch.object(compatibility, "export_proto"), |
| 91 | + patch.object(compatibility.subprocess, "run", side_effect=results) as run, |
| 92 | + pytest.raises(subprocess.CalledProcessError), |
| 93 | + ): |
| 94 | + compatibility.compare("candidate", "baseline", True) |
| 95 | + assert run.call_count == failed_snapshot + 1 |
| 96 | + |
| 97 | + |
| 98 | +@pytest.mark.skipif(shutil.which("buf") is None, reason="Buf is not installed") |
| 99 | +@pytest.mark.parametrize("allows_breaks", [False, True]) |
| 100 | +def test_real_buf_field_rename(monkeypatch, allows_breaks): |
| 101 | + def export_fixture(ref, destination): |
| 102 | + (destination / "proto").mkdir(parents=True) |
| 103 | + (destination / "buf.yaml").write_text( |
| 104 | + "version: v2\nmodules:\n - path: proto\nbreaking:\n use:\n - FILE\n" |
| 105 | + ) |
| 106 | + field = "old_name" if ref == "baseline" else "new_name" |
| 107 | + (destination / "proto" / "test.proto").write_text( |
| 108 | + 'syntax = "proto3";\npackage test.v1;\n' |
| 109 | + f"message Request {{ bool {field} = 3; }}\n" |
| 110 | + ) |
| 111 | + |
| 112 | + monkeypatch.setattr(compatibility, "export_proto", export_fixture) |
| 113 | + assert compatibility.compare("candidate", "baseline", allows_breaks) == ( |
| 114 | + 0 if allows_breaks else 100 |
| 115 | + ) |
| 116 | + |
| 117 | + |
| 118 | +@pytest.mark.parametrize("exception", [False, True]) |
| 119 | +@pytest.mark.parametrize("proto_result", ["success", "failure", "skipped"]) |
| 120 | +@pytest.mark.parametrize("tag", [TAG, "v0.1.6-pre.8"]) |
| 121 | +def test_qualification_records_exception_without_bypassing_results( |
| 122 | + tmp_path, exception, proto_result, tag |
| 123 | +): |
| 124 | + output = tmp_path / "qualification.json" |
| 125 | + env = { |
| 126 | + **os.environ, |
| 127 | + "RELEASE_TAG": tag, |
| 128 | + "SOURCE_SHA": "candidate-sha", |
| 129 | + "IS_PRERELEASE": "true", |
| 130 | + "GITHUB_RUN_ID": "123", |
| 131 | + "GITHUB_RUN_ATTEMPT": "1", |
| 132 | + "GITHUB_SERVER_URL": "https://github.com", |
| 133 | + "GITHUB_REPOSITORY": "NVIDIA/OpenShell", |
| 134 | + "PROTO_COMPATIBILITY_RESULT": proto_result, |
| 135 | + "PROTO_COMPATIBILITY_EXCEPTION_TAG": tag if exception else "", |
| 136 | + "PROTO_COMPATIBILITY_EXCEPTION_REASON": REASON, |
| 137 | + } |
| 138 | + for suite in ( |
| 139 | + "SECURITY", |
| 140 | + "CONFORMANCE", |
| 141 | + "FEATURE_INTEGRATION", |
| 142 | + "DOCKER_E2E", |
| 143 | + "VM_E2E", |
| 144 | + ): |
| 145 | + env[f"{suite}_RESULT"] = "success" |
| 146 | + subprocess.run( |
| 147 | + [ |
| 148 | + "bash", |
| 149 | + str(Path(__file__).with_name("generate-qualification-summary.sh")), |
| 150 | + str(output), |
| 151 | + ], |
| 152 | + env=env, |
| 153 | + check=True, |
| 154 | + capture_output=True, |
| 155 | + text=True, |
| 156 | + ) |
| 157 | + summary = json.loads(output.read_text()) |
| 158 | + assert summary["current_profile_passed"] is (proto_result == "success") |
| 159 | + assert summary["exceptions"] == ( |
| 160 | + [{"suite": "protobuf_compatibility", "tag": tag, "reason": REASON}] |
| 161 | + if exception |
| 162 | + else [] |
| 163 | + ) |
0 commit comments