Repository navigation
Expand file tree
/
Copy pathtest_zero_install_detector.py
More file actions
2318 lines (2035 loc) · 98.6 KB
/
Copy pathtest_zero_install_detector.py
File metadata and controls
2318 lines (2035 loc) · 98.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Golden-parity tests for the zero-install ``tools/shipgate-detect.py``.
Pins the script's structural verdict to ``agents-shipgate detect --json``
(via :func:`agents_shipgate.cli.discovery.detect_workspace`) on every
sample fixture in ``samples/``. The contract is **structural parity**,
not byte parity: same ``is_agent_project``, same set of fired
frameworks, same ``suggested_sources`` and ``excluded_sources``.
Evidence/reason strings and absolute framework scores are intentionally
simplified — a coding agent uses the script to make a yes/no decision, not
to re-derive the report.
``agent_name_candidates`` is the one exception, pinned byte for byte. It is
not a yes/no signal: it names the agent a generated manifest declares as
the reviewed identity, so a script that ranked differently would send an
agent to fix a different agent than ``init`` did.
If a new sample is added or the canonical detection rules change, this
test catches drift between the script and the CLI immediately.
"""
from __future__ import annotations
import importlib.util
import subprocess
import sys
from pathlib import Path
from typing import Any
import pytest
from agents_shipgate.cli.discovery import detect_workspace
from agents_shipgate.cli.discovery import mcp_source as mcp_source_discovery
from agents_shipgate.inputs import mcp_server_source
from agents_shipgate.inputs.codex_plugin import resolve_local_codex_marketplace_roots
from tests.mcp_idiom_corpus import (
ESCAPE_CASES,
PYTHON_TREES,
SCANNABLE_PATHS,
SOURCE_CASES,
)
REPO_ROOT = Path(__file__).resolve().parent.parent
SCRIPT_PATH = REPO_ROOT / "tools" / "shipgate-detect.py"
SAMPLES_ROOT = REPO_ROOT / "samples"
# Hidden directories under samples/ are reference material (anti-patterns,
# READMEs), not detector inputs. The published fixtures are the regular
# top-level dirs.
_HIDDEN_PREFIXES = ("_", ".")
CANONICAL_KEYS = frozenset(
{
"is_agent_project",
"frameworks",
"agent_name_candidates",
"project_name_candidates",
"agent_scope",
"agent_project_candidates",
"agent_scope_truncated",
"python_parse_truncated",
"suggested_sources",
"excluded_sources",
"host_boundary_candidates",
"host_discovery_incomplete_paths",
"next_action",
"workspace_signals",
}
)
def _sample_dirs() -> list[Path]:
return sorted(
p
for p in SAMPLES_ROOT.iterdir()
if p.is_dir() and not p.name.startswith(_HIDDEN_PREFIXES)
)
def _sample_ids() -> list[str]:
return [p.name for p in _sample_dirs()]
def _load_script_module() -> Any:
spec = importlib.util.spec_from_file_location(
"shipgate_detect_zero_install", SCRIPT_PATH
)
if spec is None or spec.loader is None:
raise RuntimeError(f"Could not load {SCRIPT_PATH}")
module = importlib.util.module_from_spec(spec)
sys.modules["shipgate_detect_zero_install"] = module
spec.loader.exec_module(module)
return module
@pytest.fixture(scope="module")
def script_module():
return _load_script_module()
def test_script_path_exists():
"""The zero-install script must live at the published path. The
raw GitHub URL in docs/zero-install.md and llms.txt is what coding
agents fetch — moving the file silently would break the public
contract."""
assert SCRIPT_PATH.is_file(), (
f"tools/shipgate-detect.py not found at {SCRIPT_PATH}. The "
"raw GitHub URL is part of the agent-facing public surface."
)
def test_script_does_not_claim_drop_in_parity(script_module):
"""The script is documented as a structural subset of
``agents-shipgate detect --json``, NOT a drop-in replacement.
Specifically, the canonical CLI emits ``diagnostics[]`` and
``next_actions[]`` arrays; the zero-install script does not.
Pin the absence so docs/zero-install.md, llms.txt, and the script's
docstring stay accurate. If we ever decide to ship a stdlib-only
diagnostic engine, update those wording surfaces in the same PR
that flips this test."""
result = script_module.detect(SAMPLES_ROOT / "support_refund_agent")
assert "diagnostics" not in result, (
"The zero-install script must not emit `diagnostics[]` — it's "
"documented as a structural subset of the canonical CLI. If "
"you add this field, update the docstring in "
"tools/shipgate-detect.py, docs/zero-install.md, and llms.txt "
"to match."
)
assert "next_actions" not in result, (
"The zero-install script must not emit `next_actions[]` — see "
"the docstring in tools/shipgate-detect.py for the rationale "
"(diagnostic engine is out of scope for the zero-install path)."
)
def test_framework_vocabulary_names_every_cli_omission(script_module):
"""Every framework the CLI can report is either here or a named omission.
The parity sweep compares the two on ``samples/``, which is only as strong
as the fixtures: a detection the CLI gains and the script does not is
invisible to it until a sample exercises the difference. That is exactly
what happened with ``mcp_server_source`` (#431) — the CLI learned to read
an MCP server's tool names out of TypeScript, Go or Python registration sites, no
sample contained one, and the script went on reporting
``mongodb-js/mongodb-mcp-server`` as *not an agent project* for a whole
release. #485 ported it, and ``known_omissions`` is empty again.
Keep it that way, or write the next omission down here rather than leaving
it to be discovered on someone's repository. A listed omission also has to
be legible in the script itself: a reader of the file must be able to find
it without reading this test.
"""
from agents_shipgate.cli.discovery.signals import _initial_framework_scores
known_omissions: set[str] = set()
cli = set(_initial_framework_scores())
script = set(script_module.FRAMEWORKS)
assert script <= cli, (
f"the script reports frameworks the CLI cannot: {sorted(script - cli)}"
)
assert cli - script == known_omissions, (
"the script's framework vocabulary drifted from the CLI's: "
f"{sorted((cli - script) ^ known_omissions)}. Port the detection, or "
"add it to `known_omissions` here and to the script's `Intentional "
"simplifications` list."
)
for omitted in known_omissions:
assert omitted in SCRIPT_PATH.read_text(encoding="utf-8"), (
f"{omitted!r} is a known omission but the script never says so; "
"a reader of the script cannot discover it."
)
def test_conventional_directory_frameworks_match_the_cli(script_module):
"""A conventional directory is weak evidence for the same list on both sides.
Not every framework: ``mcp_server_source``'s evidence is already a
conjunction, so a ``tools/`` directory adds nothing it does not have, and
adding it would carry the published confidence to ``high`` for a route the
engine caps at ``medium``. That exclusion never changes which frameworks
fire, so no parity assertion over a workspace can see it — which is why it
is pinned by value here.
"""
from agents_shipgate.cli.discovery.signals import CONVENTIONAL_DIR_FRAMEWORKS
assert (
tuple(script_module.CONVENTIONAL_DIR_FRAMEWORKS)
== CONVENTIONAL_DIR_FRAMEWORKS
)
def test_script_emits_canonical_top_level_keys(script_module):
"""The script's JSON output must carry the same top-level keys as
DetectResult, plus ``script_version`` to distinguish it from the
canonical CLI."""
result = script_module.detect(SAMPLES_ROOT / "support_refund_agent")
missing = CANONICAL_KEYS - set(result)
assert not missing, (
f"Zero-install detector output missing canonical keys: {sorted(missing)}. "
"Output must be a structural superset of DetectResult."
)
assert "script_version" in result, (
"Zero-install detector must emit script_version so consumers "
"can distinguish it from the canonical CLI's output."
)
# Samples the zero-install script does not yet detect with full parity.
# Adding parity to `tools/shipgate-detect.py` is tracked separately; the
# CLI's signals.py already covers these cases.
SCRIPT_PARITY_GAPS: frozenset[str] = frozenset({"n8n_workflow_agent"})
def _write_skipped_fixture_signals(root: Path) -> None:
root.mkdir(parents=True, exist_ok=True)
(root / "agent.py").write_text(
"from langchain.tools import tool\n\n@tool\ndef lookup():\n return 'x'\n",
encoding="utf-8",
)
tools = root / "tools"
tools.mkdir()
(tools / "payments-mcp.json").write_text(
'{"tools": [{"name": "create_payment_link", "description": "Create link."}]}',
encoding="utf-8",
)
(root / "broken-mcp.json").write_text("{not json", encoding="utf-8")
plugin = root / "plugin" / ".codex-plugin"
plugin.mkdir(parents=True)
(plugin / "plugin.json").write_text("{}", encoding="utf-8")
def _framework(result: dict[str, Any], framework: str) -> dict[str, Any] | None:
return next(
(item for item in result["frameworks"] if item["type"] == framework), None
)
def _assert_detect_parity(script_module, workspace: Path, label: str) -> None:
"""Structural parity on one workspace: the script must agree with the
canonical CLI on (a) ``is_agent_project``, (b) the set of fired
frameworks, (c) the set of suggested-source types and paths, (d) the set
of excluded-source types and paths, and (e) workspace-signals keys.
One function, called by the ``samples/`` sweep and by the constructed
workspaces below. A second comparison written beside this one would be a
weaker comparison: the route added in #485 is exercised by workspaces no
sample has, and pinning it against a hand-written subset of these
assertions is how the two detectors would agree on everything anybody
checked and differ everywhere else.
"""
script_result = script_module.detect(workspace)
cli_result = detect_workspace(workspace.resolve()).model_dump(mode="json")
assert script_result["is_agent_project"] == cli_result["is_agent_project"], (
f"{label}: is_agent_project diverged "
f"(script={script_result['is_agent_project']}, "
f"cli={cli_result['is_agent_project']})."
)
script_frameworks = sorted(f["type"] for f in script_result["frameworks"])
cli_frameworks = sorted(f["type"] for f in cli_result["frameworks"])
assert script_frameworks == cli_frameworks, (
f"{label}: framework set diverged "
f"(script={script_frameworks!r}, cli={cli_frameworks!r}). "
"The script's scoring rules must match cli/discovery/signals.py."
)
# `mcp_server_source` is the one framework whose score and evidence are
# pinned as well as its presence. The looseness the rest of this contract
# grants — ±0.5 on the score, descriptive rather than byte-identical
# strings — was granted to detections scored from many heuristic signals
# and described in the script's own words. This one has exactly two
# scoring inputs, a resolved registration and a declared dependency, and
# its lines are rendered by a function ported verbatim; there is nothing
# here for the script to paraphrase.
#
# Both halves are load-bearing and invisible to every other assertion. The
# dependency is what carries the published label to `medium` rather than to
# the weakest thing discovery can say, and the framework fires either way.
# And the conditional lines are claims, not prose: "61 tools" without "and
# 3 more this reader cannot name" is the over-claim this whole input exists
# to avoid, and a detector that dropped it would agree on the verdict and
# publish a different fact about it (#485).
script_mcp = _framework(script_result, mcp_server_source.SOURCE_TYPE)
cli_mcp = _framework(cli_result, mcp_server_source.SOURCE_TYPE)
assert (script_mcp is None) == (cli_mcp is None)
if script_mcp is not None and cli_mcp is not None:
assert (
script_mcp["score"],
script_mcp["confidence"],
script_mcp["evidence"],
) == (
cli_mcp["score"],
cli_mcp["confidence"],
cli_mcp["evidence"],
), (
f"{label}: the MCP registration route reported a different "
f"score/confidence/evidence "
f"(script={script_mcp['score']}/{script_mcp['confidence']}/"
f"{script_mcp['evidence']}, "
f"cli={cli_mcp['score']}/{cli_mcp['confidence']}/"
f"{cli_mcp['evidence']})."
)
script_sources = sorted(
(s["type"], s["path"]) for s in script_result["suggested_sources"]
)
cli_sources = sorted(
(s["type"], s["path"]) for s in cli_result["suggested_sources"]
)
assert script_sources == cli_sources, (
f"{label}: suggested_sources diverged "
f"(script={script_sources!r}, cli={cli_sources!r})."
)
assert sorted(
(s["path"], s["reason"])
for s in script_result["excluded_sources"]
if s["type"] == mcp_server_source.SOURCE_TYPE
) == sorted(
(s["path"], s["reason"])
for s in cli_result["excluded_sources"]
if s["type"] == mcp_server_source.SOURCE_TYPE
), (
f"{label}: the withheld MCP registration route was explained "
"differently. The reason is the only thing a reader gets when a route "
"disappears, and a route that vanishes without one is "
"indistinguishable from one nobody implemented."
)
script_excluded = sorted(
(s["type"], s["path"]) for s in script_result["excluded_sources"]
)
cli_excluded = sorted(
(s["type"], s["path"]) for s in cli_result["excluded_sources"]
)
assert script_excluded == cli_excluded, (
f"{label}: excluded_sources diverged "
f"(script={script_excluded!r}, cli={cli_excluded!r}). "
"The script's stdlib parse probe must reject the same JSON "
"candidates as cli/discovery/artifacts.py:probe_suggested_source."
)
script_codex = sorted(
(s["mode"], s["path"]) for s in script_result["codex_plugin_candidates"]
)
cli_codex = sorted(
(s["mode"], s["path"]) for s in cli_result["codex_plugin_candidates"]
)
assert script_codex == cli_codex, (
f"{label}: codex_plugin_candidates diverged "
f"(script={script_codex!r}, cli={cli_codex!r})."
)
assert script_result["agent_scope"] == cli_result["agent_scope"], (
f"{label}: agent_scope diverged "
f"(script={script_result['agent_scope']!r}, "
f"cli={cli_result['agent_scope']!r}). An agent that consults the "
"zero-install path must not adopt a scope the CLI refuses."
)
assert (
script_result["python_parse_truncated"]
== cli_result["python_parse_truncated"]
), (
f"{label}: python_parse_truncated diverged "
f"(script={script_result['python_parse_truncated']!r}, "
f"cli={cli_result['python_parse_truncated']!r}). It is the guard every "
"whole-workspace negative is gated on, `is_agent_project: false` "
"included, so the two detectors must agree about whether their own "
"classification is complete (#399 review)."
)
assert (
script_result["agent_scope_truncated"]
== cli_result["agent_scope_truncated"]
), (
f"{label}: agent_scope_truncated diverged "
f"(script={script_result['agent_scope_truncated']!r}, "
f"cli={cli_result['agent_scope_truncated']!r}). It says whether "
"agent_project_candidates enumerates the workspace or only the part "
"the parse reached; a caller that reads a truncated list as complete "
"concludes its own project is not an agent project (#395)."
)
script_projects = sorted(
(c["path"], tuple(c["agent_names"]))
for c in script_result["agent_project_candidates"]
)
cli_projects = sorted(
(c["path"], tuple(c["agent_names"]))
for c in cli_result["agent_project_candidates"]
)
assert script_projects == cli_projects, (
f"{label}: agent_project_candidates diverged "
f"(script={script_projects!r}, cli={cli_projects!r})."
)
cli_signals = cli_result["workspace_signals"]
script_signals = script_result["workspace_signals"]
assert set(script_signals) == set(cli_signals), (
f"{label}: workspace_signals keys diverged "
f"(script={set(script_signals)!r}, cli={set(cli_signals)!r})."
)
# Keys alone let the same named field mean two different things. When the
# canonical detector started locating conventional directories anywhere in
# the tree (#441), the script kept reading only the workspace root — so
# `has_tools_dir` was `true` on one side and `false` on the other while
# this test stayed green. These three are the located-directory answer, so
# they are compared by value.
for key in ("has_prompts_dir", "has_tools_dir", "conventional_dirs"):
assert script_signals[key] == cli_signals[key], (
f"{label}: workspace_signals[{key!r}] diverged "
f"(script={script_signals[key]!r}, cli={cli_signals[key]!r}). "
"The script must mirror cli/discovery/signals.py:"
"_conventional_dir_locations exactly."
)
assert script_result["agent_name_candidates"] == cli_result["agent_name_candidates"], (
f"{label}: agent_name_candidates diverged.\n"
f"script={script_result['agent_name_candidates']!r}\n"
f"cli={cli_result['agent_name_candidates']!r}\n"
"The ranking decides which agent the generated manifest declares as "
"the reviewed identity, so this one is byte parity, not structural: "
"the script's rules must match "
"cli/discovery/signals.py:_rank_agent_name_candidates exactly."
)
@pytest.mark.parametrize("sample_dir", _sample_dirs(), ids=_sample_ids())
def test_script_verdict_matches_cli(script_module, sample_dir):
if sample_dir.name in SCRIPT_PARITY_GAPS:
pytest.skip(
f"{sample_dir.name}: zero-install script parity not yet implemented "
"(see SCRIPT_PARITY_GAPS)."
)
_assert_detect_parity(script_module, sample_dir, sample_dir.name)
@pytest.mark.parametrize("sample_dir", _sample_dirs(), ids=_sample_ids())
def test_script_finds_at_least_one_python_file_when_cli_does(
script_module, sample_dir
):
"""The script's ``os.walk`` and the CLI's git-aware walker may
legitimately differ on file counts (e.g. samples with build
artifacts), but if the CLI sees Python files in a sample, the
script must too — otherwise framework detection is impossible."""
script_result = script_module.detect(sample_dir)
cli_result = detect_workspace(sample_dir.resolve()).model_dump(mode="json")
cli_count = cli_result["workspace_signals"]["python_file_count"]
script_count = script_result["workspace_signals"]["python_file_count"]
if cli_count > 0:
assert script_count > 0, (
f"{sample_dir.name}: CLI found {cli_count} python files but "
f"the zero-install script found 0. Walk logic diverged — "
"check SKIP_DIRS and the os.walk pruning."
)
_ADK_AGENT = """\
from google.adk.agents import Agent
def act(x: str) -> str:
\"\"\"Act.\"\"\"
return "ok"
root_agent = Agent(name="{name}", tools=[act])
"""
def test_script_and_cli_agree_on_a_truncated_walk(script_module, tmp_path):
"""The parity samples all sit far under the cap, so they only ever pin
``agent_scope_truncated: false``. A workspace that actually truncates is
where the two detectors could disagree about whether their own candidate
list is complete — and an agent that consults the zero-install path must
not read a capped list as an enumeration any more than it may read a
capped scope verdict as settled (#395, #399 review)."""
repo = tmp_path / "capped"
for name in ("aa_one", "aa_two", "zz_hidden"):
project = repo / name
project.mkdir(parents=True)
(project / "pyproject.toml").write_text(
f'[project]\nname = "{name}"\nversion = "0.1.0"\n', encoding="utf-8"
)
(project / "agent.py").write_text(
_ADK_AGENT.format(name=name), encoding="utf-8"
)
filler = repo / "mm_filler"
filler.mkdir()
for index in range(1200):
(filler / f"mod{index:05d}.py").write_text("x = 1\n", encoding="utf-8")
script_result = script_module.detect(repo)
cli_result = detect_workspace(repo.resolve()).model_dump(mode="json")
assert cli_result["agent_scope_truncated"] is True
assert script_result["agent_scope_truncated"] is True
assert cli_result["python_parse_truncated"] is True
assert script_result["python_parse_truncated"] is True
assert (
script_result["workspace_signals"]["python_file_total"]
== cli_result["workspace_signals"]["python_file_total"]
)
assert script_result["agent_scope"] == cli_result["agent_scope"]
assert (
script_result["workspace_signals"]["project_root_count"]
== cli_result["workspace_signals"]["project_root_count"]
)
# Both name a recovery that actually changes the outcome: repeating the
# same capped run reproduces the same verdict.
assert "--max-python-files" in script_result["next_action"]
assert "--max-python-files" in cli_result["next_action"]
def test_script_and_cli_agree_on_a_capped_single_scope(script_module, tmp_path):
"""A one-project workspace is `agent_scope: "single"` however early the
parse stopped, so the scope branches never fire and both detectors fell
through to "Workspace does not appear to be an agent project. No action."
— a terminal false negative for an agent sitting past the cap (#399
review)."""
repo = tmp_path / "capped-single"
filler = repo / "aa_filler"
filler.mkdir(parents=True)
(repo / "pyproject.toml").write_text(
'[project]\nname = "capped"\nversion = "0.1.0"\n', encoding="utf-8"
)
for index in range(1001):
(filler / f"mod{index:05d}.py").write_text("x = 1\n", encoding="utf-8")
(repo / "zz_agent.py").write_text(
_ADK_AGENT.format(name="hidden_agent"), encoding="utf-8"
)
script_result = script_module.detect(repo)
cli_result = detect_workspace(repo.resolve()).model_dump(mode="json")
for label, result in (("script", script_result), ("cli", cli_result)):
assert result["python_parse_truncated"] is True, label
assert result["agent_scope"] == "single", label
assert result["is_agent_project"] is False, label
assert "does not appear to be an agent project" not in result["next_action"], (
f"{label}: a capped parse published a terminal false negative"
)
total = result["workspace_signals"]["python_file_total"]
assert f"--max-python-files {total}" in result["next_action"], label
# Human output takes the same route, ahead of every verdict below it.
import contextlib
import io
buffer = io.StringIO()
with contextlib.redirect_stdout(buffer):
script_module.main(["--workspace", str(repo)])
printed = buffer.getvalue()
assert "does not appear to be an agent project" not in printed
assert "--max-python-files" in printed
def test_script_and_cli_skip_common_fixture_dirs(script_module, tmp_path):
_write_skipped_fixture_signals(tmp_path / "fixtures")
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
for result in (script_result, cli_result):
assert result["is_agent_project"] is False
assert result["frameworks"] == []
assert result["suggested_sources"] == []
assert result["excluded_sources"] == []
assert result["codex_plugin_candidates"] == []
assert result["workspace_signals"]["python_file_count"] == 0
def test_script_and_cli_dedupe_marketplace_covered_package(script_module, tmp_path):
plugin = tmp_path / "plugins/reviewer/.codex-plugin/plugin.json"
plugin.parent.mkdir(parents=True)
plugin.write_text("{not-json", encoding="utf-8")
marketplace = tmp_path / ".agents/plugins/marketplace.json"
marketplace.parent.mkdir(parents=True)
marketplace.write_text(
'{"plugins":[{"name":"reviewer","source":'
'{"source":"local","path":"plugins/reviewer"}}]}',
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
expected = [("marketplace", ".agents/plugins/marketplace.json")]
assert [
(item["mode"], item["path"])
for item in script_result["codex_plugin_candidates"]
] == expected
assert [
(item["mode"], item["path"])
for item in cli_result["codex_plugin_candidates"]
] == expected
def test_script_and_cli_reject_oversized_marketplace_for_dedupe(
script_module,
tmp_path,
):
plugin = tmp_path / "plugins/reviewer/.codex-plugin/plugin.json"
plugin.parent.mkdir(parents=True)
plugin.write_text('{"name":"reviewer"}', encoding="utf-8")
marketplace = tmp_path / ".agents/plugins/marketplace.json"
marketplace.parent.mkdir(parents=True)
marketplace.write_text(
'{"plugins":[{"name":"reviewer","source":'
'{"source":"local","path":"plugins/reviewer"}}],"padding":"'
+ ("x" * (10 * 1024 * 1024))
+ '"}',
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
expected = {
("marketplace", ".agents/plugins/marketplace.json"),
("package", "plugins/reviewer"),
}
assert {
(item["mode"], item["path"])
for item in script_result["codex_plugin_candidates"]
} == expected
assert {
(item["mode"], item["path"])
for item in cli_result["codex_plugin_candidates"]
} == expected
def test_script_and_cli_reject_plugin_manifest_symlink_escape_from_coverage(
script_module,
tmp_path,
):
plugin = tmp_path / "plugins/reviewer/.codex-plugin/plugin.json"
plugin.parent.mkdir(parents=True)
outside_manifest = tmp_path.parent / f"{tmp_path.name}-outside-plugin.json"
outside_manifest.write_text('{"name":"reviewer"}', encoding="utf-8")
try:
plugin.symlink_to(outside_manifest)
except OSError as exc: # pragma: no cover - platform permission fallback
pytest.skip(f"symlinks unavailable: {exc}")
marketplace = tmp_path / ".agents/plugins/marketplace.json"
marketplace.parent.mkdir(parents=True)
marketplace.write_text(
'{"plugins":[{"name":"reviewer","source":'
'{"source":"local","path":"plugins/reviewer"}}]}',
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
assert script_module._local_marketplace_roots(tmp_path, [marketplace]) == set()
assert resolve_local_codex_marketplace_roots(
marketplace_path=marketplace,
base_dir=tmp_path,
) == ()
expected = {("marketplace", ".agents/plugins/marketplace.json")}
assert {
(item["mode"], item["path"])
for item in script_result["codex_plugin_candidates"]
} == expected
assert {
(item["mode"], item["path"])
for item in cli_result["codex_plugin_candidates"]
} == expected
def test_script_detects_workspace_named_fixture_dir(script_module, tmp_path):
workspace = tmp_path / "fixtures"
workspace.mkdir()
(workspace / "agent.py").write_text(
"from langchain.tools import tool\n\n@tool\ndef lookup():\n return 'x'\n",
encoding="utf-8",
)
script_result = script_module.detect(workspace)
cli_result = detect_workspace(workspace.resolve()).model_dump(mode="json")
assert script_result["is_agent_project"] is True
assert cli_result["is_agent_project"] is True
assert [fw["type"] for fw in script_result["frameworks"]] == ["langchain"]
assert [fw["type"] for fw in cli_result["frameworks"]] == ["langchain"]
assert script_result["workspace_signals"]["python_file_count"] == 1
assert cli_result["workspace_signals"]["python_file_count"] == 1
def test_script_excludes_mcpservers_config_like_cli(script_module, tmp_path):
"""The load-bearing parse-probe case: a Cursor-style ``mcpServers``
host config matches the ``*mcp*.json`` glob but is not a tools-array
export. No published sample carries one, so this pins the behavior
directly — script and CLI must BOTH drop it from suggested_sources
and report it under excluded_sources. Regression for the zero-install
half of the init->scan cold-start fix (the canonical CLI was fixed in
cli/discovery; the script is a separate stdlib re-implementation)."""
plugin_dir = tmp_path / "providers" / "cursor" / "plugin"
plugin_dir.mkdir(parents=True)
(plugin_dir / "mcp.json").write_text(
'{"mcpServers": {"stripe": {"command": "npx"}}}', encoding="utf-8"
)
(tmp_path / "tools").mkdir()
(tmp_path / "tools" / "payments-mcp.json").write_text(
'{"tools": [{"name": "create_payment_link", "description": '
'"Create a payment link for checkout."}]}',
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
assert script_result["suggested_sources"] == [
{"type": "mcp", "path": "tools/payments-mcp.json"}
]
assert [
(s["type"], s["path"]) for s in script_result["suggested_sources"]
] == [(s["type"], s["path"]) for s in cli_result["suggested_sources"]]
assert [
(s["type"], s["path"]) for s in script_result["excluded_sources"]
] == [(s["type"], s["path"]) for s in cli_result["excluded_sources"]] == [
("mcp", "providers/cursor/plugin/mcp.json")
]
assert "mcpServers" in script_result["excluded_sources"][0]["reason"]
def test_script_keeps_yaml_openapi_as_suggestion(script_module, tmp_path):
"""The stdlib probe is JSON-only (no YAML parser). A ``.yaml`` OpenAPI
spec must be kept as a suggestion, never wrongly excluded — and a
valid spec stays in parity with the CLI, which parses it for real."""
specs = tmp_path / "specs"
specs.mkdir()
(specs / "support.openapi.yaml").write_text(
"openapi: 3.1.0\ninfo:\n title: T\n version: '1'\npaths: {}\n",
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
assert script_result["suggested_sources"] == [
{"type": "openapi", "path": "specs/support.openapi.yaml"}
]
assert script_result["excluded_sources"] == []
assert [
(s["type"], s["path"]) for s in cli_result["suggested_sources"]
] == [("openapi", "specs/support.openapi.yaml")]
def test_script_excludes_swagger2_json_like_cli(script_module, tmp_path):
"""A Swagger 2.0 *JSON* document matches the ``*swagger*.json`` glob
but the openapi adapter only accepts OpenAPI 3.x. JSON is parseable
with stdlib, so the script must exclude it exactly like the CLI."""
(tmp_path / "legacy-swagger.json").write_text(
'{"swagger": "2.0", "info": {"title": "t", "version": "1"}, "paths": {}}',
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
assert script_result["suggested_sources"] == []
assert [
(s["type"], s["path"]) for s in script_result["excluded_sources"]
] == [(s["type"], s["path"]) for s in cli_result["excluded_sources"]] == [
("openapi", "legacy-swagger.json")
]
#: The bound the size-gate tests below patch in, and padding that clears it.
#: A faithful fixture would be 10 MB of writes per candidate; the rule under
#: test is "compare the file's size against the configured bound", and that
#: rule is exercised identically at 512 bytes.
_PATCHED_INPUT_BOUND = 512
_OVER_BOUND_PAD = "x" * 4096
def _patch_structured_input_bound(monkeypatch, script_module) -> None:
"""Lower the pre-parse size bound on both sides of the parity contract.
Three constants, because the refusal has three enforcement points: the
adapters (``MAX_INPUT_FILE_BYTES``, what actually makes ``scan`` reject an
oversized source), CLI discovery (``MAX_STRUCTURED_FILE_BYTES``, which
must reach the same verdict without loading the file), and the
zero-install script's stdlib copy. They are pinned equal by
``test_structured_input_bound_matches_the_adapter_bound``; patching one
and not the others would make these tests pass on a build whose surfaces
disagree.
"""
from agents_shipgate.cli.discovery import artifacts
from agents_shipgate.inputs import common
monkeypatch.setattr(common, "MAX_INPUT_FILE_BYTES", _PATCHED_INPUT_BOUND)
monkeypatch.setattr(
artifacts, "MAX_STRUCTURED_FILE_BYTES", _PATCHED_INPUT_BOUND
)
monkeypatch.setattr(
script_module, "MAX_STRUCTURED_FILE_BYTES", _PATCHED_INPUT_BOUND
)
def _oversize_reason(rel: str) -> str:
"""The reason ``load_structured_file``'s refusal reaches a user as."""
return (
f"Unable to read input file {rel}: Input file too large "
f"(limit: {_PATCHED_INPUT_BOUND} bytes): {rel}"
)
def test_structured_input_bound_matches_the_adapter_bound(script_module):
"""The three copies of the pre-parse size bound must be one value.
``MAX_INPUT_FILE_BYTES`` is the only one that decides anything at scan
time; the other two exist so discovery and the zero-install script can
predict that decision without reading the file. Let them drift and the
prediction is wrong in exactly the window between them — a file the
script suggests and ``scan`` refuses, which is the cold-start break the
parse probe exists to prevent.
"""
from agents_shipgate.cli.discovery.artifacts import MAX_STRUCTURED_FILE_BYTES
from agents_shipgate.inputs.common import MAX_INPUT_FILE_BYTES
assert (
MAX_STRUCTURED_FILE_BYTES
== MAX_INPUT_FILE_BYTES
== script_module.MAX_STRUCTURED_FILE_BYTES
)
def test_script_and_cli_exclude_oversized_sources_alike(
script_module, tmp_path, monkeypatch
):
"""An oversized candidate is refused by the adapters before they parse
it, so the CLI reports it under ``excluded_sources``. The script has to
reach that verdict from ``stat`` alone — both because reading a
several-hundred-megabyte ``*mcp*.json`` into memory is not something a
``curl | python3`` detector may do on an unknown repository, and because
suggesting a source ``scan`` refuses breaks the documented
``init --write`` → ``scan`` step.
Three shapes, each valid but for its size, so size is the only thing
that can exclude them:
* JSON the stdlib probe *could* have parsed and accepted;
* a ``.yaml`` OpenAPI spec, which the script keeps whenever the verdict
would depend on content it has no parser for — the size gate is asked
ahead of that, because it is the one content-independent rejection;
* an ``mcpServers``-style host config, which must still be excluded
under the *size* reason: the CLI's host-config sniff re-reads the
file, so it is skipped above the bound, and the two surfaces would
otherwise name different causes for the same exclusion.
"""
_patch_structured_input_bound(monkeypatch, script_module)
(tmp_path / "tools").mkdir()
(tmp_path / "tools" / "payments-mcp.json").write_text(
'{"tools": [{"name": "create_payment_link", "description": "Pay."}]}',
encoding="utf-8",
)
(tmp_path / "tools" / "bulk-mcp.json").write_text(
'{"tools": [{"name": "bulk_export", "description": "'
+ _OVER_BOUND_PAD
+ '"}]}',
encoding="utf-8",
)
(tmp_path / "providers").mkdir()
(tmp_path / "providers" / "host-mcp.json").write_text(
'{"mcpServers": {"stripe": {"command": "npx", "note": "'
+ _OVER_BOUND_PAD
+ '"}}}',
encoding="utf-8",
)
(tmp_path / "specs").mkdir()
(tmp_path / "specs" / "support.openapi.yaml").write_text(
"openapi: 3.1.0\ninfo:\n title: T\n version: '1'\n"
f" description: {_OVER_BOUND_PAD}\npaths: {{}}\n",
encoding="utf-8",
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
assert script_result["suggested_sources"] == [
{"type": "mcp", "path": "tools/payments-mcp.json"}
]
assert [(s["type"], s["path"]) for s in script_result["suggested_sources"]] == [
(s["type"], s["path"]) for s in cli_result["suggested_sources"]
]
expected_excluded = [
("openapi", "specs/support.openapi.yaml"),
("mcp", "providers/host-mcp.json"),
("mcp", "tools/bulk-mcp.json"),
]
assert (
[(s["type"], s["path"]) for s in script_result["excluded_sources"]]
== [(s["type"], s["path"]) for s in cli_result["excluded_sources"]]
== expected_excluded
)
# The reason is pinned byte for byte here, unlike the script's other
# evidence strings: an agent that reads ``excluded_sources`` decides
# whether to shrink, split, or ignore the file, and "too large" and
# "wrong shape" call for different work.
for result in (script_result, cli_result):
assert {s["path"]: s["reason"] for s in result["excluded_sources"]} == {
path: _oversize_reason(path) for _kind, path in expected_excluded
}
def test_script_and_cli_ignore_oversized_workflows_for_scoring(
script_module, tmp_path, monkeypatch
):
"""The n8n and Conductor framework signals read their glob hits whole.
Both adapters load workflows through ``load_structured_file``, so a
workflow above the bound cannot be scanned as one — scoring its framework
off it would name a framework nobody can verify, and reading it to decide
is the same unbounded read. Small siblings hold the frameworks fired, so
this pins that the *oversized* files stopped contributing rather than
that the detector stopped looking at workflows.
"""
_patch_structured_input_bound(monkeypatch, script_module)
n8n_node = (
'{"nodes": [{"type": "n8n-nodes-base.httpRequest", "name": "call"}], '
'"connections": {}'
)
conductor_task = (
'{"name": "review", "schemaVersion": 2, '
'"tasks": [{"type": "CALL_MCP_TOOL", "name": "call"}]'
)
(tmp_path / "n8n").mkdir()
(tmp_path / "n8n" / "small.json").write_text(n8n_node + "}", encoding="utf-8")
(tmp_path / "n8n" / "bulk.json").write_text(
f'{n8n_node}, "notes": "{_OVER_BOUND_PAD}"}}', encoding="utf-8"
)
(tmp_path / "conductor").mkdir()
(tmp_path / "conductor" / "small.json").write_text(
conductor_task + "}", encoding="utf-8"
)
(tmp_path / "conductor" / "bulk.json").write_text(
f'{conductor_task}, "notes": "{_OVER_BOUND_PAD}"}}', encoding="utf-8"
)
script_result = script_module.detect(tmp_path)
cli_result = detect_workspace(tmp_path.resolve()).model_dump(mode="json")
for result in (script_result, cli_result):
fired = {f["type"] for f in result["frameworks"]}
assert fired == {"n8n", "conductor"}
cited = {
path
for framework in result["frameworks"]
for path in framework["candidate_files"]
}
assert cited == {"n8n/small.json", "conductor/small.json"}
def _write_ranking_probe(root: Path) -> None:
"""Both issue shapes in one workspace: an ADK coordinator bound through
``App(root_agent=…)`` with a name resolved from an adjacent config
module, two literal sub-agents, and a one-character test literal."""
root.mkdir(parents=True, exist_ok=True)
(root / "config.py").write_text(
'import os\n\nAGENT_NAME = os.environ.get("AGENT_NAME", "SmartCloserAgent")\n',
encoding="utf-8",
)
(root / "agent.py").write_text(
"from config import AGENT_NAME\n"
"from google.adk.agents import LlmAgent\n"
"from google.adk.apps import App\n"
"from google.adk.tools import FunctionTool\n\n"
'salesforce_agent = LlmAgent(name="SalesforceAgent")\n'
'sap_agent = LlmAgent(name="SapAgent")\n'
# Annotated on purpose: the assignment-target lookup that resolves
# `App(root_agent=root_agent)` has to read AnnAssign as well as Assign.
"root_agent: LlmAgent = LlmAgent(\n"
" name=AGENT_NAME,\n"
" sub_agents=[salesforce_agent, sap_agent],\n"
" tools=[FunctionTool(func=lambda: None)],\n"
")\n"
'app = App(name="smart_closer_app", root_agent=root_agent)\n',
encoding="utf-8",
)
tests_dir = root / "tests"
tests_dir.mkdir()
(tests_dir / "test_agent.py").write_text(
'from google.adk.agents import LlmAgent\n\nfixture = LlmAgent(name="t")\n',
encoding="utf-8",
)
@pytest.mark.parametrize(
"non_product", ["tests/test_agent.py", "skills/recipe/resources/templates/app/agent.py"]