Retry with GET when HEAD fails, not just on 405

docs/how-it-works.md

@@ -7,7 +7,7 @@
 | No scheme, starts with `.`, `/`, or `#` | Internal | File existence on disk |
 | `https://github.com/<same-owner>/...` | Same-org | GitHub REST API |
 | `https://github.com/user-attachments/assets/...` | External | Skipped (always valid) |
-| Everything else | External | HTTP HEAD request, falling back to GET if HEAD returns 405 |
+| Everything else | External | HTTP HEAD request, falling back to GET if HEAD returns any 4xx or 5xx |
 
 Each unique URL is checked only once per run, regardless of how many files reference it.
 

docs/troubleshooting.md

@@ -4,7 +4,7 @@
 Ensure the workflow has `pull-requests: write` permission and that the `token` input is set (it defaults to `GITHUB_TOKEN` which should work in most cases).
 
 **External links are timing out.**
-Increase the `timeout` input (value is in milliseconds). The default is `10000` (10 seconds). Some sites are slow to respond to HEAD requests; HyperHawk falls back to GET automatically when HEAD returns 405.
+Increase the `timeout` input (value is in milliseconds). The default is `10000` (10 seconds). Some sites are slow to respond to HEAD requests; HyperHawk falls back to GET automatically whenever HEAD returns an error status, so a host that mishandles HEAD (for example `support.google.com`, which answers 404 to HEAD but serves the page on GET) is not reported as broken.
 
 **A link is reported as broken but it works in my browser.**
 Some sites block requests from CI runners or require cookies. Add the domain to `ignore-patterns` to skip it.

src/check.ts

@@ -676,8 +676,13 @@ async function checkExternal(link: LinkInfo, config: Config): Promise<CheckResul
   try {
     let result = await followRedirects(link.url, 'HEAD', timeout, BROWSER_HEADERS);
 
-    // HEAD returned Method Not Allowed; retry with GET
-    if (result.status === 405) {
+    // Plenty of hosts mishandle HEAD: some answer 405 Method Not Allowed, others
+    // return an outright error for a page they serve fine on GET (support.google.com
+    // answers 404 to HEAD but 200 to GET). Retry every failing HEAD with GET so the
+    // verdict matches what a reader actually gets. Only failures pay the extra
+    // request, and followRedirects discards the body as soon as headers arrive.
+    if (result.status >= 400) {
+      core.debug(`[external] ${link.url} returned HTTP ${result.status} for HEAD - retrying with GET`);
       result = await followRedirects(link.url, 'GET', timeout, BROWSER_HEADERS);
     }
 

tests/expected-output.txt

@@ -6,9 +6,14 @@
 ::debug::[same-org] URL points to current repo, suggesting local path: https://github.com/dvdstelt/hyperhawk/blob/main/does-not-exist.md
 ::debug::[same-org] Checking https://github.com/dvdstelt/hyperhawk/blob/main/docs/configuraton.md
 ::debug::[same-org] URL points to current repo, suggesting local path: https://github.com/dvdstelt/hyperhawk/blob/main/docs/configuraton.md
+::debug::[external] <server>/redirect-to-403 returned HTTP 403 for HEAD - retrying with GET
 ::debug::[external] <server>/redirect-to-403 returned HTTP 403 - treating as auth/bot-blocked, skipping
+::debug::[external] <server>/auth-required returned HTTP 401 for HEAD - retrying with GET
 ::debug::[external] <server>/auth-required returned HTTP 401 - treating as auth/bot-blocked, skipping
 ::debug::[external] <server>/cross-domain-redirect redirected to different host <server-alt>/final - skipping suggestion
+::debug::[external] <server>/not-found returned HTTP 404 for HEAD - retrying with GET
+::debug::[external] <server>/head-404-get-200 returned HTTP 404 for HEAD - retrying with GET
+::debug::[external] <server>/head-404-get-redirect returned HTTP 404 for HEAD - retrying with GET
 ::debug::[external] https://github.com/user-attachments/assets/35163316-65df-4f8d-bf85-03650025a7b4 is a GitHub user-attachment asset, treating as valid
 tests/test-document.md:1001 | <server>/redirect-301 | suggestion -> <server>/final
 tests/test-document.md:1002 | <server>/redirect-302 | suggestion -> <server>/final
@@ -21,6 +26,8 @@ tests/test-document.md:1008 | <server>/auth-required | ok
 tests/test-document.md:1009 | <server>/cross-domain-redirect | ok
 tests/test-document.md:1010 | <server>/not-found | broken | HTTP 404
 tests/test-document.md:1011 | https://github.com/user-attachments/assets/35163316-65df-4f8d-bf85-03650025a7b4 | ok
+tests/test-document.md:1012 | <server>/head-404-get-200 | ok
+tests/test-document.md:1013 | <server>/head-404-get-redirect | suggestion -> <server>/final
 tests/test-document.md:11 | ../release.ps | broken | File not found: <root>/release.ps
 tests/test-document.md:13 | ../tsconfig.jso | broken | File not found: <root>/tsconfig.jso
 tests/test-document.md:15 | ../action.ym | broken | File not found: <root>/action.ym

tests/snapshot.ts

@@ -79,6 +79,27 @@ function startRedirectServer(): Promise<{ baseUrl: string; close: () => Promise<
         res.writeHead(302, { Location: `http://localhost:${addr.port}/final` });
         res.end();
 
+      // Errors on HEAD but serves the page on GET (support.google.com behaviour)
+      } else if (url === '/head-404-get-200') {
+        if (req.method === 'HEAD') {
+          res.writeHead(404);
+          res.end();
+        } else {
+          res.writeHead(200);
+          res.end('OK');
+        }
+
+      // Errors on HEAD but redirects on GET: the GET retry must still yield a
+      // redirect suggestion rather than a broken-link report
+      } else if (url === '/head-404-get-redirect') {
+        if (req.method === 'HEAD') {
+          res.writeHead(404);
+          res.end();
+        } else {
+          res.writeHead(301, { Location: '/final' });
+          res.end();
+        }
+
       // 404 (broken link)
       } else {
         res.writeHead(404);
@@ -119,6 +140,8 @@ function buildExternalLinks(baseUrl: string, testFile: string): LinkInfo[] {
     { url: `${baseUrl}/auth-required`, text: 'auth required', line: 1008, lineContent: `[auth required](${baseUrl}/auth-required)` },
     { url: `${baseUrl}/cross-domain-redirect`, text: 'cross-domain redirect', line: 1009, lineContent: `[cross-domain redirect](${baseUrl}/cross-domain-redirect)` },
     { url: `${baseUrl}/not-found`, text: 'plain 404', line: 1010, lineContent: `[plain 404](${baseUrl}/not-found)` },
+    { url: `${baseUrl}/head-404-get-200`, text: 'head 404 get 200', line: 1012, lineContent: `[head 404 get 200](${baseUrl}/head-404-get-200)` },
+    { url: `${baseUrl}/head-404-get-redirect`, text: 'head 404 get redirect', line: 1013, lineContent: `[head 404 get redirect](${baseUrl}/head-404-get-redirect)` },
     { url: 'https://github.com/user-attachments/assets/35163316-65df-4f8d-bf85-03650025a7b4', text: 'user attachment', line: 1011, lineContent: '[user attachment](https://github.com/user-attachments/assets/35163316-65df-4f8d-bf85-03650025a7b4)' },
   ];