From 746a905e5cca19a52b8b0cb62aadbf9234bd2f80 Mon Sep 17 00:00:00 2001 From: yuxuanj Date: Wed, 8 Jul 2026 16:26:47 -0700 Subject: [PATCH] fix(no-dead-urls): fix bad User-Agent header, suppress redirect noise, tolerate transient blips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Patch dead-or-alive (via patch-package) to send the User-Agent under the correct `user-agent` header key. It was previously sent as `userAgent`, which isn't a valid HTTP header name, so the real User-Agent the target server saw was undici's default ("undici") — trivially identifiable as bot/script traffic to UA-based bot mitigation, causing intermittent false "dead URL" reports on otherwise-alive sites (e.g. business.adobe.com). - Suppress "Unexpected redirecting URL" advisories from no-dead-urls: they fired even when a link resolved successfully, contradicting the existing `followRedirects: true` config's intent to not flag redirects as issues. - Raise deadOrAliveOptions maxRetries from 0 to 1 (1s backoff) and timeout from the 3s default to 10s. remark-lint-no-dead-urls checks up to 10 URLs concurrently per file; under that load some requests queue long enough to approach the old 3s timeout even though the target is alive, and zero retries meant a single blip was reported as a permanent dead link. See DEVSITE-2493 for root-cause details and before/after verification data from a full local lint run against AdobeDocs/aep-mobile-sdkdocs. Co-Authored-By: Claude Sonnet 5 --- bin/runLint.js | 35 +++++++++++++++++++++++++++---- package.json | 4 +++- patches/dead-or-alive+1.0.5.patch | 18 ++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 patches/dead-or-alive+1.0.5.patch diff --git a/bin/runLint.js b/bin/runLint.js index 7b096c1..75136af 100644 --- a/bin/runLint.js +++ b/bin/runLint.js @@ -183,8 +183,15 @@ function createProcessor(includeFrontmatterCheck) { .use(remarkLintNoDeadUrls, { skipUrlPatterns: [...skipUrlPatterns, /^mailto:/, /^#/], deadOrAliveOptions: { - maxRetries: 0, - sleep: 0, + // A single transient blip (rate limit, cold CDN edge, brief network + // hiccup) would otherwise be reported as a dead link with zero + // chance to recover, so allow one retry with a short backoff. + maxRetries: 1, + sleep: () => 1000, + // Under concurrent load (10 URLs checked in parallel per file), + // some requests queue long enough to approach the default 3s + // timeout even though the target is alive; give them more room. + timeout: 10000, https: { rejectUnauthorized: false, // Don't fail on SSL cert issues }, @@ -231,8 +238,17 @@ function createProcessor(includeFrontmatterCheck) { .use(remarkLintNoDeadUrls, { skipUrlPatterns: [...skipUrlPatterns, /^mailto:/, /^#/], deadOrAliveOptions: { - maxRetries: 0, - sleep: 0, + // A single transient blip (rate limit, cold CDN edge, brief + // network hiccup) would otherwise be reported as a dead link + // with zero chance to recover, so allow one retry with a + // short backoff. + maxRetries: 1, + sleep: () => 1000, + // Under concurrent load (10 URLs checked in parallel per + // file), some requests queue long enough to approach the + // default 3s timeout even though the target is alive; give + // them more room. + timeout: 10000, https: { rejectUnauthorized: false, // Don't fail on SSL cert issues }, @@ -372,6 +388,17 @@ for (const filePath of markdownFiles) { } } + // Suppress "redirecting URL" advisories from no-dead-urls: the URL was + // confirmed alive (dead-or-alive followed the redirect successfully), + // matching the `followRedirects: true` config below which is meant to + // treat redirects as non-issues rather than flag them. + for (let i = result.messages.length - 1; i >= 0; i--) { + const message = result.messages[i]; + if (message.ruleId === 'no-dead-urls' && /^Unexpected redirecting URL/.test(message.message)) { + result.messages.splice(i, 1); + } + } + if (result.messages.length > 0) { filesWithIssues++; totalIssues += result.messages.length; diff --git a/package.json b/package.json index fb32861..7bd4d43 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ }, "scripts": { "start": "node index.js", - "list": "node index.js" + "list": "node index.js", + "postinstall": "patch-package" }, "author": "", "license": "Apache-2.0", @@ -20,6 +21,7 @@ "dotenv": "^17.2.2", "express": "^5.1.0", "glob": "^13.0.0", + "patch-package": "^8.0.1", "remark": "^15.0.1", "remark-cli": "^12.0.1", "remark-frontmatter": "^5.0.0", diff --git a/patches/dead-or-alive+1.0.5.patch b/patches/dead-or-alive+1.0.5.patch new file mode 100644 index 0000000..911f2c5 --- /dev/null +++ b/patches/dead-or-alive+1.0.5.patch @@ -0,0 +1,18 @@ +diff --git a/node_modules/dead-or-alive/lib/index.js b/node_modules/dead-or-alive/lib/index.js +index 38cc6ea..4d9481f 100644 +--- a/node_modules/dead-or-alive/lib/index.js ++++ b/node_modules/dead-or-alive/lib/index.js +@@ -210,7 +210,12 @@ async function deadOrAliveInternal(state, url) { + + response = await fetch(url, { + headers: { +- userAgent: state.userAgent, ++ // Note: must be the real `user-agent` header name; `userAgent` is not ++ // a valid HTTP header and was silently sent as a bogus extra header, ++ // leaving the real `user-agent` header at undici's default (`undici`), ++ // which trivially identifies the request as non-browser/bot traffic ++ // to any user-agent-based bot mitigation (e.g. Akamai). ++ 'user-agent': state.userAgent, + // + 'Upgrade-Insecure-Requests': '1', + accept: