diff --git a/custom-next-sitemap.js b/custom-next-sitemap.js index 8ae3445..2134790 100644 --- a/custom-next-sitemap.js +++ b/custom-next-sitemap.js @@ -116,7 +116,11 @@ module.exports = { policies: [ // Single combined wildcard record so crawlers that only honor the first // matching User-agent don't miss the Disallow directives. - { userAgent: '*', allow: '/', disallow: ['/api/', '/_next/', '/static/'] }, + // `/api/og` is carved out of the `/api/` block: it backs every og:image + // meta tag, so blocking it makes Google report each social card as + // "Blocked by robots.txt" and skip it for rich results. Longest-match + // wins in Google/Bing, so the Allow overrides the broader Disallow. + { userAgent: '*', allow: ['/', '/api/og'], disallow: ['/api/', '/_next/', '/static/'] }, // AI / LLM crawlers — explicitly allow each so they don't fall back to // wildcard rules and so we have a single source of truth. { userAgent: 'GPTBot', allow: '/' }, diff --git a/public/robots.txt b/public/robots.txt index 78ee17b..8ade6e9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,6 +1,7 @@ # * User-agent: * Allow: / +Allow: /api/og Disallow: /api/ Disallow: /_next/ Disallow: /static/ diff --git a/scripts/validate-sitemap.js b/scripts/validate-sitemap.js index 5a8f528..71c0e61 100644 --- a/scripts/validate-sitemap.js +++ b/scripts/validate-sitemap.js @@ -102,17 +102,28 @@ if (origins.has('http://localhost:3000') || [...origins].some((o) => o.includes( // robots.txt is committed while the sitemap is generated, so the two drift // independently. A Sitemap: line pointing at the wrong host is invisible until a // crawler follows it. -if (fs.existsSync(robotsFile) && origins.size === 1) { +if (fs.existsSync(robotsFile)) { const robots = fs.readFileSync(robotsFile, 'utf8') - const declared = [...robots.matchAll(/^\s*Sitemap:\s*(\S+)\s*$/gim)].map((m) => m[1]) - if (declared.length === 0) fail(`${rel(robotsFile)} declares no Sitemap: line.`) - for (const entry of declared) { - try { - if (new URL(entry).origin !== [...origins][0]) { - fail(`${rel(robotsFile)} points at ${entry}, but the sitemap uses ${[...origins][0]}.`) + + // /api/og renders every og:image. It sits under the broader `/api/` Disallow, + // so it needs an explicit Allow carve-out (longest-match wins in Google/Bing). + // Without it, Google silently reports each social card as "Blocked by + // robots.txt" and skips it for rich results. + if (!/^\s*Allow:\s*\/api\/og\s*$/im.test(robots)) { + fail(`${rel(robotsFile)} is missing "Allow: /api/og". og:image URLs fall under the /api/ Disallow.`) + } + + if (origins.size === 1) { + const declared = [...robots.matchAll(/^\s*Sitemap:\s*(\S+)\s*$/gim)].map((m) => m[1]) + if (declared.length === 0) fail(`${rel(robotsFile)} declares no Sitemap: line.`) + for (const entry of declared) { + try { + if (new URL(entry).origin !== [...origins][0]) { + fail(`${rel(robotsFile)} points at ${entry}, but the sitemap uses ${[...origins][0]}.`) + } + } catch { + fail(`${rel(robotsFile)} has an unparseable Sitemap: entry: ${entry}`) } - } catch { - fail(`${rel(robotsFile)} has an unparseable Sitemap: entry: ${entry}`) } } }