# https://www.audionotes.app robots.txt

# Default: allow all standard crawlers
User-agent: *
Allow: /
# /_next/static/* is intentionally crawlable so Googlebot can fetch JS/CSS to
# render pages. Those files carry an `X-Robots-Tag: noindex` header (see
# next.config.mjs) which keeps them out of the index without a robots block.
Disallow: /api/
Disallow: /admin/
Disallow: /login/
Disallow: /*/opengraph-image
# Tracking/affiliate query variants (?aff=, ?ref=, ?via=, ?utm_, ?_f=, and
# one-off directory keys) are intentionally NOT disallowed. Every page emits a
# self-referencing canonical, so Google folds them into the clean URL on
# recrawl. Blocking them froze ~230 URLs in GSC "Crawled - currently not
# indexed" because Google could never recrawl to read the canonical.
# (Removed 2026-09-12 per the crawled-not-indexed audit.)

# Bing: no crawl-delay. Bing now respects URL-Submission via IndexNow,
# and crawl-delay slows reindexing on a small site for no benefit.
# (Removed 2026-05-05 per 2026 best-practices audit.)

# AI search & assistant crawlers (explicit allow for visibility in AI Overviews,
# ChatGPT browsing, Perplexity, Claude, etc.)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

# Social/share-preview crawlers (OG image fetching for link previews)
User-agent: Applebot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: YouBot
Allow: /

# Sitemaps
Sitemap: https://www.audionotes.app/sitemap.xml
Sitemap: https://www.audionotes.app/blog/sitemap-index.xml