# Robots.txt for frenchydigital.com
# Optimized for SEO crawling

# Google
User-agent: Googlebot
Allow: /
Disallow: /content/markdown/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Disallow: /content/markdown/
Crawl-delay: 1

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterest
Allow: /

# Other Major Search Engines
User-agent: Slurp
Allow: /
Disallow: /content/markdown/
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /
Disallow: /content/markdown/

User-agent: Baiduspider
Allow: /
Disallow: /content/markdown/
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Disallow: /content/markdown/
Crawl-delay: 2

# AI Crawlers (allow for visibility)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# Anthropic's current tokens. anthropic-ai and Claude-Web are retired but kept
# below so any crawler still sending them stays allowed rather than falling
# through to the User-agent: * group.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# SEO Audit Tools
User-agent: RSiteAuditor
Allow: /

# Default rule for all other bots
User-agent: *
Allow: /
Crawl-delay: 2

# Disallow patterns for all bots
User-agent: *
Disallow: /api/
Disallow: /_*
Disallow: /private/
Disallow: /*.json$
Disallow: /node_modules/
Disallow: /content/markdown/

# Block paginated URLs (various pagination parameter patterns)
Disallow: /*?*_page=
Disallow: /*?*page=
Disallow: /*?617b332e_page=
Disallow: /*?1aa987e3_page=

# Block tracking and campaign parameters
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?yandex-source=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?msclkid=
Disallow: /*?mc_cid=
Disallow: /*?mc_eid=

# Block WordPress legacy parameters
Disallow: /*?p=

# Block multiple parameter combinations
Disallow: /*?*&*_page=

# Sitemap location (both www and non-www for Search Console compatibility)
Sitemap: https://frenchydigital.com/sitemap.xml
Sitemap: https://www.frenchydigital.com/sitemap.xml

# AI/LLM Sitemap for enhanced AI crawler discovery
Sitemap: https://frenchydigital.com/sitemap-llm.xml

# Note: sitemap-markdown.xml intentionally NOT listed here. It duplicates
# every blog post as an uncanonicalized text/markdown URL with no signal
# back to the canonical HTML page, and submitting it invited Googlebot/Bingbot
# to crawl ~750 near-duplicate URLs (surfacing as "Crawled/Discovered - currently
# not indexed" in Search Console). AI agents still discover this content via
# llms.txt / llms-full.txt below; traditional search engines are now also
# blocked from /content/markdown/ directly, above.

# RSS Feed for blog syndication
# Contains all blog articles with excerpts for feed readers and aggregators

# LLM Context Files
# llms.txt - Quick reference for AI assistants
# llms-full.txt - Comprehensive content index with FAQs and article summaries

# Host directive (for Yandex)
Host: https://frenchydigital.com