# robots.txt for Sugarberry Memorials
# Purpose: Allow search indexing, permit AI crawlers, and protect server performance.

# --- Standard search engine bots ---
User-agent: Pinterestbot
Allow: /
Crawl-delay: 1

User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

# --- AI and LLM crawlers ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Microsoft-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /
User-agent: Amazonbot-Extended
Allow: /


# --- Specific bot tuning and rate limiting ---
# Block aggressive or redundant crawlers
User-agent: Pinterestbot
Disallow: /*?*
Disallow: /*add-to-cart=*
Disallow: /cart/
Disallow: /checkout/
Crawl-delay: 1

User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /

# --- General fallback for all other bots ---
User-agent: *
Allow: /
Disallow: /*?attribute_pa_*
Disallow: /*?filter_*
Disallow: /*?orderby=*
Disallow: /*?v=
Disallow: /?s=
Disallow: /*add-to-cart=*
Disallow: /*?add-to-cart=*
Crawl-delay: 10

# --- Allow media, images, and product data ---
Allow: /wp-content/uploads/

# --- Disallow sensitive WordPress paths (for security and cleanliness) ---
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# --- Reference for crawlers ---
Sitemap: https://www.sugarberrymemorials.com/sitemap.xml
AI-Policy: https://www.sugarberrymemorials.com/llms.txt