# Search engines — welcome
User-Agent: Googlebot
Allow: /
Allow: /_next/static/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Disallow: /opengraph-image
# Store detail pages (/stores/<domain>): Google-noindexed, so don't spend crawl on them.
# Hubs (/stores, /stores/top-100, /stores/countries/*) contain no dot and stay crawlable.
Disallow: /stores/*.
# App detail pages (/apps/<slug>): same. /apps and /apps/category/* hubs stay crawlable
# (Google applies the longest matching rule, so the Allow wins for category hubs).
Disallow: /apps/
Allow: /apps/category/

User-Agent: Bingbot
Allow: /
Allow: /_next/static/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Disallow: /opengraph-image
Crawl-delay: 2

User-Agent: Yandex
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

# SEO tool bots — allow with rate limit
User-Agent: SemrushBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 10

User-Agent: AhrefsBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 10

# Low-value bots — block (aggressive crawling, no benefit)
User-Agent: MJ12bot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-Agent: PetalBot
Disallow: /

User-Agent: ByteSpider
Disallow: /

User-Agent: ZoominfoBot
Disallow: /

User-Agent: DataForSeoBot
Disallow: /

User-Agent: BomboraBot
Disallow: /

# AI crawlers — allow (so AI tools can reference/recommend the site)
User-Agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

# OpenAI search (ChatGPT citations) — not the same as GPTBot
User-Agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

# Anthropic — training / search / user-initiated fetch
User-Agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

# Gemini / Apple Intelligence grounding (not Googlebot)
User-Agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: cohere-ai
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

User-Agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: Meta-ExternalAgent
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Crawl-delay: 5

User-Agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/

# CCBot — block (Common Crawl, training data only, no referral value)
User-Agent: CCBot
Disallow: /

# Default — allow with rate limit
User-Agent: *
Allow: /
Allow: /_next/static/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /ingest/
Disallow: /opengraph-image
Crawl-delay: 3

Host: https://ecomscout.com
# One sitemap for every bot. Hubs only — no store/app leaves.
Sitemap: https://ecomscout.com/sitemap.xml

# LLM discovery (not a substitute for Sitemap:)
# https://ecomscout.com/llms.txt
# https://ecomscout.com/llms-full.txt
# https://ecomscout.com/.well-known/llms.txt