# Block useless or abusive bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Crawl-delay: 10

# Block scrapers
User-agent: Scrapy
Disallow: /

User-agent: curl
Disallow: /

User-agent: python
Disallow: /

User-agent: wget
Disallow: /

# Allow all other search engines explicitly
User-agent: *
Allow: /
# Keep private, account, auth and search-result pages out of the index
Disallow: /account
Disallow: /api/
Disallow: /signin
Disallow: /login
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /search

# Sitemap index references every per-type sitemap (businesses, professionals,
# products, services, articles, guides, static pages) — keep this as the only
# entry so a new sitemap route only needs to be added in one place
# (app/sitemap-index.xml/route.ts) instead of drifting out of sync here.
Sitemap: https://www.rectopages.com/sitemap-index.xml