# =========================
# Robots.txt for ErotikMaps
# =========================

# Main rules for all crawlers
User-agent: *

# Allow important resources for rendering
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /uploads/

# Block internal/admin paths
Disallow: /cdn-cgi/
Disallow: /admin/
Disallow: /api/
Disallow: /storage/
Disallow: /_debugbar/

# Ad plumbing for external sites: an iframe/script that returns ONE geo-targeted
# banner, and the click redirect. No content, a different body on every request,
# and satellites link it with ?source=<domain>, so Googlebot was crawling dozens
# of variants of it (2026-09). Nothing here should ever be indexed or crawled.
Disallow: /banner/serve
Disallow: /banner/embed
Disallow: /banner/click/

# Block URL parameters that create duplicate content
# Note: Pagination is allowed (canonical URLs handle consolidation)
# Disallow: /*?page=  <- Removed: Google recommends allowing access to paginated content
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?ref=
Disallow: /*?utm_

# =========================
# Block SEO scraper bots (save crawl budget)
# =========================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: dotbot
Disallow: /

# =========================
# LLM Documentation
# =========================
# Summary: https://www.erotikmaps.com/llm.txt
# Full: https://www.erotikmaps.com/llm-full.txt

# =========================
# Sitemap (IMPORTANT: use www domain)
# =========================
Sitemap: https://www.erotikmaps.com/sitemap.xml