# -----------------------------------------------
# ALLOW ALL LEGITIMATE SEARCH ENGINE CRAWLERS
# -----------------------------------------------
User-agent: *
Allow: /
Disallow: /api
Disallow: /event/
Disallow: /webhook/
Disallow: /*cid=*

# -----------------------------------------------
# SEMRUSH - Multiple sub-bots
# -----------------------------------------------
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: SemrushBot-CT
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: SemrushBot-COUB
Disallow: /
User-agent: SemrushBot-OCOB
Disallow: /
User-agent: SiteAuditBot
Disallow: /
User-agent: SplitSignalBot
Disallow: /

# -----------------------------------------------
# SIMILARWEB
# -----------------------------------------------
User-agent: similarweb
Disallow: /

# -----------------------------------------------
# AHREFS
# -----------------------------------------------
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /

# -----------------------------------------------
# MOZ
# -----------------------------------------------
User-agent: rogerbot
Disallow: /
User-agent: dotbot
Disallow: /

# -----------------------------------------------
# MAJESTIC
# -----------------------------------------------
User-agent: MJ12bot
Disallow: /

# -----------------------------------------------
# SISTRIX
# -----------------------------------------------
User-agent: sistrix
Disallow: /

# -----------------------------------------------
# MANGOOLS / KWFINDER
# -----------------------------------------------
User-agent: MangoolsBot
Disallow: /

# -----------------------------------------------
# SERPSTAT
# -----------------------------------------------
User-agent: serpstatbot
Disallow: /

# -----------------------------------------------
# SPYFU / COGNITIVE SEO
# -----------------------------------------------
User-agent: CognitiveSEO
Disallow: /

# -----------------------------------------------
# SEOKICKS / BABBAR / OTHER EU TOOLS
# -----------------------------------------------
User-agent: SEOkicks
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: Barkrowler
Disallow: /

# -----------------------------------------------
# DATAFORSEO
# -----------------------------------------------
User-agent: DataForSeoBot
Disallow: /

# -----------------------------------------------
# EXABOT (Used by various intelligence tools)
# -----------------------------------------------
User-agent: Exabot
Disallow: /

# -----------------------------------------------
# GIGABOT
# -----------------------------------------------
User-agent: Gigabot
Disallow: /

# -----------------------------------------------
# NETCRAFT (Infrastructure intelligence)
# -----------------------------------------------
User-agent: NetcraftSurveyAgent
Disallow: /

# -----------------------------------------------
# ARCHIVE / WAYBACK MACHINE (optional - remove
# if you want historical archiving)
# -----------------------------------------------
User-agent: ia_archiver
Disallow: /