# --- Commercial SEO crawlers: no business value, full stop ----------------
User-agent: AhrefsBot
Disallow: /
 
User-agent: SemrushBot
Disallow: /
 
User-agent: MJ12bot
Disallow: /
 
User-agent: DotBot
Disallow: /
 
User-agent: BLEXBot
Disallow: /
 
User-agent: DataForSeoBot
Disallow: /
 
User-agent: Barkrowler
Disallow: /
 
User-agent: PetalBot
Disallow: /
 
User-agent: Bytespider
Disallow: /
 
 
# --- AI crawlers: decide per your policy ---------------------------------
# Blocking these costs you nothing commercially today. Allowing them may
# surface your products in AI answers. Pick one stance and be consistent.
User-agent: GPTBot
Disallow: /
 
User-agent: OAI-SearchBot
Disallow: /
 
User-agent: ClaudeBot
Disallow: /
 
User-agent: anthropic-ai
Disallow: /
 
User-agent: CCBot
Disallow: /
 
User-agent: PerplexityBot
Disallow: /
 
User-agent: meta-externalagent
Disallow: /
 
User-agent: Amazonbot
Disallow: /
 
User-agent: Google-Extended
Disallow: /


# --- Search engines you want ---------------------------------------------
User-agent: Googlebot
Allow: /
# Googlebot ignores Crawl-delay. Use Search Console > Settings > Crawl rate
# if you need to slow it, though it self-throttles on slow responses.
 
User-agent: bingbot
Allow: /
Crawl-delay: 5
 
User-agent: YandexBot
Crawl-delay: 10

User-agent: *
Crawl-delay: 10

Disallow: *?*searchfilter=
Disallow: *?*perpage=
Disallow: *?*sorting=
Disallow: *?*lang=
Disallow: *?*searchtype=
Disallow: *?*groups=
Disallow: *?*searchsubmit=
Disallow: *?*query=
Disallow: *?*mode=
Disallow: *?*di=
Disallow: *?*lc=
Disallow: *?*cc=
Disallow: *?*regType=
Disallow: *?*nginxhost=
Disallow: *?*method=
Disallow: *?*action=
Disallow: *?*user=
Disallow: *?*type=
Disallow: *?*country=
Disallow: *?nr=


Sitemap: https://www.rolling.ee/sitemap.xml