User-agent: *
Disallow: /ajax/
Disallow: /userfiles/archive/
Disallow: /r/
Disallow: /my-bspc/

User-agent: Googlebot-Image
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: PetalBot
Disallow: / 

User-agent: Bytespider
Disallow: / 

User-agent: dotbot
Disallow: / 

User-agent: SemrushBot
Disallow: /


User-agent: AhrefsBot
Crawl-Delay: 60 

User-agent: Yandex
Crawl-Delay: 60 

User-agent: MSNBot
Crawl-Delay: 60 

User-agent: Bingbot
Crawl-Delay: 3

User-agent: googlebot
Crawl-delay: 1

User-agent: applebot
Crawl-delay: 3

User-agent: MJ12bot
Crawl-delay: 20

User-agent: Barkrowler
Crawl-delay: 20

User-agent: GeedoProductSearch
Crawl-delay: 20

User-Agent: serpstatbot
Crawl-Delay: 20

User-agent: coccocbot
Crawl-delay: 20

User-agent: Sogou web spider
Crawl-Delay: 20


Sitemap: https://www.bspc.co.uk/xml/