User-agent: *
Disallow: /go/
Disallow: /gt/
Crawl-delay: 20

User-agent: Yandex
Disallow: /tools/info/
Disallow: /tools/ip/
Disallow: /tools/keyword-density-analyzer/

User-Agent: Amazonbot
Disallow: /tools/info/
Disallow: /tools/ip/
Disallow: /tools/keyword-density-analyzer/

User-Agent: Neevabot
Disallow: /tools/

User-agent: Twiceler
Disallow: /tools/

User-agent: Speedy Spider
Disallow: /tools/

User-agent: Accelobot
Disallow: /tools/

User-agent: MJ12bot
Disallow: /tools/
Disallow: /search/

User-agent: AhrefsBot
Disallow: /tools/

User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Yisouspider
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: AboutUsBot 
Disallow: / 

User-agent: panscient.com
Disallow: /

User-agent: Xenu's
Disallow: /

User-agent: Xenu's Link Sleuth 1.1c
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: DBLBot
Disallow: /

User-agent: ScoutJet
Disallow: /

User-agent: netseer 
Disallow: /

User-agent: bixolabs
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: adsbot
Disallow: /

User-Agent: BrandVerity/1.0
Disallow: /

User-agent: Linespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: VelenPublicWebCrawler 
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: LCC
Disallow: /

User-agent: GPTBot
Disallow: /