User-agent: * Disallow: /engine/go.php Disallow: /engine/download.php Disallow: /user/ Disallow: /newposts/ Disallow: /statistics.html Disallow: /*subaction=userinfo Disallow: /*subaction=newposts Disallow: /*do=lastcomments Disallow: /*do=feedback Disallow: /*do=register Disallow: /*do=lostpassword Disallow: /*do=addnews Disallow: /*do=stats Disallow: /*do=pm Disallow: /*do=search User-agent: BLEXBot Crawl-delay: 15 User-agent: SeekportBot Disallow: / User-agent: MJ12bot Crawl-Delay: 15 User-agent: SemrushBot Disallow: / User-agent: dotbot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: GPTBot Disallow: / User-agent: Exabot Disallow: / User-agent: Slurp Disallow: / User-agent: Cliqzbot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: CriteoBot/0.1 Crawl-Delay: 15