# www.laneneave.co.nz User-agent: Baiduspider Disallow: / User-agent: BecomeBot Disallow: / # Charlotte/1.0b; 20060525 209.249.86.4 User-agent: Charlotte Disallow: / User-agent: ConveraCrawler Disallow: / User-agent: ConveraMultiMediaCrawler Disallow: / User-agent: crawl Disallow: / User-agent: FAST Disallow: / User-agent: FAST Enterprise Crawler Disallow: / User-agent: FAST Enterprise Crawler 6 Disallow: / User-agent: FAST MetaWeb Crawler Disallow: / User-agent: favorstarbot/1.0 Disallow: / User-agent: FrontPage Disallow: / User-agent: Funnelback Disallow: / User-agent: Gaisbot Disallow: / User-agent: Geona Disallow: / User-agent: GigaBot Disallow: / User-agent: heritrix Disallow: / User-agent: HTTrack Disallow: / User-agent: HuriSearchBot Disallow: / User-agent: ia_archiver Disallow: # 20060525 210.173.180.16 # 20070527 210.150.10.109 User-agent: ichiro/2.0 Disallow: / User-agent: IRLbot Disallow: / User-agent: Java Disallow: / User-agent: Java/1.4.2_05 Disallow: / User-agent: Java/1.5.0_04 Disallow: / User-agent: Java/1.6.0-ea Disallow: / # 20050525 209.167.50.22 User-agent: LinkWalker Disallow: / User-agent: MJ12bot Disallow: / #Doesnt seem to be a pain anymore #User-Agent: MSIECrawler #Disallow:/ # Too greedy at present, downloads same links upto 80 times #User-Agent: msnbot-media #Disallow:/ User-agent: Nutch Disallow: / User-Agent: OmniExplorer_Bot/1.09 Disallow: / User-agent: panscient.com Disallow: / User-agent: picmole Disallow: / User-agent: polybot Disallow: / User-agent: Pompos Disallow: / User-agent: psbot Disallow: / User-agent: Scirus Disallow: / User-agent: Qihoobot Disallow: / User-agent: Shim-Crawler Disallow: / User-agent: Snapbot/1.0 Disallow: / User-agent: sogou Disallow: / User-agent: sogou spider Disallow: / #User-agent: Slurp #Disallow: / # User-agent: Teleport Disallow: / User-agent: Teoma Disallow: / User-agent: Twiceler0.9 Disallow: / User-agent: Twiceler-0.9 Disallow: / User-agent: W3Crobot Disallow: / User-agent: WebCopier Disallow: / User-agent: WebStripper Disallow: / User-agent: Webupd Disallow: / User-agent: www.webwombat.com.au Disallow: / User-agent: webwombat Disallow: / User-agent: Webwombat Disallow: / User-agent: WebZIP Disallow: / User-agent: WISENutbot Disallow: / User-agent: YodaoBot/1.0 Disallow: / User-agent: Yahoo-MMCrawler Disallow: / User-agent: zibber-v0.1(www.zibb.com/crawler/) Disallow: / User-agent: ZyBorg Disallow: /