User-agent: *
Allow: /

User-agent: msnbot
Disallow: /

user-agent: AhrefsBot
disallow: /

user-agent: SEOkicks-Robot
Disallow: /

User-agent: BLP_bbot/0.1
Disallow: /

User-agent: Twitterbot
Disallow: /

User-agent: Applebot
Disallow: /

User-agent: AppleNewsBot
Disallow: /

User-agent: Go 1.1 package http
Disallow: /

# The grub distributed search engine behaves very badly.
User-agent: grub-client
Disallow: /

# They totally overwhelm servers with traffic.
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /



# This spider's output isn't public.
User-agent: 008
Disallow: /

# Entireweb
User-agent: Speedy
Disallow: /

# Foreign-language bot
User-agent: Sogou web spider
Disallow: /

# Poorly behaved bot
User-agent: Ezooms
Disallow: /

# Brandwatch
User-agent: magpie-crawler
Disallow: /

# Russian image search engine
User-agent: YandexImages
Disallow: /

# Magestic-12
User-agent: MJ12bot
Disallow: /

# Yoydao
User-agent: YodaoBot
Disallow: /

# NerdByNature.Net
User-agent: NerdByNature.Bot
Disallow: /

# Discovery Engine
User-agent: discobot
Disallow: /

# No idea.
User-agent: knowaboutBot
Disallow:/ 

# Gnip
User-agent: UnwindFetchor
Disallow: /

# These bots are designed to duplicate entire sites.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /




User-agent: BLEXBot
Disallow: /



User-agent: bingbot
Crawl-delay: 45

User-agent: CCBot
Crawl-Delay: 60

User-agent: Slurp
Crawl-delay: 60


# Aweful Yahoo garbage bots

User-agent: YahooMobile 
User-agent: YahooCacheSystem 
User-agent: Yahoo! Slurp/Site Explorer 
User-agent: Mozilla/4.05 [en] 
User-agent: LTI/LemurProject 
User-agent: Yahoo-Blogs 
User-agent: Yahoo-Blogs/v3.9 
User-agent: Yahoo-MMCrawler 
User-agent: Yahoo-MMCrawler/3.x 
User-agent: YahooYSMcm 
User-agent: YahooYSMcm/2.0.0 
User-agent: Yahoo-Test 
User-agent: Yahoo! Mindset 
User-agent: Y!J-BSC 
User-agent: Y!J-BSC/1.0 
User-agent: Y!J-BSC/1.0(http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) 
User-agent: y!j-bsc 
User-agent: y!j-bsc/1.0 
User-agent: y!j-bsc/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) 
User-agent: Y!J 
User-agent: Y!J/1.0 
User-agent: Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) 
User-agent: y!j 
User-agent: y!j/1.0 
User-agent: y!j/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) 
User-agent: Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid) 
User-agent: Mozilla/4.0 (compatible; y!j; for robot study; keyoshid) 
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html) 
User-agent: Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp) 
User-agent: Mozilla/5.0 (Yahoo-Test/4.0 mailto:vertical-crawl-support@yahoo-inc.com) 
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
Disallow: /