User-agent: *
Disallow: /dev
Disallow: /sandbox
Disallow: millercenter.org/index.php
Disallow: millercenter.org/expressionengine.php

# The grub distributed search engine behaves very badly.
User-agent: grub-client
Disallow: /

# They totally overwhelm servers with traffic.
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# Proprietary German backlinks service.
User-agent: SEOkicks-Robot
Disallow: /

# wget run recusively just *wrecks* server capacity.
User-agent: wget
Disallow: /

# This spider's output isn't public.
User-agent: 008
Disallow: /

# Entireweb
User-agent: Speedy
Disallow: /

# Foreign-language bot
User-agent: Sogou web spider
Disallow: /

# Poorly behaved bot
User-agent: Ezooms
Disallow: /

# Brandwatch
User-agent: magpie-crawler
Disallow: /

# Russian image search engine
User-agent: YandexImages
Disallow: /

# Magestic-12
User-agent: MJ12bot
Disallow: /

# Yoydao
User-agent: YodaoBot
Disallow: /

# NerdByNature.Net
User-agent: NerdByNature.Bot
Disallow: /

# Discovery Engine
User-agent: discobot
Disallow: /

# No idea.
User-agent: knowaboutBot
Disallow:/ 

# Gnip
User-agent: UnwindFetchor
Disallow: /

# These bots are designed to duplicate entire sites.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# My Testing Crawler is allowed
User-agent: MattBot
Disallow: