#
# robots.txt
#
# version 1.0
#

# exclude
User-Agent: *
Disallow: /js/
Disallow: /w3c/
Disallow: /xml/
Disallow: /css/
Disallow: /images/
Disallow: /error.asp
Disallow: /email.asp
Disallow: /external-link.asp
Disallow: /forgot_password.asp
Disallow: /forum-signature.asp
Disallow: /login.asp
Disallow: /logout.asp
Disallow: /profile.asp
Disallow: /confirm-email.asp
Disallow: /confirm-email-resend.asp
Disallow: /set-blocking-cookie.asp

User-agent: Googlebot-Image
Disallow: /

User-agent: DIIbot 
Disallow: / 

User-agent: LinkWalker 
Disallow: / 

User-agent: NPBot
Disallow: / 

User-agent: BaySpider 
Disallow: / 

# Genealogy SE http://www.GenDoor.com/crawler.html 
User-agent: GenCrawler 
Disallow: / 

# http://www.gozilla.com/ 
User-agent: Go!Zilla 
Disallow: / 

# Bad Links 
User-agent: Linkidator 
Disallow: / 

# Bad Links http://www.elsop.com/linkscan/overview.html 
User-agent: LinkScan 
Disallow: / 

# Link Checking http://www.linkguard.com/ 
User-agent: LinkGuard 
Disallow: / 

User-agent: TurnitinBot
Disallow: /