# /robots.txt
# exclude all robots from part of the server
User-Agent: *
Disallow: /cgi-bin/

User-Agent: W3C-checklink
Disallow: