# robots.txt file for http://www.pilot-link.net
#
# Please contact [desrod at gnu dash designs dot com] with
# any questions or concerns you have spidering this content

User-agent: Mediapartners-Google*
Disallow: 

User-agent: Googlebot/2.1*
Disallow:

User-agent: msnbot/1.0
Crawl-delay: 300  

User-agent: ia_archiver
Disallow: /

User-agent: Wget
Disallow: /

# Do NOT visit the following pathname or your host will be
# blocked from this site. This is a trap for mal-configured
# bots which do not follow RFCs.
User-agent: *
Disallow: /cgi-bin/block_crawler.pl