# $Id: robots.txt,v 1.0 2004/01/23 10:23:21 phil Exp $ # # This is a file retrieved by webwalkers a.k.a. spiders that # conform to a defacto standard. # See # # Comments to the webmaster should be posted at # # Format is: # User-agent: # Disallow: | # Flag Date By Reason # $L1= 20061115 phil updated rules # ----------------------------------------------------------------------------- User-agent: * Disallow: // Disallow: /data/ Disallow: /cgi-bin/ Disallow: /wwwroot/pwlookup.cfm Disallow: /wwwroot/Basket/ Disallow: /wwwroot/users/ User-agent: Fast corporate crawler Disallow: // Disallow: /data/ Disallow: /cgi-bin/ Disallow: /wwwroot/pwlookup.cfm Disallow: /wwwroot/Basket/ Disallow: /wwwroot/users/