# # robots.txt # See http://www.robotstxt.org/ for more information # # $Id: robots.txt 56 2009-10-17 01:16:36Z jeremiah $ # # SemrushBot creating a lot of traffic crawling nonexistant # URLs User-agent: SemrushBot Disallow: / # Baiduspider is blocked until they can explain to me why they're # looking for stuff in random capitalization (i.e. /JEreMiaH.CSs). # That is, of course, assuming they follow the /robots.txt standard. User-agent: baiduspider Disallow: / User-agent: ia_archiver Disallow: User-agent: * Disallow: /source Disallow: /wwwstats Disallow: /finger Disallow: /mt Disallow: /cgi-bin/cvsweb.cgi Disallow: /w/ #Begin Attracta SEO Tools Sitemap. Do not remove sitemap: http://cdn.attracta.com/sitemap/2092341.xml.gz #End Attracta SEO Tools Sitemap. Do not remove