# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent:webcrawler
Disallow:/oldsite
Disallow:/mail
Disallow:/00.FileTransfers
Disallow:/aspnet_client
Disallow:/effect_XXV_375.swf
Disallow:/cgi-bin
Disallow:/form33

User-agent:*
Disallow:/tmp
Disallow:/logs
Disallow:/oldsite
Disallow:/mail
Disallow:/00.FileTransfers
Disallow:/aspnet_client
Disallow:/effect_XXV_375.swf
Disallow:/cgi-bin
Disallow:/form33