User-agent: dotbot
Disallow: /

User-agent: *
Disallow: /*.txt$
Disallow: /*.php$
Disallow: /*.doc$
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.exe$
Disallow: /*.html$
Disallow: /*.swf$
Disallow: /*.fla$
Disallow: /*.rar$
Disallow: /*.mov$
Disallow: /*.mpg$
Disallow: /*.avi$
Disallow: /*.ico$