# # robots.txt # User-agent: msnbot Crawl-delay: 30 # User-agent: Googlebot Crawl-delay: 30 # User-agent: HagenDerRobotMirago Disallow: / User-agent: MagentaNews Disallow: / User-agent: wget Disallow: / User-agent: webzip Disallow: / User-agent: webmirror Disallow: / User-agent: webcopy Disallow: / User-agent: WebReaper Disallow: / # User-agent: * Allow: /landundwelt/ Allow: /region/hallertagblatt/ Disallow: /RealMedia/ Disallow: /cgi-bin/ Disallow: /logs Disallow: /hilfe Disallow: /hilfe/email.php Disallow: /login