################################### # Detected Spider/Bot: None # # Headers Sent: # Content-Type: text/plain # Sitemap: http://www.degreesofzero.com/index.php?zc=xml;get=sitemap # Google - Most Important bot # Unfortunately a robots.txt will only stop it crawling certain urls, and NOT adding any # urls which it comes across into its index. So we're relying on a meta noindex tag. User-agent: Googlebot # Don't index mobile versions Disallow: /index.php?*;wap Disallow: /index.php?*;wap2 Disallow: /index.php?*;imode # Default Folders Disallow: /Languages/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help # Now allow bits and then disallow bits Allow: /robots.txt$ Allow: /index.php$ # Bad bot - Often ignores robots.txt - Waste of bandwidth # Despite claiming on their website to be a search engine in development # I'm suspicious as to whether they are a harvester pretending to be SE User-agent: Twiceler Disallow: / User-Agent: W3C-checklink Disallow: / # Stop following PHPSESSID's User-Agent: MJ12bot Disallow: /index.php?PHPSESSID # Default Folders Disallow: /Languages/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help # Catch all (remainder) # Will be followed by any bots other than ones identified above # Uses BASIC robots.txt directives without wildcards, end-anchors etc # So Spiders should understand these (including MSNBOT) User-agent: * # Default Folders Disallow: /Languages/ Disallow: /lib/ Disallow: /Plugins/ Disallow: /Sources/ Disallow: /Themes/ Disallow: /cache/ # Default Actions Disallow: /index.php?zc=cp Disallow: /index.php?zc=login Disallow: /index.php?zc=logout Disallow: /index.php?zc=register Disallow: /index.php?zc=help