# Allow all user agents # --------------------- User-agent: * # List of auto generated sitemaps # ------------------------------- Sitemap: http://openbook.ie/sitemaps/openbook_ie_com_sitemap.xml Sitemap: http://ioe.johnsmith.co.uk/sitemaps/ioe_johnsmith_co_uk_sitemap.xml Sitemap: http://barcouncil_hammickslegal_com/sitemaps/barcouncil_hammickslegal_com_sitemap.xml # Start by allowing the live directory # ------------------------------------ Allow: /live/ # There's generally nothing worth indexing on these pages and most are protected / inaccessible # --------------------------------------------------------------------------------------------- Disallow: /live/Basket.asp Disallow: /live/Confirm_Order_Details.asp Disallow: /live/Error.asp Disallow: /live/Forgotten_Password.asp Disallow: /live/Logout.asp Disallow: /live/My_Details.asp Disallow: /live/My_Orders.asp Disallow: /live/Order_Confirmation.asp Disallow: /live/Payment_Details.asp Disallow: /live/Results.asp Disallow: /live/Three_Dee_Secure.asp # This is used to catch bad robots not obeying the robots.txt standard Disallow: /live/Robots_Trap.asp # Mozilla/5.0 (compatible; DotBot/1.1; http://www.dotnetdotcom.org/, crawler@dotnetdotcom.org) User-agent: dotbot Disallow: /