# $Id: robots.txt,v 1.2 1996/06/26 04:15:40 W.Edmunds Exp $ # robots.txt for http://www.ex.ac.uk/ # see for an explanation. User-agent: htdig # ht://Dig should avoid Disallow: /cgi-bin/ # Script files Disallow: /errordocs/ # Local Error docs Disallow: /spool/ # Local MOMspider output Disallow: /web/ # Local Web Tools Disallow: /config/ # Local Web Config Disallow: /cgi-src/ # Local CGI Source Disallow: /brad/noindex/ # Brads noindex text stuff Disallow: /~BBagilho/noindex/ # Brads noindex text stuff Disallow: /alumni/ # Local alumni Disallow: /alumnidb/ # Local alumni Disallow: /iais/pgc/ # Causes looping Disallow: /its/mwr/ # Web Stats Disallow: /its/meetingmaker/ # MeetingMaker Disallow: /its/software/cd/localstuff/ # ITS CD info Disallow: /its/email2003/ # email2003 Disallow: /tusage/ # Web Stats Disallow: /library/ # Library web pages have moved Disallow: /wusage/ # Web Stats Disallow: /web/ # Common Manuals, etc #Disallow: /admin/personnel/ # Personnel files cause looping process Disallow: /projects/admin/ # Personnel files Disallow: /Projects/admin/ # Personnel files User-agent: * # All other spiders should avoid Disallow: /cgi-bin/ # Script files Disallow: /errordocs/ # Local Error docs Disallow: /spool/ # Local MOMspider output Disallow: /web/ # Local Web Tools Disallow: /config/ # Local Web Config Disallow: /cgi-src/ # Local CGI Source Disallow: /noindex/ # noindex stuff Disallow: /no-index/ # noindex stuff Disallow: /alumni/ # Local alumni Disallow: /alumnidb/ # Local alumni Disallow: /alumnisupporters/contactus/updateyourdetails/ # Local alumni Disallow: /its/mwr/ # Web Stats Disallow: /its/meetingmaker/ # MeetingMaker Disallow: /its/software/cd/localstuff/ # ITS CD info Disallow: /its/email2003/ # email2003 Disallow: /its/ipc2003/ # ipc2003 Disallow: /tusage/ # Web Stats Disallow: /wusage/ # Web Stats Disallow: /web/ # Common Manuals, etc Disallow: /proxy/ # Local proxy info Disallow: /admin/personnel/ # Personnel files cause looping process Disallow: /projects/admin/ # Personnel files Disallow: /Projects/admin/ # Personnel files Disallow: /about/governance/business_areas/fundraising/index.shtml Disallow: /departments/communication/marketing/aboutus/ Disallow: /undergraduate/2004/ # Archived files Disallow: /undergraduate/2005/ # Archived files Disallow: /undergraduate/2006/ # Archived files Disallow: /undergraduate/2007/ # Archived files Disallow: /undergraduate/2008/ # Archived files Disallow: /undergraduate/2009/ # Archived files Disallow: /undergraduate/2010/ # Archived files Disallow: /postgraduate/pgp2004/ # Archived files Disallow: /postgraduate/pgp2005/ # Archived files Disallow: /postgraduate/pgp2006/ # Archived files Disallow: /postgraduate/pgp2007/ # Archived files Disallow: /postgraduate/pgp2008/ # Archived files Disallow: /postgraduate/pgp2009/ # Archived files Disallow: /postgraduate/pgp2010/ # Archived files Disallow: /codebox/ # Archived files Disallow: /postgraduate/2011/ # code not intended to be indexed directly