# allow our htdig
User-agent: mcdig
Disallow: /intra

# allow intra digger
User-agent: midig
Allow: /intra

# allow google
User-agent: Googlebot
Disallow: /apache
Disallow: /intra
Disallow: /staff
Disallow: /pub
Disallow: /gov_data
Disallow: /docs
Disallow: /cgi-bin

# link checker from w3.org
User-Agent: W3C-checklink
Disallow:

# everyone else - disallow almost everything
# /spcoll is the exception
User-agent: *
Disallow: /archives
Disallow: /cisti
Disallow: /clic
Disallow: /cs
Disallow: /dept
Disallow: /ej
Disallow: /en
Disallow: /erf
Disallow: /et
Disallow: /gateway
Disallow: /gov_data
Disallow: /help
Disallow: /illo
Disallow: /intra
Disallow: /is
Disallow: /ji
Disallow: /ji
Disallow: /lib
Disallow: /micro
Disallow: /ot
Disallow: /status
Disallow: /subj
Disallow: /top
Disallow: /cgi-bin
#Disallow: /spcoll