robots.txt

# robots.txt for OU Media Player
# Mostly don't publicise - except the home/about pages.
# (See http://youtube.com/robots.txt)
# http://archive.org/details/archive.org_bot
# https://webarchive.jira.com/wiki/display/ARIH/Robots+Exclusion+Protocol
#
#Disallow: /demo

User-agent: archive.org_bot
Disallow:

User-agent: *
Disallow: /assets/
Disallow: /blog/
Disallow: /dev/
Disallow: /docs/
Disallow: /drupal/
Disallow: /build
Disallow: /test
Disallow: /embed
Disallow: /localize
Disallow: /oembed
Disallow: /popup
Disallow: /scripts
Disallow: /services
Disallow: /svg
Disallow: /timedtext
Disallow: /uptime
Disallow: /2012