2024-11-24 14:28:34 +01:00
|
|
|
# This file is mirrored under /robots.txt and /static/robots.txt.
|
|
|
|
|
|
|
|
|
|
User-Agent: *
|
|
|
|
|
|
2024-11-26 20:55:49 +01:00
|
|
|
Allow: *
|
|
|
|
|
|
2024-11-24 14:28:34 +01:00
|
|
|
# /static does not contain any pages.
|
|
|
|
|
Disallow: /static
|
|
|
|
|
# /_treehouse contains system pages such as the 404 page.
|
|
|
|
|
Disallow: /_treehouse
|
|
|
|
|
|
|
|
|
|
# In addition to this, some system pages contain a <meta name="robots" content="noindex"> tag to
|
|
|
|
|
# prevent them from appearing in search results.
|
