2024-11-24 14:28:34 +01:00
|
|
|
# This file is mirrored under /robots.txt and /static/robots.txt.
|
|
|
|
|
|
|
|
|
|
User-Agent: *
|
|
|
|
|
|
2024-11-26 20:55:49 +01:00
|
|
|
Allow: *
|
|
|
|
|
|
2024-11-27 19:02:30 +01:00
|
|
|
# Some /static subdirectories are not crucial to rendering content and should not be crawled by bots.
|
|
|
|
|
Disallow: /static/chat
|
|
|
|
|
Disallow: /static/font
|
|
|
|
|
Disallow: /static/js
|
|
|
|
|
Disallow: /static/syntax
|
|
|
|
|
Disallow: /static/text
|
|
|
|
|
Disallow: /static/wasm
|
2024-11-24 14:28:34 +01:00
|
|
|
# /_treehouse contains system pages such as the 404 page.
|
|
|
|
|
Disallow: /_treehouse
|
|
|
|
|
|
|
|
|
|
# In addition to this, some system pages contain a <meta name="robots" content="noindex"> tag to
|
|
|
|
|
# prevent them from appearing in search results.
|
