19 lines
559 B
Plaintext
19 lines
559 B
Plaintext
# This file is mirrored under /robots.txt and /static/robots.txt.
|
|
|
|
User-Agent: *
|
|
|
|
Allow: *
|
|
|
|
# Some /static subdirectories are not crucial to rendering content and should not be crawled by bots.
|
|
Disallow: /static/chat
|
|
Disallow: /static/font
|
|
Disallow: /static/js
|
|
Disallow: /static/syntax
|
|
Disallow: /static/text
|
|
Disallow: /static/wasm
|
|
# /_treehouse contains system pages such as the 404 page.
|
|
Disallow: /_treehouse
|
|
|
|
# In addition to this, some system pages contain a <meta name="robots" content="noindex"> tag to
|
|
# prevent them from appearing in search results.
|