diff --git a/static/robots.txt b/static/robots.txt index 48f397f..c2a37d8 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -4,8 +4,13 @@ User-Agent: * Allow: * -# /static does not contain any pages. -Disallow: /static +# Some /static subdirectories are not crucial to rendering content and should not be crawled by bots. +Disallow: /static/chat +Disallow: /static/font +Disallow: /static/js +Disallow: /static/syntax +Disallow: /static/text +Disallow: /static/wasm # /_treehouse contains system pages such as the 404 page. Disallow: /_treehouse