From 1e3a1f35271aca67dcbb126a38059f105b1040f8 Mon Sep 17 00:00:00 2001
From: liquidev <liquidev@tutanota.com>
Date: Wed, 27 Nov 2024 19:02:30 +0100
Subject: [PATCH] update robots.txt to only exclude specific subdirectories of
 /static

---
 static/robots.txt | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/static/robots.txt b/static/robots.txt
index 48f397f..c2a37d8 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -4,8 +4,13 @@ User-Agent: *
 
 Allow: *
 
-# /static does not contain any pages.
-Disallow: /static
+# Some /static subdirectories are not crucial to rendering content and should not be crawled by bots.
+Disallow: /static/chat
+Disallow: /static/font
+Disallow: /static/js
+Disallow: /static/syntax
+Disallow: /static/text
+Disallow: /static/wasm
 # /_treehouse contains system pages such as the 404 page.
 Disallow: /_treehouse