From 41fa245150d2cf29339ee3b41f15dd7d3a801669 Mon Sep 17 00:00:00 2001
From: liquidev <liquidev@tutanota.com>
Date: Sun, 24 Nov 2024 14:28:34 +0100
Subject: [PATCH] rudimentary search engine optimisation (robots.txt, <meta
 name="robots"> in /b)

---
 crates/treehouse/src/cli/serve.rs |  1 +
 crates/treehouse/src/generate.rs  |  4 ++++
 static/robots.txt                 | 11 +++++++++++
 3 files changed, 16 insertions(+)
 create mode 100644 static/robots.txt

diff --git a/crates/treehouse/src/cli/serve.rs b/crates/treehouse/src/cli/serve.rs
index cf0b7d8..6ac8355 100644
--- a/crates/treehouse/src/cli/serve.rs
+++ b/crates/treehouse/src/cli/serve.rs
@@ -156,6 +156,7 @@ async fn branch(RawQuery(named_id): RawQuery, State(state): State<Arc<Server>>)
                         String::from("<meta property=\"og:description\" content=\"");
                     write!(per_page_metadata, "{}", EscapeHtml(branch_markup)).unwrap();
                     per_page_metadata.push_str("\">");
+                    per_page_metadata.push_str(r#"<meta name="robots" content="noindex">"#);
 
                     const PER_PAGE_METADATA_REPLACEMENT_STRING: &str = "<!-- treehouse-ca37057a-cff5-45b3-8415-3b02dbf6c799-per-branch-metadata -->";
                     return Html(content.replacen(
diff --git a/crates/treehouse/src/generate.rs b/crates/treehouse/src/generate.rs
index 614057a..3e9d596 100644
--- a/crates/treehouse/src/generate.rs
+++ b/crates/treehouse/src/generate.rs
@@ -511,6 +511,10 @@ where
 pub fn target(dirs: Arc<Dirs>, sources: Arc<Sources>) -> DynDir {
     let mut root = MemDir::new();
     root.add(VPath::new("static"), dirs.static_.clone());
+    root.add(
+        VPath::new("robots.txt"),
+        Cd::new(dirs.static_.clone(), VPathBuf::new("robots.txt")).to_dyn(),
+    );
 
     let dir_index = DirIndex::new(sources.parsed_trees.keys().map(|x| &**x));
     let tree_view = TreehouseDir::new(dirs, sources, dir_index);
diff --git a/static/robots.txt b/static/robots.txt
new file mode 100644
index 0000000..cb62554
--- /dev/null
+++ b/static/robots.txt
@@ -0,0 +1,11 @@
+# This file is mirrored under /robots.txt and /static/robots.txt.
+
+User-Agent: *
+
+# /static does not contain any pages.
+Disallow: /static
+# /_treehouse contains system pages such as the 404 page.
+Disallow: /_treehouse
+
+# In addition to this, some system pages contain a <meta name="robots" content="noindex"> tag to
+# prevent them from appearing in search results.