diff --git a/crates/treehouse/src/cli/serve.rs b/crates/treehouse/src/cli/serve.rs index cf0b7d8..6ac8355 100644 --- a/crates/treehouse/src/cli/serve.rs +++ b/crates/treehouse/src/cli/serve.rs @@ -156,6 +156,7 @@ async fn branch(RawQuery(named_id): RawQuery, State(state): State>) String::from(""); + per_page_metadata.push_str(r#""#); const PER_PAGE_METADATA_REPLACEMENT_STRING: &str = ""; return Html(content.replacen( diff --git a/crates/treehouse/src/generate.rs b/crates/treehouse/src/generate.rs index 614057a..3e9d596 100644 --- a/crates/treehouse/src/generate.rs +++ b/crates/treehouse/src/generate.rs @@ -511,6 +511,10 @@ where pub fn target(dirs: Arc, sources: Arc) -> DynDir { let mut root = MemDir::new(); root.add(VPath::new("static"), dirs.static_.clone()); + root.add( + VPath::new("robots.txt"), + Cd::new(dirs.static_.clone(), VPathBuf::new("robots.txt")).to_dyn(), + ); let dir_index = DirIndex::new(sources.parsed_trees.keys().map(|x| &**x)); let tree_view = TreehouseDir::new(dirs, sources, dir_index); diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 0000000..cb62554 --- /dev/null +++ b/static/robots.txt @@ -0,0 +1,11 @@ +# This file is mirrored under /robots.txt and /static/robots.txt. + +User-Agent: * + +# /static does not contain any pages. +Disallow: /static +# /_treehouse contains system pages such as the 404 page. +Disallow: /_treehouse + +# In addition to this, some system pages contain a tag to +# prevent them from appearing in search results.