split treehouse::generate apart into smaller modules

crates/treehouse/src/cli/serve.rs

@@ -18,7 +18,7 @@ use serde::Deserialize;
 use tokio::net::TcpListener;
 use tracing::{info, instrument};
 
-use crate::generate::Sources;
+use crate::sources::Sources;
 use crate::vfs::asynch::AsyncDir;
 use crate::vfs::VPath;
 use crate::{html::EscapeHtml, state::Source};

crates/treehouse/src/generate.rs

@@ -3,11 +3,10 @@ mod include_static_helper;
 
 use std::{collections::HashMap, fmt, ops::ControlFlow, sync::Arc};
 
-use anyhow::{anyhow, ensure, Context};
+use anyhow::{ensure, Context};
 use dir_helper::DirHelper;
 use handlebars::{handlebars_helper, Handlebars};
 use include_static_helper::IncludeStaticHelper;
-use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
 use serde::Serialize;
 use tracing::{error, info_span, instrument};
 
@@ -15,19 +14,15 @@ use crate::{
     config::Config,
     dirs::Dirs,
     fun::seasons::Season,
-    html::{breadcrumbs::breadcrumbs_to_html, navmap::NavigationMap, tree::branches_to_html},
+    html::{breadcrumbs::breadcrumbs_to_html, tree::branches_to_html},
-    import_map::ImportMap,
+    sources::Sources,
-    parse::parse_tree_with_diagnostics,
+    state::FileId,
-    state::{report_diagnostics, FileId, Source},
-    tree::SemaRoots,
     vfs::{
-        self, Cd, ContentCache, Dir, DirEntry, DynDir, EditPath, ImageSize, MemDir, Overlay,
+        self, Cd, ContentCache, Dir, DirEntry, DynDir, HtmlCanonicalize, MemDir, Overlay, ToDynDir,
-        ToDynDir, VPath, VPathBuf,
+        VPath, VPathBuf,
     },
 };
 
-use crate::state::Treehouse;
-
 #[derive(Serialize)]
 struct Page {
     title: String,
@@ -90,83 +85,6 @@ fn load_templates(handlebars: &mut Handlebars, dir: &dyn Dir) {
     });
 }
 
-#[instrument(skip(config, dirs))]
-fn load_trees(config: &Config, dirs: &Dirs) -> anyhow::Result<Treehouse> {
-    let mut treehouse = Treehouse::new();
-    let mut diagnostics = vec![];
-    let mut parsed_trees = HashMap::new();
-
-    let mut paths = vec![];
-
-    vfs::walk_dir_rec(&*dirs.content, VPath::ROOT, &mut |path| {
-        if path.extension() == Some("tree") {
-            paths.push(path.to_owned());
-        }
-        ControlFlow::Continue(())
-    });
-
-    // NOTE: Sources are filled in later; they can be left out until a call to report_diagnostics.
-    let file_ids: Vec<_> = paths
-        .iter()
-        .map(|path| treehouse.add_file(path.clone(), Source::Other(String::new())))
-        .collect();
-
-    let parse_results: Vec<_> = {
-        let _span = info_span!("load_trees::parse").entered();
-        paths
-            .into_par_iter()
-            .zip(&file_ids)
-            .flat_map(|(path, &file_id)| {
-                dirs.content
-                    .content(&path)
-                    .and_then(|b| String::from_utf8(b).ok())
-                    .map(|input| {
-                        let parse_result = parse_tree_with_diagnostics(file_id, &input);
-                        (path, file_id, input, parse_result)
-                    })
-            })
-            .collect()
-    };
-
-    for (path, file_id, input, _) in &parse_results {
-        let tree_path = path.with_extension("");
-        treehouse
-            .files_by_tree_path
-            .insert(tree_path.clone(), *file_id);
-        treehouse.set_source(
-            *file_id,
-            Source::Tree {
-                input: input.clone(),
-                tree_path,
-            },
-        );
-    }
-
-    {
-        let _span = info_span!("load_trees::sema").entered();
-        for (path, file_id, _, result) in parse_results {
-            match result {
-                Ok(roots) => {
-                    let roots = SemaRoots::from_roots(
-                        &mut treehouse,
-                        &mut diagnostics,
-                        config,
-                        file_id,
-                        roots,
-                    );
-                    treehouse.roots.insert(file_id, roots);
-                    parsed_trees.insert(path, file_id);
-                }
-                Err(mut parse_diagnostics) => diagnostics.append(&mut parse_diagnostics),
-            }
-        }
-    }
-
-    report_diagnostics(&treehouse, &diagnostics)?;
-
-    Ok(treehouse)
-}
-
 #[instrument(skip(sources, handlebars))]
 fn generate_simple_template(
     sources: &Sources,
@@ -281,52 +199,6 @@ fn generate_tree_or_error(
     }
 }
 
-pub struct Sources {
-    pub config: Config,
-    pub treehouse: Treehouse,
-    pub navigation_map: NavigationMap,
-    pub import_map: ImportMap,
-}
-
-impl Sources {
-    pub fn load(dirs: &Dirs) -> anyhow::Result<Self> {
-        let config = {
-            let _span = info_span!("load_config").entered();
-            let mut config: Config = toml_edit::de::from_str(
-                &dirs
-                    .root
-                    .content(VPath::new("treehouse.toml"))
-                    .map(String::from_utf8)
-                    .ok_or_else(|| anyhow!("config file does not exist"))??,
-            )
-            .context("failed to deserialize config")?;
-            config.site = std::env::var("TREEHOUSE_SITE").unwrap_or(config.site);
-            config.autopopulate_emoji(&*dirs.emoji)?;
-            config.autopopulate_pics(&*dirs.pic)?;
-            config.load_syntaxes(dirs.syntax.clone())?;
-            config
-        };
-
-        let treehouse = load_trees(&config, dirs)?;
-        let navigation_map = NavigationMap::build(
-            &treehouse,
-            treehouse.files_by_tree_path[VPath::new("index")],
-        );
-        let import_map = ImportMap::generate(
-            &config.site,
-            &Cd::new(dirs.static_.clone(), VPathBuf::new("js")),
-            &config.build.javascript.import_roots,
-        );
-
-        Ok(Sources {
-            config,
-            treehouse,
-            navigation_map,
-            import_map,
-        })
-    }
-}
-
 /// Acceleration structure for `dir` operations on [`TreehouseDir`]s.
 #[derive(Debug, Default)]
 struct DirIndex {
@@ -449,59 +321,6 @@ impl fmt::Debug for TreehouseDir {
     }
 }
 
-struct HtmlCanonicalize<T> {
-    inner: T,
-}
-
-impl<T> HtmlCanonicalize<T> {
-    pub fn new(inner: T) -> Self {
-        Self { inner }
-    }
-}
-
-impl<T> Dir for HtmlCanonicalize<T>
-where
-    T: Dir,
-{
-    fn dir(&self, path: &VPath) -> Vec<DirEntry> {
-        self.inner.dir(path)
-    }
-
-    fn content(&self, path: &VPath) -> Option<Vec<u8>> {
-        let mut path = path.to_owned();
-        if path.extension() == Some("html") {
-            path.set_extension("");
-        }
-
-        self.inner.content(&path)
-    }
-
-    fn content_version(&self, path: &VPath) -> Option<String> {
-        self.inner.content_version(path)
-    }
-
-    fn image_size(&self, path: &VPath) -> Option<ImageSize> {
-        self.inner.image_size(path)
-    }
-
-    fn anchor(&self, path: &VPath) -> Option<VPathBuf> {
-        self.inner.anchor(path)
-    }
-
-    fn edit_path(&self, path: &VPath) -> Option<EditPath> {
-        self.inner.edit_path(path)
-    }
-}
-
-impl<T> fmt::Debug for HtmlCanonicalize<T>
-where
-    T: fmt::Debug,
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "HtmlCanonicalize({:?})", self.inner)
-    }
-}
-
 pub fn target(dirs: Arc<Dirs>, sources: Arc<Sources>) -> DynDir {
     let mut root = MemDir::new();
     root.add(VPath::new("static"), dirs.static_.clone());

crates/treehouse/src/lib.rs

@@ -8,6 +8,7 @@ pub mod html;
 pub mod import_map;
 pub mod parse;
 pub mod paths;
+pub mod sources;
 pub mod state;
 pub mod tree;
 pub mod vfs;

crates/treehouse/src/main.rs

@@ -9,7 +9,8 @@ use tracing_subscriber::layer::SubscriberExt as _;
 use tracing_subscriber::util::SubscriberInitExt as _;
 use treehouse::cli::serve::serve;
 use treehouse::dirs::Dirs;
-use treehouse::generate::{self, Sources};
+use treehouse::generate;
+use treehouse::sources::Sources;
 use treehouse::vfs::asynch::AsyncDir;
 use treehouse::vfs::{
     AnchoredAtExt, Blake3ContentVersionCache, DynDir, ImageSizeCache, ToDynDir, VPathBuf,

crates/treehouse/src/sources.rs

@@ -0,0 +1,139 @@
+use std::{collections::HashMap, ops::ControlFlow};
+
+use anyhow::{anyhow, Context};
+use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
+use tracing::{info_span, instrument};
+
+use crate::{
+    config::Config,
+    dirs::Dirs,
+    html::navmap::NavigationMap,
+    import_map::ImportMap,
+    parse::parse_tree_with_diagnostics,
+    state::{report_diagnostics, Source, Treehouse},
+    tree::SemaRoots,
+    vfs::{self, Cd, VPath, VPathBuf},
+};
+
+pub struct Sources {
+    pub config: Config,
+    pub treehouse: Treehouse,
+    pub navigation_map: NavigationMap,
+    pub import_map: ImportMap,
+}
+
+impl Sources {
+    pub fn load(dirs: &Dirs) -> anyhow::Result<Self> {
+        let config = {
+            let _span = info_span!("load_config").entered();
+            let mut config: Config = toml_edit::de::from_str(
+                &dirs
+                    .root
+                    .content(VPath::new("treehouse.toml"))
+                    .map(String::from_utf8)
+                    .ok_or_else(|| anyhow!("config file does not exist"))??,
+            )
+            .context("failed to deserialize config")?;
+            config.site = std::env::var("TREEHOUSE_SITE").unwrap_or(config.site);
+            config.autopopulate_emoji(&*dirs.emoji)?;
+            config.autopopulate_pics(&*dirs.pic)?;
+            config.load_syntaxes(dirs.syntax.clone())?;
+            config
+        };
+
+        let treehouse = load_trees(&config, dirs)?;
+        let navigation_map = NavigationMap::build(
+            &treehouse,
+            treehouse.files_by_tree_path[VPath::new("index")],
+        );
+        let import_map = ImportMap::generate(
+            &config.site,
+            &Cd::new(dirs.static_.clone(), VPathBuf::new("js")),
+            &config.build.javascript.import_roots,
+        );
+
+        Ok(Sources {
+            config,
+            treehouse,
+            navigation_map,
+            import_map,
+        })
+    }
+}
+
+#[instrument(skip(config, dirs))]
+fn load_trees(config: &Config, dirs: &Dirs) -> anyhow::Result<Treehouse> {
+    let mut treehouse = Treehouse::new();
+    let mut diagnostics = vec![];
+    let mut parsed_trees = HashMap::new();
+
+    let mut paths = vec![];
+
+    vfs::walk_dir_rec(&*dirs.content, VPath::ROOT, &mut |path| {
+        if path.extension() == Some("tree") {
+            paths.push(path.to_owned());
+        }
+        ControlFlow::Continue(())
+    });
+
+    // NOTE: Sources are filled in later; they can be left out until a call to report_diagnostics.
+    let file_ids: Vec<_> = paths
+        .iter()
+        .map(|path| treehouse.add_file(path.clone(), Source::Other(String::new())))
+        .collect();
+
+    let parse_results: Vec<_> = {
+        let _span = info_span!("load_trees::parse").entered();
+        paths
+            .into_par_iter()
+            .zip(&file_ids)
+            .flat_map(|(path, &file_id)| {
+                dirs.content
+                    .content(&path)
+                    .and_then(|b| String::from_utf8(b).ok())
+                    .map(|input| {
+                        let parse_result = parse_tree_with_diagnostics(file_id, &input);
+                        (path, file_id, input, parse_result)
+                    })
+            })
+            .collect()
+    };
+
+    for (path, file_id, input, _) in &parse_results {
+        let tree_path = path.with_extension("");
+        treehouse
+            .files_by_tree_path
+            .insert(tree_path.clone(), *file_id);
+        treehouse.set_source(
+            *file_id,
+            Source::Tree {
+                input: input.clone(),
+                tree_path,
+            },
+        );
+    }
+
+    {
+        let _span = info_span!("load_trees::sema").entered();
+        for (path, file_id, _, result) in parse_results {
+            match result {
+                Ok(roots) => {
+                    let roots = SemaRoots::from_roots(
+                        &mut treehouse,
+                        &mut diagnostics,
+                        config,
+                        file_id,
+                        roots,
+                    );
+                    treehouse.roots.insert(file_id, roots);
+                    parsed_trees.insert(path, file_id);
+                }
+                Err(mut parse_diagnostics) => diagnostics.append(&mut parse_diagnostics),
+            }
+        }
+    }
+
+    report_diagnostics(&treehouse, &diagnostics)?;
+
+    Ok(treehouse)
+}

crates/treehouse/src/vfs.rs

@@ -57,6 +57,7 @@ mod content_version_cache;
 mod edit;
 mod empty;
 mod file;
+mod html_canonicalize;
 mod image_size_cache;
 mod mem_dir;
 mod overlay;
@@ -70,6 +71,7 @@ pub use content_version_cache::*;
 pub use edit::*;
 pub use empty::*;
 pub use file::*;
+pub use html_canonicalize::*;
 pub use image_size_cache::*;
 pub use mem_dir::*;
 pub use overlay::*;

crates/treehouse/src/vfs/html_canonicalize.rs

@@ -0,0 +1,56 @@
+use core::fmt;
+
+use super::{Dir, DirEntry, EditPath, ImageSize, VPath, VPathBuf};
+
+pub struct HtmlCanonicalize<T> {
+    inner: T,
+}
+
+impl<T> HtmlCanonicalize<T> {
+    pub fn new(inner: T) -> Self {
+        Self { inner }
+    }
+}
+
+impl<T> Dir for HtmlCanonicalize<T>
+where
+    T: Dir,
+{
+    fn dir(&self, path: &VPath) -> Vec<DirEntry> {
+        self.inner.dir(path)
+    }
+
+    fn content(&self, path: &VPath) -> Option<Vec<u8>> {
+        let mut path = path.to_owned();
+        if path.extension() == Some("html") {
+            path.set_extension("");
+        }
+
+        self.inner.content(&path)
+    }
+
+    fn content_version(&self, path: &VPath) -> Option<String> {
+        self.inner.content_version(path)
+    }
+
+    fn image_size(&self, path: &VPath) -> Option<ImageSize> {
+        self.inner.image_size(path)
+    }
+
+    fn anchor(&self, path: &VPath) -> Option<VPathBuf> {
+        self.inner.anchor(path)
+    }
+
+    fn edit_path(&self, path: &VPath) -> Option<EditPath> {
+        self.inner.edit_path(path)
+    }
+}
+
+impl<T> fmt::Debug for HtmlCanonicalize<T>
+where
+    T: fmt::Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "HtmlCanonicalize({:?})", self.inner)
+    }
+}