From fd40f99810acfd3508d10953f55b0e4137e80fd7 Mon Sep 17 00:00:00 2001 From: liquidev Date: Wed, 27 Nov 2024 18:46:10 +0100 Subject: [PATCH] split treehouse::generate apart into smaller modules --- crates/treehouse/src/cli/serve.rs | 2 +- crates/treehouse/src/generate.rs | 193 +----------------- crates/treehouse/src/lib.rs | 1 + crates/treehouse/src/main.rs | 3 +- crates/treehouse/src/sources.rs | 139 +++++++++++++ crates/treehouse/src/vfs.rs | 2 + crates/treehouse/src/vfs/html_canonicalize.rs | 56 +++++ 7 files changed, 207 insertions(+), 189 deletions(-) create mode 100644 crates/treehouse/src/sources.rs create mode 100644 crates/treehouse/src/vfs/html_canonicalize.rs diff --git a/crates/treehouse/src/cli/serve.rs b/crates/treehouse/src/cli/serve.rs index 7e365b3..7dff90b 100644 --- a/crates/treehouse/src/cli/serve.rs +++ b/crates/treehouse/src/cli/serve.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use tokio::net::TcpListener; use tracing::{info, instrument}; -use crate::generate::Sources; +use crate::sources::Sources; use crate::vfs::asynch::AsyncDir; use crate::vfs::VPath; use crate::{html::EscapeHtml, state::Source}; diff --git a/crates/treehouse/src/generate.rs b/crates/treehouse/src/generate.rs index 0f87a2e..f2b6ab7 100644 --- a/crates/treehouse/src/generate.rs +++ b/crates/treehouse/src/generate.rs @@ -3,11 +3,10 @@ mod include_static_helper; use std::{collections::HashMap, fmt, ops::ControlFlow, sync::Arc}; -use anyhow::{anyhow, ensure, Context}; +use anyhow::{ensure, Context}; use dir_helper::DirHelper; use handlebars::{handlebars_helper, Handlebars}; use include_static_helper::IncludeStaticHelper; -use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use serde::Serialize; use tracing::{error, info_span, instrument}; @@ -15,19 +14,15 @@ use crate::{ config::Config, dirs::Dirs, fun::seasons::Season, - html::{breadcrumbs::breadcrumbs_to_html, navmap::NavigationMap, tree::branches_to_html}, - import_map::ImportMap, - parse::parse_tree_with_diagnostics, - state::{report_diagnostics, FileId, Source}, - tree::SemaRoots, + html::{breadcrumbs::breadcrumbs_to_html, tree::branches_to_html}, + sources::Sources, + state::FileId, vfs::{ - self, Cd, ContentCache, Dir, DirEntry, DynDir, EditPath, ImageSize, MemDir, Overlay, - ToDynDir, VPath, VPathBuf, + self, Cd, ContentCache, Dir, DirEntry, DynDir, HtmlCanonicalize, MemDir, Overlay, ToDynDir, + VPath, VPathBuf, }, }; -use crate::state::Treehouse; - #[derive(Serialize)] struct Page { title: String, @@ -90,83 +85,6 @@ fn load_templates(handlebars: &mut Handlebars, dir: &dyn Dir) { }); } -#[instrument(skip(config, dirs))] -fn load_trees(config: &Config, dirs: &Dirs) -> anyhow::Result { - let mut treehouse = Treehouse::new(); - let mut diagnostics = vec![]; - let mut parsed_trees = HashMap::new(); - - let mut paths = vec![]; - - vfs::walk_dir_rec(&*dirs.content, VPath::ROOT, &mut |path| { - if path.extension() == Some("tree") { - paths.push(path.to_owned()); - } - ControlFlow::Continue(()) - }); - - // NOTE: Sources are filled in later; they can be left out until a call to report_diagnostics. - let file_ids: Vec<_> = paths - .iter() - .map(|path| treehouse.add_file(path.clone(), Source::Other(String::new()))) - .collect(); - - let parse_results: Vec<_> = { - let _span = info_span!("load_trees::parse").entered(); - paths - .into_par_iter() - .zip(&file_ids) - .flat_map(|(path, &file_id)| { - dirs.content - .content(&path) - .and_then(|b| String::from_utf8(b).ok()) - .map(|input| { - let parse_result = parse_tree_with_diagnostics(file_id, &input); - (path, file_id, input, parse_result) - }) - }) - .collect() - }; - - for (path, file_id, input, _) in &parse_results { - let tree_path = path.with_extension(""); - treehouse - .files_by_tree_path - .insert(tree_path.clone(), *file_id); - treehouse.set_source( - *file_id, - Source::Tree { - input: input.clone(), - tree_path, - }, - ); - } - - { - let _span = info_span!("load_trees::sema").entered(); - for (path, file_id, _, result) in parse_results { - match result { - Ok(roots) => { - let roots = SemaRoots::from_roots( - &mut treehouse, - &mut diagnostics, - config, - file_id, - roots, - ); - treehouse.roots.insert(file_id, roots); - parsed_trees.insert(path, file_id); - } - Err(mut parse_diagnostics) => diagnostics.append(&mut parse_diagnostics), - } - } - } - - report_diagnostics(&treehouse, &diagnostics)?; - - Ok(treehouse) -} - #[instrument(skip(sources, handlebars))] fn generate_simple_template( sources: &Sources, @@ -281,52 +199,6 @@ fn generate_tree_or_error( } } -pub struct Sources { - pub config: Config, - pub treehouse: Treehouse, - pub navigation_map: NavigationMap, - pub import_map: ImportMap, -} - -impl Sources { - pub fn load(dirs: &Dirs) -> anyhow::Result { - let config = { - let _span = info_span!("load_config").entered(); - let mut config: Config = toml_edit::de::from_str( - &dirs - .root - .content(VPath::new("treehouse.toml")) - .map(String::from_utf8) - .ok_or_else(|| anyhow!("config file does not exist"))??, - ) - .context("failed to deserialize config")?; - config.site = std::env::var("TREEHOUSE_SITE").unwrap_or(config.site); - config.autopopulate_emoji(&*dirs.emoji)?; - config.autopopulate_pics(&*dirs.pic)?; - config.load_syntaxes(dirs.syntax.clone())?; - config - }; - - let treehouse = load_trees(&config, dirs)?; - let navigation_map = NavigationMap::build( - &treehouse, - treehouse.files_by_tree_path[VPath::new("index")], - ); - let import_map = ImportMap::generate( - &config.site, - &Cd::new(dirs.static_.clone(), VPathBuf::new("js")), - &config.build.javascript.import_roots, - ); - - Ok(Sources { - config, - treehouse, - navigation_map, - import_map, - }) - } -} - /// Acceleration structure for `dir` operations on [`TreehouseDir`]s. #[derive(Debug, Default)] struct DirIndex { @@ -449,59 +321,6 @@ impl fmt::Debug for TreehouseDir { } } -struct HtmlCanonicalize { - inner: T, -} - -impl HtmlCanonicalize { - pub fn new(inner: T) -> Self { - Self { inner } - } -} - -impl Dir for HtmlCanonicalize -where - T: Dir, -{ - fn dir(&self, path: &VPath) -> Vec { - self.inner.dir(path) - } - - fn content(&self, path: &VPath) -> Option> { - let mut path = path.to_owned(); - if path.extension() == Some("html") { - path.set_extension(""); - } - - self.inner.content(&path) - } - - fn content_version(&self, path: &VPath) -> Option { - self.inner.content_version(path) - } - - fn image_size(&self, path: &VPath) -> Option { - self.inner.image_size(path) - } - - fn anchor(&self, path: &VPath) -> Option { - self.inner.anchor(path) - } - - fn edit_path(&self, path: &VPath) -> Option { - self.inner.edit_path(path) - } -} - -impl fmt::Debug for HtmlCanonicalize -where - T: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "HtmlCanonicalize({:?})", self.inner) - } -} - pub fn target(dirs: Arc, sources: Arc) -> DynDir { let mut root = MemDir::new(); root.add(VPath::new("static"), dirs.static_.clone()); diff --git a/crates/treehouse/src/lib.rs b/crates/treehouse/src/lib.rs index 3f994f1..d901192 100644 --- a/crates/treehouse/src/lib.rs +++ b/crates/treehouse/src/lib.rs @@ -8,6 +8,7 @@ pub mod html; pub mod import_map; pub mod parse; pub mod paths; +pub mod sources; pub mod state; pub mod tree; pub mod vfs; diff --git a/crates/treehouse/src/main.rs b/crates/treehouse/src/main.rs index dd0fc61..4be3df0 100644 --- a/crates/treehouse/src/main.rs +++ b/crates/treehouse/src/main.rs @@ -9,7 +9,8 @@ use tracing_subscriber::layer::SubscriberExt as _; use tracing_subscriber::util::SubscriberInitExt as _; use treehouse::cli::serve::serve; use treehouse::dirs::Dirs; -use treehouse::generate::{self, Sources}; +use treehouse::generate; +use treehouse::sources::Sources; use treehouse::vfs::asynch::AsyncDir; use treehouse::vfs::{ AnchoredAtExt, Blake3ContentVersionCache, DynDir, ImageSizeCache, ToDynDir, VPathBuf, diff --git a/crates/treehouse/src/sources.rs b/crates/treehouse/src/sources.rs new file mode 100644 index 0000000..a60faa5 --- /dev/null +++ b/crates/treehouse/src/sources.rs @@ -0,0 +1,139 @@ +use std::{collections::HashMap, ops::ControlFlow}; + +use anyhow::{anyhow, Context}; +use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; +use tracing::{info_span, instrument}; + +use crate::{ + config::Config, + dirs::Dirs, + html::navmap::NavigationMap, + import_map::ImportMap, + parse::parse_tree_with_diagnostics, + state::{report_diagnostics, Source, Treehouse}, + tree::SemaRoots, + vfs::{self, Cd, VPath, VPathBuf}, +}; + +pub struct Sources { + pub config: Config, + pub treehouse: Treehouse, + pub navigation_map: NavigationMap, + pub import_map: ImportMap, +} + +impl Sources { + pub fn load(dirs: &Dirs) -> anyhow::Result { + let config = { + let _span = info_span!("load_config").entered(); + let mut config: Config = toml_edit::de::from_str( + &dirs + .root + .content(VPath::new("treehouse.toml")) + .map(String::from_utf8) + .ok_or_else(|| anyhow!("config file does not exist"))??, + ) + .context("failed to deserialize config")?; + config.site = std::env::var("TREEHOUSE_SITE").unwrap_or(config.site); + config.autopopulate_emoji(&*dirs.emoji)?; + config.autopopulate_pics(&*dirs.pic)?; + config.load_syntaxes(dirs.syntax.clone())?; + config + }; + + let treehouse = load_trees(&config, dirs)?; + let navigation_map = NavigationMap::build( + &treehouse, + treehouse.files_by_tree_path[VPath::new("index")], + ); + let import_map = ImportMap::generate( + &config.site, + &Cd::new(dirs.static_.clone(), VPathBuf::new("js")), + &config.build.javascript.import_roots, + ); + + Ok(Sources { + config, + treehouse, + navigation_map, + import_map, + }) + } +} + +#[instrument(skip(config, dirs))] +fn load_trees(config: &Config, dirs: &Dirs) -> anyhow::Result { + let mut treehouse = Treehouse::new(); + let mut diagnostics = vec![]; + let mut parsed_trees = HashMap::new(); + + let mut paths = vec![]; + + vfs::walk_dir_rec(&*dirs.content, VPath::ROOT, &mut |path| { + if path.extension() == Some("tree") { + paths.push(path.to_owned()); + } + ControlFlow::Continue(()) + }); + + // NOTE: Sources are filled in later; they can be left out until a call to report_diagnostics. + let file_ids: Vec<_> = paths + .iter() + .map(|path| treehouse.add_file(path.clone(), Source::Other(String::new()))) + .collect(); + + let parse_results: Vec<_> = { + let _span = info_span!("load_trees::parse").entered(); + paths + .into_par_iter() + .zip(&file_ids) + .flat_map(|(path, &file_id)| { + dirs.content + .content(&path) + .and_then(|b| String::from_utf8(b).ok()) + .map(|input| { + let parse_result = parse_tree_with_diagnostics(file_id, &input); + (path, file_id, input, parse_result) + }) + }) + .collect() + }; + + for (path, file_id, input, _) in &parse_results { + let tree_path = path.with_extension(""); + treehouse + .files_by_tree_path + .insert(tree_path.clone(), *file_id); + treehouse.set_source( + *file_id, + Source::Tree { + input: input.clone(), + tree_path, + }, + ); + } + + { + let _span = info_span!("load_trees::sema").entered(); + for (path, file_id, _, result) in parse_results { + match result { + Ok(roots) => { + let roots = SemaRoots::from_roots( + &mut treehouse, + &mut diagnostics, + config, + file_id, + roots, + ); + treehouse.roots.insert(file_id, roots); + parsed_trees.insert(path, file_id); + } + Err(mut parse_diagnostics) => diagnostics.append(&mut parse_diagnostics), + } + } + } + + report_diagnostics(&treehouse, &diagnostics)?; + + Ok(treehouse) +} diff --git a/crates/treehouse/src/vfs.rs b/crates/treehouse/src/vfs.rs index 20015c9..4e2e538 100644 --- a/crates/treehouse/src/vfs.rs +++ b/crates/treehouse/src/vfs.rs @@ -57,6 +57,7 @@ mod content_version_cache; mod edit; mod empty; mod file; +mod html_canonicalize; mod image_size_cache; mod mem_dir; mod overlay; @@ -70,6 +71,7 @@ pub use content_version_cache::*; pub use edit::*; pub use empty::*; pub use file::*; +pub use html_canonicalize::*; pub use image_size_cache::*; pub use mem_dir::*; pub use overlay::*; diff --git a/crates/treehouse/src/vfs/html_canonicalize.rs b/crates/treehouse/src/vfs/html_canonicalize.rs new file mode 100644 index 0000000..109bfe6 --- /dev/null +++ b/crates/treehouse/src/vfs/html_canonicalize.rs @@ -0,0 +1,56 @@ +use core::fmt; + +use super::{Dir, DirEntry, EditPath, ImageSize, VPath, VPathBuf}; + +pub struct HtmlCanonicalize { + inner: T, +} + +impl HtmlCanonicalize { + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +impl Dir for HtmlCanonicalize +where + T: Dir, +{ + fn dir(&self, path: &VPath) -> Vec { + self.inner.dir(path) + } + + fn content(&self, path: &VPath) -> Option> { + let mut path = path.to_owned(); + if path.extension() == Some("html") { + path.set_extension(""); + } + + self.inner.content(&path) + } + + fn content_version(&self, path: &VPath) -> Option { + self.inner.content_version(path) + } + + fn image_size(&self, path: &VPath) -> Option { + self.inner.image_size(path) + } + + fn anchor(&self, path: &VPath) -> Option { + self.inner.anchor(path) + } + + fn edit_path(&self, path: &VPath) -> Option { + self.inner.edit_path(path) + } +} + +impl fmt::Debug for HtmlCanonicalize +where + T: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "HtmlCanonicalize({:?})", self.inner) + } +}