pre-warm content cache

in parallel!
This commit is contained in:
liquidex 2024-11-23 21:42:25 +01:00
parent 42eaa326ab
commit 5193fc2be0
5 changed files with 40 additions and 7 deletions

1
Cargo.lock generated
View file

@ -1951,6 +1951,7 @@ dependencies = [
"jotdown", "jotdown",
"log", "log",
"rand", "rand",
"rayon",
"regex", "regex",
"serde", "serde",
"serde_json", "serde_json",

View file

@ -25,6 +25,7 @@ indexmap = { version = "2.2.6", features = ["serde"] }
jotdown = { version = "0.4.1", default-features = false } jotdown = { version = "0.4.1", default-features = false }
log = { workspace = true } log = { workspace = true }
rand = "0.8.5" rand = "0.8.5"
rayon = "1.10.0"
regex = "1.10.3" regex = "1.10.3"
serde = { version = "1.0.183", features = ["derive"] } serde = { version = "1.0.183", features = ["derive"] }
serde_json = "1.0.105" serde_json = "1.0.105"

View file

@ -514,7 +514,9 @@ pub fn target(dirs: Arc<Dirs>, sources: Arc<Sources>) -> DynDir {
let dir_index = DirIndex::new(sources.parsed_trees.keys().map(|x| &**x)); let dir_index = DirIndex::new(sources.parsed_trees.keys().map(|x| &**x));
let tree_view = TreehouseDir::new(dirs, sources, dir_index); let tree_view = TreehouseDir::new(dirs, sources, dir_index);
let tree_view = ContentCache::new(tree_view); let tree_view = ContentCache::new(tree_view);
tree_view.warm_up();
let tree_view = HtmlCanonicalize::new(tree_view); let tree_view = HtmlCanonicalize::new(tree_view);
Overlay::new(tree_view.to_dyn(), root.to_dyn()).to_dyn() Overlay::new(tree_view.to_dyn(), root.to_dyn()).to_dyn()

View file

@ -1,8 +1,13 @@
use std::fmt::{self, Debug}; use std::{
fmt::{self, Debug},
ops::ControlFlow,
};
use dashmap::DashMap; use dashmap::DashMap;
use log::debug;
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use super::{Dir, DirEntry, EditPath, ImageSize, VPath, VPathBuf}; use super::{walk_dir_rec, Dir, DirEntry, EditPath, ImageSize, VPath, VPathBuf};
pub struct ContentCache<T> { pub struct ContentCache<T> {
inner: T, inner: T,
@ -18,6 +23,22 @@ impl<T> ContentCache<T> {
} }
} }
impl<T> ContentCache<T>
where
T: Dir + Send + Sync,
{
pub fn warm_up(&self) {
debug!("warm_up({self:?})");
let mut paths = vec![];
walk_dir_rec(&self.inner, VPath::ROOT, &mut |path| {
paths.push(path.to_owned());
ControlFlow::Continue(())
});
paths.par_iter().for_each(|path| _ = self.content(path));
}
}
impl<T> Dir for ContentCache<T> impl<T> Dir for ContentCache<T>
where where
T: Dir, T: Dir,

View file

@ -91,10 +91,16 @@ impl VPath {
} }
pub fn segments(&self) -> impl Iterator<Item = &Self> { pub fn segments(&self) -> impl Iterator<Item = &Self> {
self.as_str().split(Self::SEPARATOR).map(|s| unsafe { if self.is_root() {
None.into_iter().flatten()
} else {
Some(self.as_str().split(Self::SEPARATOR).map(|s| unsafe {
// SAFETY: Since we're splitting on the separator, the path cannot start or end with it. // SAFETY: Since we're splitting on the separator, the path cannot start or end with it.
Self::new_unchecked(s) Self::new_unchecked(s)
}) }))
.into_iter()
.flatten()
}
} }
pub fn rsegments(&self) -> impl Iterator<Item = &Self> { pub fn rsegments(&self) -> impl Iterator<Item = &Self> {
@ -203,7 +209,9 @@ impl VPathBuf {
pub fn push(&mut self, sub: &VPath) { pub fn push(&mut self, sub: &VPath) {
if !sub.is_empty() { if !sub.is_empty() {
if !self.is_empty() {
self.path.push('/'); self.path.push('/');
}
self.path.push_str(&sub.path); self.path.push_str(&sub.path);
} }
} }