parallelize HTML checking tool

This commit is contained in:
Alex Gaynor 2022-07-29 23:26:00 -04:00
parent 211637d080
commit 5b0ec1ebe4
Failed to extract signature
3 changed files with 27 additions and 21 deletions

View File

@ -1788,6 +1788,7 @@ dependencies = [
name = "html-checker" name = "html-checker"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"rayon",
"walkdir", "walkdir",
] ]

View File

@ -9,3 +9,4 @@ path = "main.rs"
[dependencies] [dependencies]
walkdir = "2" walkdir = "2"
rayon = "1.5"

View File

@ -1,3 +1,4 @@
use rayon::iter::{ParallelBridge, ParallelIterator};
use std::env; use std::env;
use std::path::Path; use std::path::Path;
use std::process::{Command, Output}; use std::process::{Command, Output};
@ -56,27 +57,30 @@ const DOCS_TO_CHECK: &[&str] =
// Returns the number of files read and the number of errors. // Returns the number of files read and the number of errors.
fn find_all_html_files(dir: &Path) -> (usize, usize) { fn find_all_html_files(dir: &Path) -> (usize, usize) {
let mut files_read = 0; walkdir::WalkDir::new(dir)
let mut errors = 0; .into_iter()
.filter_entry(|e| {
for entry in walkdir::WalkDir::new(dir).into_iter().filter_entry(|e| { e.depth() != 1
e.depth() != 1 || e.file_name()
|| e.file_name() .to_str()
.to_str() .map(|s| DOCS_TO_CHECK.into_iter().any(|d| *d == s))
.map(|s| DOCS_TO_CHECK.into_iter().any(|d| *d == s)) .unwrap_or(false)
.unwrap_or(false) })
}) { .par_bridge()
let entry = entry.expect("failed to read file"); .map(|entry| {
if !entry.file_type().is_file() { let entry = entry.expect("failed to read file");
continue; if !entry.file_type().is_file() {
} return (0, 0);
let entry = entry.path(); }
if entry.extension().and_then(|s| s.to_str()) == Some("html") { let entry = entry.path();
errors += check_html_file(&entry); // (Number of files processed, number of errors)
files_read += 1; if entry.extension().and_then(|s| s.to_str()) == Some("html") {
} (1, check_html_file(&entry))
} } else {
(files_read, errors) (0, 0)
}
})
.reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1))
} }
/// Default `tidy` command for macOS is too old that it does not have `mute-id` and `mute` options. /// Default `tidy` command for macOS is too old that it does not have `mute-id` and `mute` options.