mod checkers;
mod explicit;
pub(crate) mod generators;
mod creators;
pub mod lua;
use anyhow::{Context, Result};
use serde::Serialize;
#[cfg(debug_assertions)]
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::time::Duration;
use crate::color;
use crate::config::{
output_config_hash, resolve_extra_inputs,
CheckerConfigWithCommand, SimpleCheckerParams, StandardConfig,
};
use crate::file_index::FileIndex;
use crate::graph::{BuildGraph, Product};
pub mod names {
pub const TERA: &str = "tera";
pub const CC_SINGLE_FILE: &str = "cc_single_file";
pub const CC: &str = "cc";
pub const ZSPELL: &str = "zspell";
pub const MAKE: &str = "make";
pub const CARGO: &str = "cargo";
pub const CLIPPY: &str = "clippy";
pub const TAGS: &str = "tags";
pub const PIP: &str = "pip";
pub const SPHINX: &str = "sphinx";
pub const MDBOOK: &str = "mdbook";
pub const NPM: &str = "npm";
pub const GEM: &str = "gem";
pub const MDL: &str = "mdl";
pub const MARKDOWNLINT: &str = "markdownlint";
pub const ASPELL: &str = "aspell";
pub const PDFLATEX: &str = "pdflatex";
pub const MAKO: &str = "mako";
pub const JINJA2: &str = "jinja2";
pub const PDFUNITE: &str = "pdfunite";
pub const IPDFUNITE: &str = "ipdfunite";
pub const SCRIPT: &str = "script";
pub const GENERATOR: &str = "generator";
pub const EXPLICIT: &str = "explicit";
pub const LINUX_MODULE: &str = "linux_module";
pub const JEKYLL: &str = "jekyll";
pub const RUST_SINGLE_FILE: &str = "rust_single_file";
}
pub(crate) fn resolve_anchor_path(anchor_dir: &Path, rel: &str) -> PathBuf {
if anchor_dir.as_os_str().is_empty() {
PathBuf::from(rel)
} else {
anchor_dir.join(rel)
}
}
#[cfg(debug_assertions)]
thread_local! {
static DECLARED_TOOLS: RefCell<Option<Vec<String>>> = const { RefCell::new(None) };
}
#[cfg(debug_assertions)]
pub(crate) fn set_declared_tools(tools: Option<Vec<String>>) {
DECLARED_TOOLS.with(|dt| {
*dt.borrow_mut() = tools;
});
}
#[cfg(not(debug_assertions))]
pub(crate) fn set_declared_tools(_tools: Option<Vec<String>>) {}
#[cfg(debug_assertions)]
pub(crate) fn suspend_tool_check() -> ToolCheckGuard {
let prev = DECLARED_TOOLS.with(|dt| dt.borrow_mut().take());
ToolCheckGuard { prev }
}
#[cfg(not(debug_assertions))]
pub(crate) fn suspend_tool_check() -> ToolCheckGuard {
ToolCheckGuard { _private: () }
}
#[cfg(debug_assertions)]
pub(crate) struct ToolCheckGuard {
prev: Option<Vec<String>>,
}
#[cfg(debug_assertions)]
impl Drop for ToolCheckGuard {
fn drop(&mut self) {
DECLARED_TOOLS.with(|dt| {
*dt.borrow_mut() = self.prev.take();
});
}
}
#[cfg(not(debug_assertions))]
pub(crate) struct ToolCheckGuard {
_private: (),
}
pub(crate) fn format_command(cmd: &Command) -> String {
let program = cmd.get_program().to_string_lossy();
let args: Vec<_> = cmd.get_args()
.map(|a| a.to_string_lossy())
.collect();
if args.is_empty() {
program.into_owned()
} else {
format!("{} {}", program, args.join(" "))
}
}
pub(crate) fn log_command(cmd: &Command) {
if crate::runtime_flags::show_child_processes() {
let cwd = cmd.get_current_dir()
.map(|p| p.display().to_string())
.unwrap_or_default();
if cwd.is_empty() {
eprintln!("{} {}", color::dim("[exec]"), format_command(cmd));
} else {
let cwd_info = format!("(in {})", cwd);
eprintln!("{} {} {}", color::dim("[exec]"), format_command(cmd), color::dim(&cwd_info));
}
}
}
fn run_command_inner(ctx: &crate::build_context::BuildContext, cmd: &mut Command, inherit_stdio: bool) -> Result<Output> {
log_command(cmd);
#[cfg(debug_assertions)]
DECLARED_TOOLS.with(|dt| {
if let Some(ref tools) = *dt.borrow() {
let program = cmd.get_program().to_string_lossy();
let basename = program.rsplit('/').next().unwrap_or(&program);
assert!(
tools.iter().any(|t| {
let t_basename = t.rsplit('/').next().unwrap_or(t);
t_basename == basename
}),
"Processor executed undeclared tool '{basename}'. Declared: {tools:?}",
);
}
});
if ctx.is_interrupted() {
anyhow::bail!("Interrupted");
}
let program = cmd.get_program().to_os_string();
let args: Vec<_> = cmd.get_args().map(|a| a.to_os_string()).collect();
let current_dir = cmd.get_current_dir().map(|p| p.to_path_buf());
let envs: Vec<_> = cmd.get_envs()
.filter_map(|(k, v)| v.map(|val| (k.to_os_string(), val.to_os_string())))
.collect();
ctx.runtime().block_on(async {
let mut tokio_cmd = tokio::process::Command::new(&program);
tokio_cmd.args(&args);
if let Some(dir) = ¤t_dir {
tokio_cmd.current_dir(dir);
}
for (key, val) in &envs {
tokio_cmd.env(key, val);
}
if inherit_stdio {
tokio_cmd.stdout(std::process::Stdio::inherit());
tokio_cmd.stderr(std::process::Stdio::inherit());
} else {
tokio_cmd.stdout(std::process::Stdio::piped());
tokio_cmd.stderr(std::process::Stdio::piped());
}
tokio_cmd.kill_on_drop(true);
let child = tokio_cmd.spawn()
.with_context(|| {
let prog = program.to_string_lossy();
let total_len: usize = prog.len() + args.iter().map(|a| a.len() + 1).sum::<usize>();
let arg_count = args.len();
if total_len > 100_000 {
format!(
"Failed to spawn '{}' with {} arguments (total command length ~{} bytes). \
This usually means the argument list is too long for the OS (E2BIG). \
Consider reducing the number of files or excluding directories.",
prog, arg_count, total_len
)
} else {
format!("Failed to spawn: {} {}", prog,
args.iter().map(|a| a.to_string_lossy()).collect::<Vec<_>>().join(" "))
}
})?;
let mut interrupt_rx = ctx.interrupt_receiver();
if ctx.is_interrupted() {
anyhow::bail!("Interrupted");
}
tokio::select! {
biased;
_ = interrupt_rx.changed() => {
anyhow::bail!("Interrupted")
}
result = child.wait_with_output() => {
let output = crate::errors::ctx(result, "Failed to wait for child process")?;
Ok(output)
}
}
})
}
pub(crate) fn run_command(ctx: &crate::build_context::BuildContext, cmd: &mut Command) -> Result<Output> {
let show = crate::runtime_flags::show_output();
run_command_inner(ctx, cmd, show)
}
pub(crate) fn run_command_capture(ctx: &crate::build_context::BuildContext, cmd: &mut Command) -> Result<Output> {
run_command_inner(ctx, cmd, false)
}
pub(crate) fn check_command_output(output: &Output, context: impl std::fmt::Display) -> Result<()> {
if !output.status.success() {
use std::fmt::Write;
let mut msg = format!("{context} failed");
let stderr = String::from_utf8_lossy(&output.stderr);
let stdout = String::from_utf8_lossy(&output.stdout);
if !stderr.is_empty() {
let _ = write!(msg, "\nstderr:\n{}", stderr.trim_end());
}
if !stdout.is_empty() {
let _ = write!(msg, "\nstdout:\n{}", stdout.trim_end());
}
anyhow::bail!("{msg}");
}
Ok(())
}
pub(crate) fn scan_root_valid(_scan: &crate::config::StandardConfig) -> bool {
true
}
pub(crate) fn stub_path(stub_dir: &Path, source: &Path, suffix: &str) -> PathBuf {
let stub_name = format!(
"{}.{}",
source.display().to_string().replace(['/', '\\'], "_"),
suffix,
);
stub_dir.join(stub_name)
}
pub(crate) fn dot_to_svg(dot_content: &str) -> Result<String> {
use std::process::{Command, Stdio};
let mut cmd = Command::new("dot");
cmd.arg("-Tsvg")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
log_command(&cmd);
let mut child = cmd
.spawn()
.map_err(|_| anyhow::anyhow!("Graphviz 'dot' command not found. Install Graphviz to use SVG format"))?;
child.stdin.take()
.context("stdin was not piped to dot command")?
.write_all(dot_content.as_bytes())?;
let output = child.wait_with_output()?;
check_command_output(&output, "dot")?;
Ok(String::from_utf8(output.stdout)?)
}
pub(crate) fn flush_words(
existing: &HashSet<String>,
new_words: &HashSet<String>,
words_path: &Path,
header_line: Option<&str>,
) -> Result<()> {
let to_add: Vec<_> = new_words.iter()
.filter(|w| !existing.contains(*w))
.collect();
if to_add.is_empty() {
return Ok(());
}
let mut sorted: Vec<_> = to_add;
sorted.sort();
let file_exists = words_path.exists();
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(words_path)
.with_context(|| format!("Failed to open words file: {}", words_path.display()))?;
if !file_exists
&& let Some(header) = header_line {
writeln!(file, "{}", header)?;
}
for word in &sorted {
writeln!(file, "{}", word)?;
}
println!("Added {} word(s) to {}", sorted.len(), words_path.display());
Ok(())
}
pub(crate) fn config_file_inputs(path: &str) -> Vec<String> {
if Path::new(path).exists() {
vec![path.to_string()]
} else {
Vec::new()
}
}
pub(crate) fn ensure_output_dir(output: &Path) -> Result<()> {
if let Some(parent) = output.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create output directory: {}", parent.display()))?;
}
Ok(())
}
pub(crate) fn clean_output_dir(product: &Product, processor_name: &str, verbose: bool) -> Result<usize> {
let mut count = 0;
for output_dir in &product.output_dirs {
if output_dir.exists() {
if verbose {
println!("Removing {} output directory: {}", processor_name, output_dir.display());
}
crate::errors::ctx(fs::remove_dir_all(output_dir.as_ref()), &format!("Failed to remove output directory: {}", output_dir.display()))?;
count += 1;
}
}
Ok(count)
}
pub(crate) fn build_anchor_inputs(anchor: &Path, sibling_files: &[PathBuf], extra: &[PathBuf]) -> Vec<PathBuf> {
let mut inputs: Vec<PathBuf> = Vec::with_capacity(1 + sibling_files.len() + extra.len());
inputs.push(anchor.to_path_buf());
for file in sibling_files {
if *file != anchor {
inputs.push(file.clone());
}
}
inputs.extend_from_slice(extra);
inputs
}
pub(crate) fn scan_or_skip(scan: &crate::config::StandardConfig, file_index: &FileIndex) -> Option<Vec<PathBuf>> {
if !scan_root_valid(scan) {
return None;
}
let files = file_index.scan(scan, true);
if files.is_empty() {
return None;
}
Some(files)
}
pub(crate) fn clean_outputs(product: &Product, label: &str, verbose: bool) -> Result<usize> {
let mut count = 0;
for output in &product.outputs {
match fs::remove_file(output) {
Ok(()) => {
count += 1;
if verbose {
println!("Removed {} output: {}", label, output.display());
}
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
}
Ok(count)
}
#[derive(Debug)]
pub(crate) struct SiblingFilter<'a> {
pub extensions: &'a [&'a str],
pub excludes: &'a [&'a str],
}
pub(crate) struct DirectoryProductOpts<'a, H: serde::Serialize> {
pub scan: &'a crate::config::StandardConfig,
pub file_index: &'a FileIndex,
pub dep_inputs: &'a [String],
pub cfg_hash: &'a H,
pub siblings: &'a SiblingFilter<'a>,
pub processor_name: &'a str,
pub output_dir_name: Option<&'a str>,
}
pub(crate) fn discover_directory_products(
graph: &mut BuildGraph,
opts: DirectoryProductOpts<'_, impl serde::Serialize>,
) -> Result<()> {
let DirectoryProductOpts { scan, file_index, dep_inputs, cfg_hash, siblings, processor_name, output_dir_name } = opts;
let files = file_index.scan(scan, true);
if files.is_empty() {
return Ok(());
}
let hash = Some(output_config_hash(cfg_hash, &[]));
let extra = resolve_extra_inputs(dep_inputs)?;
for anchor in files {
let anchor_dir = anchor.parent().map(|p| p.to_path_buf()).unwrap_or_default();
let sibling_files = file_index.query(
&anchor_dir,
siblings.extensions,
siblings.excludes,
&[],
&[],
&[],
);
let inputs = build_anchor_inputs(&anchor, &sibling_files, &extra);
if let Some(dir_name) = output_dir_name {
let output_dir = if anchor_dir.as_os_str().is_empty() {
PathBuf::from(dir_name)
} else {
anchor_dir.join(dir_name)
};
graph.add_product_with_output_dir(inputs, vec![], processor_name, hash.clone(), output_dir)?;
} else {
graph.add_product(inputs, vec![], processor_name, hash.clone())?;
}
}
Ok(())
}
pub(crate) fn discover_checker_products(
graph: &mut BuildGraph,
scan: &crate::config::StandardConfig,
file_index: &FileIndex,
dep_inputs: &[String],
cfg_hash: &impl serde::Serialize,
processor_name: &str,
) -> Result<()> {
let files = file_index.scan(scan, true);
if files.is_empty() {
return Ok(());
}
let hash = Some(output_config_hash(cfg_hash, &[]));
let extra = resolve_extra_inputs(dep_inputs)?;
for file in files {
let mut inputs = Vec::with_capacity(1 + extra.len());
inputs.push(file);
inputs.extend_from_slice(&extra);
graph.add_product(inputs, vec![], processor_name, hash.clone())?;
}
Ok(())
}
pub(crate) fn checker_discover(
graph: &mut BuildGraph,
scan: &crate::config::StandardConfig,
file_index: &FileIndex,
dep_inputs: &[String],
dep_auto: &[String],
cfg_hash: &impl serde::Serialize,
processor_name: &str,
) -> Result<()> {
let mut all_dep_inputs = dep_inputs.to_vec();
for ai in dep_auto {
all_dep_inputs.extend(config_file_inputs(ai));
}
discover_checker_products(graph, scan, file_index, &all_dep_inputs, cfg_hash, processor_name)
}
pub(crate) fn checker_auto_detect(scan: &crate::config::StandardConfig, file_index: &FileIndex) -> bool {
!file_index.scan(scan, true).is_empty()
}
pub(crate) fn checker_auto_detect_with_scan_root(scan: &crate::config::StandardConfig, file_index: &FileIndex) -> bool {
scan_root_valid(scan) && !file_index.scan(scan, true).is_empty()
}
pub(crate) fn run_in_anchor_dir(ctx: &crate::build_context::BuildContext, cmd: &mut Command, anchor: &Path) -> Result<Output> {
let anchor_dir = anchor.parent()
.context("Anchor file has no parent directory")?;
if !anchor_dir.as_os_str().is_empty() {
cmd.current_dir(anchor_dir);
}
run_command(ctx, cmd)
}
pub(crate) fn anchor_display_dir(anchor: &Path) -> &str {
anchor.parent()
.and_then(|p| if p.as_os_str().is_empty() { None } else { p.to_str() })
.unwrap_or(".")
}
pub(crate) fn ensure_stub_dir(stub_dir: &Path, processor_name: &str) -> Result<()> {
if !stub_dir.exists() {
fs::create_dir_all(stub_dir)
.with_context(|| format!("Failed to create {} stub directory", processor_name))?;
}
Ok(())
}
const MAX_ARG_LEN: usize = 1_000_000;
pub(crate) fn run_checker(
ctx: &crate::build_context::BuildContext,
tool: &str,
subcommand: Option<&str>,
args: &[String],
files: &[&Path],
) -> Result<()> {
let mut files: Vec<&Path> = files.to_vec();
files.sort();
files.dedup();
let files = &files[..];
let base_len: usize = tool.len()
+ subcommand.map_or(0, |s| s.len() + 1)
+ args.iter().map(|a| a.len() + 1).sum::<usize>();
let files_len: usize = files.iter().map(|f| f.as_os_str().len() + 1).sum();
if base_len + files_len <= MAX_ARG_LEN {
return run_checker_once(ctx, tool, subcommand, args, files);
}
let mut chunk_start = 0;
while chunk_start < files.len() {
let mut chunk_len = base_len;
let mut chunk_end = chunk_start;
while chunk_end < files.len() {
let file_len = files[chunk_end].as_os_str().len() + 1;
if chunk_len + file_len > MAX_ARG_LEN && chunk_end > chunk_start {
break;
}
chunk_len += file_len;
chunk_end += 1;
}
run_checker_once(ctx, tool, subcommand, args, &files[chunk_start..chunk_end])?;
chunk_start = chunk_end;
}
Ok(())
}
fn run_checker_once(
ctx: &crate::build_context::BuildContext,
tool: &str,
subcommand: Option<&str>,
args: &[String],
files: &[&Path],
) -> Result<()> {
let mut cmd = Command::new(tool);
if let Some(sub) = subcommand {
cmd.arg(sub);
}
for arg in args {
cmd.arg(arg);
}
for file in files {
cmd.arg(file);
}
let output = run_command(ctx, &mut cmd)?;
check_command_output(&output, tool)
}
pub(crate) fn execute_checker_batch<F>(
ctx: &crate::build_context::BuildContext,
products: &[&Product],
batch_fn: F,
) -> Vec<Result<()>>
where
F: Fn(&crate::build_context::BuildContext, &[&Path]) -> Result<()>,
{
let input_paths: Vec<&Path> = products.iter()
.map(|p| p.primary_input())
.collect();
match batch_fn(ctx, &input_paths) {
Ok(()) => products.iter().map(|_| Ok(())).collect(),
Err(e) => {
let err_msg = e.to_string();
products.iter().map(|_| Err(anyhow::anyhow!("{}", err_msg))).collect()
}
}
}
pub(crate) fn execute_generator_batch<F>(
ctx: &crate::build_context::BuildContext,
products: &[&Product],
batch_fn: F,
) -> Vec<Result<()>>
where
F: Fn(&crate::build_context::BuildContext, &[(&Path, &Path)]) -> Result<()>,
{
let pairs: Vec<(&Path, &Path)> = products.iter()
.map(|p| (p.primary_input(), p.primary_output()))
.collect();
match batch_fn(ctx, &pairs) {
Ok(()) => products.iter().map(|_| Ok(())).collect(),
Err(e) => {
let err_msg = e.to_string();
products.iter().map(|_| Err(anyhow::anyhow!("{}", err_msg))).collect()
}
}
}
pub(crate) use checkers::terms;
pub(crate) use generators::tags as tags_cmd;
pub use lua::LuaProcessor;
pub type ProcessorMap = HashMap<String, Box<dyn Processor>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(strum::EnumIter)]
pub enum ProcessorType {
Generator,
Checker,
Creator,
Explicit,
Lua,
}
impl ProcessorType {
pub fn as_str(&self) -> &'static str {
match self {
ProcessorType::Generator => "generator",
ProcessorType::Checker => "checker",
ProcessorType::Creator => "creator",
ProcessorType::Explicit => "explicit",
ProcessorType::Lua => "lua",
}
}
pub fn description(&self) -> &'static str {
match self {
ProcessorType::Generator => "Generates output files from input files (1 input -> 1 output per format)",
ProcessorType::Checker => "Validates input files without producing outputs",
ProcessorType::Creator => "Runs a command and caches declared output files and directories",
ProcessorType::Explicit => "Many inputs aggregated into (possibly) many output files and/or directories",
ProcessorType::Lua => "User-defined processor implemented in Lua via the plugin runtime",
}
}
}
pub struct ProcessorBase {
pub description: &'static str,
pub processor_type: ProcessorType,
}
impl ProcessorBase {
pub fn generator(_name: &'static str, description: &'static str) -> Self {
Self { description, processor_type: ProcessorType::Generator }
}
pub fn creator(_name: &'static str, description: &'static str) -> Self {
Self { description, processor_type: ProcessorType::Creator }
}
pub fn checker(_name: &'static str, description: &'static str) -> Self {
Self { description, processor_type: ProcessorType::Checker }
}
pub fn explicit(_name: &'static str, description: &'static str) -> Self {
Self { description, processor_type: ProcessorType::Explicit }
}
pub fn description(&self) -> &str {
self.description
}
pub fn processor_type(&self) -> ProcessorType {
self.processor_type
}
pub fn config_json<C: Serialize>(config: &C) -> Option<String> {
serde_json::to_string(config).ok()
}
pub fn clean(product: &Product, name: &str, verbose: bool) -> anyhow::Result<usize> {
clean_outputs(product, name, verbose)
}
pub fn clean_output_dir(product: &Product, name: &str, verbose: bool) -> anyhow::Result<usize> {
clean_output_dir(product, name, verbose)
}
}
pub trait Processor: Sync + Send {
fn description(&self) -> &str;
fn processor_type(&self) -> ProcessorType {
ProcessorType::Checker
}
fn scan_config(&self) -> &crate::config::StandardConfig;
fn standard_config(&self) -> Option<&crate::config::StandardConfig> {
None
}
fn discover(&self, graph: &mut BuildGraph, file_index: &FileIndex, instance_name: &str) -> Result<()> {
let cfg = self.standard_config().expect("discover() requires standard_config() or must be overridden");
checker_discover(graph, cfg, file_index, &cfg.dep_inputs, &cfg.dep_auto, cfg, instance_name)
}
fn discover_for_clean(&self, graph: &mut BuildGraph, file_index: &FileIndex, instance_name: &str) -> Result<()> {
self.discover(graph, file_index, instance_name)
}
fn execute(&self, ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()>;
fn clean(&self, _product: &Product, _verbose: bool) -> Result<usize> {
Ok(0)
}
fn auto_detect(&self, file_index: &FileIndex) -> bool {
checker_auto_detect(self.scan_config(), file_index)
}
fn required_tools(&self) -> Vec<String> {
Vec::new()
}
fn tool_version_commands(&self) -> Vec<(String, Vec<String>)> {
self.required_tools()
.into_iter()
.map(|tool| (tool, vec!["--version".to_string()]))
.collect()
}
fn is_native(&self) -> bool {
false
}
fn supports_batch(&self) -> bool;
fn execute_batch(&self, ctx: &crate::build_context::BuildContext, products: &[&Product]) -> Vec<Result<()>> {
products.iter().map(|p| self.execute(ctx, p)).collect()
}
fn config_json(&self) -> Option<String> {
self.standard_config().and_then(|c| serde_json::to_string(c).ok())
}
fn max_jobs(&self) -> Option<usize> {
self.standard_config().and_then(|c| c.max_jobs)
}
}
pub struct InstallMethod {
pub method: &'static str,
pub package: &'static str,
}
impl InstallMethod {
pub fn command(&self) -> String {
match self.method {
"apt" => format!("sudo apt install -y {}", self.package),
"snap" => format!("sudo snap install {}", self.package),
"pip" => format!("pip install {}", self.package),
"npm" => format!("npm install -g {}", self.package),
"cargo" => format!("cargo install {}", self.package),
"gem" => format!("gem install {}", self.package),
_ => self.package.to_string(),
}
}
pub fn batch_command(method: &str, packages: &[&str]) -> String {
match method {
"apt" => format!("sudo apt install -y {}", packages.join(" ")),
"snap" => format!("sudo snap install {}", packages.join(" ")),
"pip" => format!("pip install {}", packages.join(" ")),
"npm" => format!("npm install -g {}", packages.join(" ")),
"cargo" => format!("cargo install {}", packages.join(" ")),
"gem" => format!("gem install {}", packages.join(" ")),
_ => packages.iter().map(|p| p.to_string()).collect::<Vec<_>>().join("; "),
}
}
}
pub struct ToolInfo {
pub name: &'static str,
pub runtime: &'static str,
pub install_methods: &'static [InstallMethod],
}
pub static TOOLS: &[ToolInfo] = &[
ToolInfo { name: "ruff", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "ruff" }] },
ToolInfo { name: "pylint", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "pylint" }] },
ToolInfo { name: "mypy", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "mypy" }] },
ToolInfo { name: "pyrefly", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "pyrefly" }] },
ToolInfo { name: "yamllint", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "yamllint" }] },
ToolInfo { name: "sphinx-build", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "sphinx" }] },
ToolInfo { name: "pip", runtime: "python", install_methods: &[InstallMethod { method: "system", package: "python3 -m ensurepip" }] },
ToolInfo { name: "jsonlint", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "demjson3" }] },
ToolInfo { name: "cpplint", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "cpplint" }] },
ToolInfo { name: "black", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "black" }] },
ToolInfo { name: "pytest", runtime: "python", install_methods: &[InstallMethod { method: "pip", package: "pytest" }] },
ToolInfo { name: "a2x", runtime: "python", install_methods: &[InstallMethod { method: "apt", package: "asciidoc" }] },
ToolInfo { name: "python3", runtime: "python", install_methods: &[InstallMethod { method: "apt", package: "python3" }] },
ToolInfo { name: "marp", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "@marp-team/marp-cli" }] },
ToolInfo { name: "mmdc", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "@mermaid-js/mermaid-cli" }] },
ToolInfo { name: "node_modules/.bin/markdownlint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "markdownlint-cli" }] },
ToolInfo { name: "markdownlint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "markdownlint-cli" }] },
ToolInfo { name: "prettier", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "prettier" }] },
ToolInfo { name: "eslint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "eslint" }] },
ToolInfo { name: "htmlhint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "htmlhint" }] },
ToolInfo { name: "jshint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "jshint" }] },
ToolInfo { name: "npm", runtime: "node", install_methods: &[InstallMethod { method: "apt", package: "npm" }] },
ToolInfo { name: "node", runtime: "node", install_methods: &[InstallMethod { method: "apt", package: "nodejs" }] },
ToolInfo { name: "gems/bin/mdl", runtime: "ruby", install_methods: &[InstallMethod { method: "gem", package: "mdl" }] },
ToolInfo { name: "mdl", runtime: "ruby", install_methods: &[InstallMethod { method: "gem", package: "mdl" }] },
ToolInfo { name: "bundle", runtime: "ruby", install_methods: &[InstallMethod { method: "gem", package: "bundler" }] },
ToolInfo { name: "ruby", runtime: "ruby", install_methods: &[InstallMethod { method: "apt", package: "ruby" }] },
ToolInfo { name: "mdbook", runtime: "rust", install_methods: &[InstallMethod { method: "cargo", package: "mdbook" }] },
ToolInfo { name: "rumdl", runtime: "rust", install_methods: &[InstallMethod { method: "cargo", package: "rumdl" }] },
ToolInfo { name: "taplo", runtime: "rust", install_methods: &[
InstallMethod { method: "binary", package: "curl -fsSL https://github.com/tamasfe/taplo/releases/latest/download/taplo-linux-x86_64.gz | gunzip > /tmp/taplo && chmod +x /tmp/taplo && sudo mv /tmp/taplo /usr/local/bin/taplo" },
InstallMethod { method: "cargo", package: "taplo-cli" },
]},
ToolInfo { name: "cargo", runtime: "rust", install_methods: &[InstallMethod { method: "binary", package: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" }] },
ToolInfo { name: "rustc", runtime: "rust", install_methods: &[InstallMethod { method: "binary", package: "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh" }] },
ToolInfo { name: "perl", runtime: "perl", install_methods: &[InstallMethod { method: "apt", package: "perl" }] },
ToolInfo { name: "markdown", runtime: "perl", install_methods: &[InstallMethod { method: "apt", package: "markdown" }] },
ToolInfo { name: "checkpatch.pl", runtime: "perl", install_methods: &[InstallMethod { method: "manual", package: "install from Linux kernel source: scripts/checkpatch.pl" }] },
ToolInfo { name: "shellcheck", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "shellcheck" }] },
ToolInfo { name: "luacheck", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "lua-check" }] },
ToolInfo { name: "cppcheck", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "cppcheck" }] },
ToolInfo { name: "clang-tidy", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "clang-tidy" }] },
ToolInfo { name: "gcc", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "gcc" }] },
ToolInfo { name: "g++", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "g++" }] },
ToolInfo { name: "clang", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "clang" }] },
ToolInfo { name: "clang++", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "clang" }] },
ToolInfo { name: "ar", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "binutils" }] },
ToolInfo { name: "make", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "make" }] },
ToolInfo { name: "jq", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "jq" }] },
ToolInfo { name: "aspell", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "aspell" }] },
ToolInfo { name: "pandoc", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "pandoc" }] },
ToolInfo { name: "pdflatex", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "texlive-latex-base" }] },
ToolInfo { name: "qpdf", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "qpdf" }] },
ToolInfo { name: "dot", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "graphviz" }] },
ToolInfo { name: "drawio", runtime: "system", install_methods: &[InstallMethod { method: "snap", package: "drawio" }] },
ToolInfo { name: "libreoffice", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "libreoffice" }] },
ToolInfo { name: "flock", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "util-linux" }] },
ToolInfo { name: "pdfunite", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "poppler-utils" }] },
ToolInfo { name: "google-chrome", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "google-chrome-stable" }] },
ToolInfo { name: "objdump", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "binutils" }] },
ToolInfo { name: "tidy", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "tidy" }] },
ToolInfo { name: "xmllint", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "libxml2-utils" }] },
ToolInfo { name: "svglint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "svglint" }] },
ToolInfo { name: "svgo", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "svgo" }] },
ToolInfo { name: "cmake", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "cmake" }] },
ToolInfo { name: "protoc", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "protobuf-compiler" }] },
ToolInfo { name: "sass", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "sass" }] },
ToolInfo { name: "hadolint", runtime: "system", install_methods: &[
InstallMethod { method: "binary", package: "wget -O ~/.local/bin/hadolint https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-x86_64 && chmod +x ~/.local/bin/hadolint" },
InstallMethod { method: "brew", package: "hadolint" },
InstallMethod { method: "nix", package: "hadolint" },
InstallMethod { method: "apt", package: "hadolint" },
]},
ToolInfo { name: "php", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "php-cli" }] },
ToolInfo { name: "checkstyle", runtime: "system", install_methods: &[InstallMethod { method: "apt", package: "checkstyle" }] },
ToolInfo { name: "yq", runtime: "system", install_methods: &[
InstallMethod { method: "pip", package: "yq" },
InstallMethod { method: "snap", package: "yq" },
InstallMethod { method: "apt", package: "yq" },
]},
ToolInfo { name: "stylelint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "stylelint" }] },
ToolInfo { name: "jslint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "jslint" }] },
ToolInfo { name: "standard", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "standard" }] },
ToolInfo { name: "htmllint", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "htmllint-cli" }] },
ToolInfo { name: "slidev", runtime: "node", install_methods: &[InstallMethod { method: "npm", package: "@slidev/cli" }] },
ToolInfo { name: "perlcritic", runtime: "perl", install_methods: &[InstallMethod { method: "apt", package: "libperl-critic-perl" }] },
ToolInfo { name: "jekyll", runtime: "ruby", install_methods: &[InstallMethod { method: "gem", package: "jekyll" }] },
ToolInfo { name: "true", runtime: "system", install_methods: &[InstallMethod { method: "system", package: "coreutils" }] },
];
pub fn tool_info(tool: &str) -> Option<&'static ToolInfo> {
TOOLS.iter().find(|t| t.name == tool)
}
pub fn tool_install_command(tool: &str) -> Option<String> {
tool_info(tool).and_then(|t| t.install_methods.first().map(|m| m.command()))
}
pub fn tool_runtime(tool: &str) -> Option<&'static str> {
tool_info(tool).map(|t| t.runtime)
}
#[derive(Debug, Clone, PartialEq)]
pub struct ProductTiming {
pub display: String,
pub processor: String,
pub duration: Duration,
pub start_offset: Option<Duration>,
}
#[derive(Debug, Default, PartialEq)]
pub struct ProcessStats {
pub processed: usize,
pub failed: usize,
pub flaky: usize,
pub skipped: usize,
pub restored: usize,
pub files_created: usize,
pub files_restored: usize,
pub duration: Duration,
pub product_timings: Vec<ProductTiming>,
}
impl ProcessStats {
pub fn total(&self) -> usize {
self.processed + self.failed + self.skipped + self.restored
}
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct FailedProduct {
pub file: String,
pub processor: String,
pub error: String,
}
#[derive(Default)]
pub struct BuildStats {
pub categories: Vec<ProcessStats>,
pub total_duration: Duration,
pub failed_count: usize,
pub failed_messages: Vec<String>,
pub failed_details: Vec<FailedProduct>,
pub phase_timings: Vec<(String, Duration)>,
}
impl BuildStats {
pub fn add(&mut self, stats: ProcessStats) {
if stats.total() > 0 {
self.categories.push(stats);
}
}
pub fn total_processed(&self) -> usize {
self.categories.iter().map(|s| s.processed).sum()
}
pub fn total_skipped(&self) -> usize {
self.categories.iter().map(|s| s.skipped).sum()
}
pub fn total_restored(&self) -> usize {
self.categories.iter().map(|s| s.restored).sum()
}
pub fn total_files_created(&self) -> usize {
self.categories.iter().map(|s| s.files_created).sum()
}
pub fn total_files_restored(&self) -> usize {
self.categories.iter().map(|s| s.files_restored).sum()
}
pub fn total_flaky(&self) -> usize {
self.categories.iter().map(|s| s.flaky).sum()
}
pub fn print_summary(&self, summary: bool, timings: bool) {
if crate::json_output::is_json_mode() || crate::runtime_flags::quiet() {
return;
}
if !summary && !timings {
return;
}
if self.categories.is_empty() && self.failed_count == 0 {
if summary {
println!("{}", color::dim("Nothing to build."));
}
return;
}
if summary {
let total_processed = self.total_processed();
let total_restored = self.total_restored();
let total_failed = self.failed_count;
let total_skipped = self.total_skipped();
let total_files_created = self.total_files_created();
let total_files_restored = self.total_files_restored();
let total_flaky = self.total_flaky();
let built_part = if total_files_created > 0 {
format!("{} built ({} files created)", total_processed, total_files_created)
} else {
format!("{} built", total_processed)
};
let restored_part = if total_files_restored > 0 {
format!("{} restored ({} files)", total_restored, total_files_restored)
} else {
format!("{} restored", total_restored)
};
let lead = format!(
"{}, {}, {} failed",
built_part, restored_part, total_failed,
);
let aside = format!("{} unchanged, {} flaky", total_skipped, total_flaky);
println!("[build] summary: {} ({})", lead, aside);
}
if self.failed_count > 0 {
println!("{}", color::red(&format!("Build finished with {} error(s):", self.failed_count)));
for msg in &self.failed_messages {
println!("{} {}", color::red("*"), msg);
}
}
if timings {
println!();
println!("{}", color::bold("Timing:"));
if !self.phase_timings.is_empty() {
let mut builder = tabled::builder::Builder::new();
builder.push_record(["Phase", "Duration"]);
for (name, dur) in &self.phase_timings {
builder.push_record([name.to_string(), format!("{:.3}s", dur.as_secs_f64())]);
}
crate::color::print_table(builder.build());
}
for cat in &self.categories {
for pt in &cat.product_timings {
println!("[{}] {} {}", pt.processor, pt.display,
color::dim(&format!("({:.3}s)", pt.duration.as_secs_f64())));
}
}
let total: f64 = self.phase_timings.iter().map(|(_, d)| d.as_secs_f64()).sum();
println!("{}", color::bold(&format!("Total: {:.3}s", total)));
}
}
}
pub struct SimpleChecker {
config: CheckerConfigWithCommand,
params: SimpleCheckerParams,
}
impl SimpleChecker {
pub fn new(config: CheckerConfigWithCommand, params: SimpleCheckerParams) -> Self {
Self { config, params }
}
fn check_files(&self, ctx: &crate::build_context::BuildContext, files: &[&Path]) -> Result<()> {
let tool = self.config.standard.require_command(self.params.description)?;
if self.params.prepend_args.is_empty() {
run_checker(ctx, tool, self.params.subcommand, &self.config.standard.args, files)
} else {
let mut combined_args: Vec<String> = self.params.prepend_args.iter().map(|s| s.to_string()).collect();
combined_args.extend_from_slice(&self.config.standard.args);
run_checker(ctx, tool, self.params.subcommand, &combined_args, files)
}
}
}
impl Processor for SimpleChecker {
fn scan_config(&self) -> &StandardConfig {
&self.config.standard
}
fn standard_config(&self) -> Option<&StandardConfig> {
Some(&self.config.standard)
}
fn description(&self) -> &str {
self.params.description
}
fn auto_detect(&self, file_index: &FileIndex) -> bool {
!file_index.scan(&self.config.standard, true).is_empty()
}
fn required_tools(&self) -> Vec<String> {
let mut tools = vec![self.config.standard.command.clone()];
for t in self.params.extra_tools {
tools.push(t.to_string());
}
tools
}
fn discover(
&self,
graph: &mut BuildGraph,
file_index: &FileIndex,
instance_name: &str,
) -> Result<()> {
let mut dep_inputs = self.config.standard.dep_inputs.clone();
for ai in &self.config.standard.dep_auto {
dep_inputs.extend(config_file_inputs(ai));
}
discover_checker_products(
graph, &self.config.standard, file_index, &dep_inputs, &self.config, instance_name,
)
}
fn execute(&self, ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()> {
self.check_files(ctx, &[product.primary_input()])
}
fn supports_batch(&self) -> bool {
self.config.standard.batch
}
fn execute_batch(&self, ctx: &crate::build_context::BuildContext, products: &[&Product]) -> Vec<Result<()>> {
execute_checker_batch(ctx, products, |ctx, files| self.check_files(ctx, files))
}
}
#[derive(Copy, Clone)]
pub(crate) enum DiscoverMode {
MultiFormat,
SingleFormat(&'static str),
}
#[derive(Copy, Clone)]
pub(crate) struct SimpleGeneratorParams {
pub description: &'static str,
pub extra_tools: &'static [&'static str],
pub discover_mode: DiscoverMode,
pub execute_fn: fn(&crate::build_context::BuildContext, &StandardConfig, &Product) -> Result<()>,
pub is_native: bool,
}
pub struct SimpleGenerator {
base: ProcessorBase,
config: StandardConfig,
params: SimpleGeneratorParams,
}
impl SimpleGenerator {
pub fn new(config: StandardConfig, params: SimpleGeneratorParams) -> Self {
Self {
base: ProcessorBase::generator("", params.description),
config,
params,
}
}
}
impl Processor for SimpleGenerator {
fn scan_config(&self) -> &StandardConfig {
&self.config
}
fn standard_config(&self) -> Option<&StandardConfig> {
Some(&self.config)
}
fn description(&self) -> &str {
self.base.description()
}
fn processor_type(&self) -> ProcessorType {
self.base.processor_type()
}
fn config_json(&self) -> Option<String> {
ProcessorBase::config_json(&self.config)
}
fn clean(&self, product: &Product, verbose: bool) -> Result<usize> {
ProcessorBase::clean(product, &product.processor, verbose)
}
fn is_native(&self) -> bool {
self.params.is_native
}
fn required_tools(&self) -> Vec<String> {
if self.params.is_native {
self.params.extra_tools.iter().map(|t| t.to_string()).collect()
} else {
let mut tools = vec![self.config.command.clone()];
for t in self.params.extra_tools {
tools.push(t.to_string());
}
tools
}
}
fn max_jobs(&self) -> Option<usize> {
self.config.max_jobs
}
fn discover(&self, graph: &mut BuildGraph, file_index: &FileIndex, instance_name: &str) -> Result<()> {
let params = generators::DiscoverParams {
scan: &self.config,
dep_inputs: &self.config.dep_inputs,
config: &self.config,
output_dir: &self.config.output_dir,
processor_name: instance_name,
};
match &self.params.discover_mode {
DiscoverMode::MultiFormat => {
generators::discover_multi_format(graph, file_index, ¶ms, &self.config.formats)
}
DiscoverMode::SingleFormat(ext) => {
generators::discover_single_format(graph, file_index, ¶ms, ext)
}
}
}
fn supports_batch(&self) -> bool { false }
fn execute(&self, ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()> {
(self.params.execute_fn)(ctx, &self.config, product)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::builder::create_all_default_processors;
#[test]
fn all_required_tools_have_registry_entries() {
let processors = create_all_default_processors();
for (proc_name, proc) in &processors {
for tool in proc.required_tools() {
if tool.is_empty() {
continue;
}
assert!(
tool_install_command(&tool).is_some(),
"Processor '{}' requires tool '{}' which has no install command in TOOLS",
proc_name, tool
);
assert!(
tool_runtime(&tool).is_some(),
"Processor '{}' requires tool '{}' which has no runtime category in TOOLS",
proc_name, tool
);
}
}
}
}