mod execution;
mod handlers;
mod policy;
pub(crate) use policy::{BuildPolicy, IncrementalPolicy, ProductAction};
use anyhow::Result;
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use indicatif::ProgressBar;
use parking_lot::Mutex;
use crate::cli::DisplayOptions;
use crate::errors;
use crate::color;
use crate::graph::BuildGraph;
use crate::object_store::{ExplainAction, ObjectStore};
use crate::processors::{FailedProduct, ProcessStats, ProcessorMap};
enum PreCheckResult {
Handled,
NeedsExecution,
}
enum RestoreOutcome {
Restored,
Failed,
NotRestorable,
}
struct WorkItem {
product_id: usize,
input_checksum: String,
needs_rebuild: bool,
}
struct HandlerContext<'b> {
product: &'b crate::graph::Product,
id: usize,
input_checksum: &'b str,
proc_name: &'b str,
keep_going: bool,
shared: &'b SharedState,
pb: &'b ProgressBar,
}
struct LevelWork {
batch_groups: HashMap<String, Vec<WorkItem>>,
non_batch_items: Vec<WorkItem>,
}
#[derive(Debug)]
pub struct ExecutorOptions {
pub parallel: usize,
pub verbose: bool,
pub display_opts: DisplayOptions,
pub batch_size: Option<usize>,
pub explain: bool,
pub retry: usize,
}
#[derive(Debug)]
struct SharedState {
stats: Arc<Mutex<HashMap<String, ProcessStats>>>,
errors: Arc<Mutex<Vec<anyhow::Error>>>,
failed_products: Arc<Mutex<HashSet<usize>>>,
failed_messages: Arc<Mutex<Vec<String>>>,
failed_details: Arc<Mutex<Vec<FailedProduct>>>,
failed_processors: Arc<Mutex<HashSet<String>>>,
unchanged_products: Arc<Mutex<HashSet<usize>>>,
global_current: Arc<AtomicUsize>,
global_total: usize,
}
pub fn classify_products(
policy: &dyn BuildPolicy,
graph: &BuildGraph,
order: &[usize],
object_store: &ObjectStore,
force: bool,
) -> (usize, usize, usize) {
let mut skip_count = 0;
let mut restore_count = 0;
let mut build_count = 0;
let mut will_change: HashSet<usize> = HashSet::new();
for &id in order {
let product = graph.get_product(id).expect(errors::INVALID_PRODUCT_ID);
let dep_changed = graph.get_dependencies(id).iter().any(|d| will_change.contains(d));
let input_checksum = match crate::checksum::combined_input_checksum(&product.inputs) {
Ok(cs) => cs,
Err(_) => {
build_count += 1;
will_change.insert(id);
continue;
}
};
match policy.classify(product, object_store, &input_checksum, dep_changed, force) {
ProductAction::Skip => {
skip_count += 1;
}
ProductAction::Restore => {
restore_count += 1;
will_change.insert(id);
}
ProductAction::Build => {
build_count += 1;
will_change.insert(id);
}
}
}
(skip_count, restore_count, build_count)
}
pub struct Executor<'a> {
processors: &'a ProcessorMap,
build_ctx: &'a crate::build_context::BuildContext,
policy: &'a dyn BuildPolicy,
parallel: usize,
verbose: bool,
display_opts: DisplayOptions,
interrupted: Arc<AtomicBool>,
batch_size: Option<usize>,
explain: bool,
retry: usize,
}
impl<'a> Executor<'a> {
pub fn new(
processors: &'a ProcessorMap,
build_ctx: &'a crate::build_context::BuildContext,
policy: &'a dyn BuildPolicy,
opts: ExecutorOptions,
interrupted: Arc<AtomicBool>,
) -> Self {
Self {
processors,
build_ctx,
policy,
parallel: opts.parallel,
verbose: opts.verbose,
display_opts: opts.display_opts,
interrupted,
batch_size: opts.batch_size,
explain: opts.explain,
retry: opts.retry,
}
}
fn is_interrupted(&self) -> bool {
self.interrupted.load(Ordering::SeqCst) || self.build_ctx.is_interrupted()
}
fn product_display(&self, product: &crate::graph::Product) -> String {
product.display(self.display_opts)
}
fn inc_global(shared: &SharedState) {
shared.global_current.fetch_add(1, Ordering::SeqCst);
}
fn inc_progress(pb: &ProgressBar, shared: &SharedState) {
Self::inc_global(shared);
pb.inc(1);
}
fn classify_products(
&self,
graph: &BuildGraph,
order: &[usize],
object_store: &ObjectStore,
force: bool,
) -> (usize, usize, usize) {
classify_products(self.policy, graph, order, object_store, force)
}
fn print_explain(&self, product: &crate::graph::Product, action: &ExplainAction) {
let styled = match action {
ExplainAction::Skip => color::dim("SKIP"),
ExplainAction::Restore(_) => color::cyan("RESTORE"),
ExplainAction::Rebuild(_) => color::yellow("BUILD"),
};
println!("[{}] {} {} ({})", product.processor,
styled,
self.product_display(product),
action);
}
fn has_failed_dependency(&self, graph: &BuildGraph, id: usize, failed: &HashSet<usize>) -> bool {
for &dep_id in graph.get_dependencies(id) {
if failed.contains(&dep_id) {
return true;
}
}
false
}
fn compute_parallel_levels(&self, graph: &BuildGraph, order: &[usize]) -> Vec<Vec<usize>> {
let mut levels: Vec<Vec<usize>> = Vec::new();
let mut product_level: HashMap<usize, usize> = HashMap::new();
for &id in order {
let max_dep_level = graph.get_dependencies(id)
.iter()
.filter_map(|&dep_id| product_level.get(&dep_id))
.max()
.copied()
.unwrap_or(0);
let my_level = if graph.get_dependencies(id).is_empty() {
0
} else {
max_dep_level + 1
};
product_level.insert(id, my_level);
while levels.len() <= my_level {
levels.push(Vec::new());
}
levels[my_level].push(id);
}
levels
}
pub fn clean(&self, graph: &BuildGraph, verbose: bool) -> Result<HashMap<String, usize>> {
let mut stats: HashMap<String, usize> = HashMap::new();
for product in graph.products() {
if let Some(processor) = self.processors.get(&product.processor) {
let count = processor.clean(product, verbose)?;
if count > 0 {
*stats.entry(product.processor.clone()).or_default() += count;
}
}
}
Ok(stats)
}
}