mod checksums;
mod config_diff;
mod management;
mod operations;
mod validity;
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use redb::{Database, ReadableDatabase, TableDefinition};
use crate::checksum;
use crate::config::RestoreMethod;
use crate::remote_cache::RemoteCache;
const CHECKSUM_PREFIX_LEN: usize = 2;
fn walk_files(dir: &Path) -> Vec<PathBuf> {
let mut result = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(current) = stack.pop() {
if let Ok(entries) = fs::read_dir(¤t) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else if path.is_file() {
result.push(path);
}
}
}
}
result
}
const RSBUILD_DIR: &str = ".rsconstruct";
const OBJECTS_DIR: &str = "objects";
const DESCRIPTORS_DIR: &str = "descriptors";
const DB_FILE: &str = "db.redb";
const CACHE_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("cache");
const CONFIGS_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("processor_configs");
#[derive(Debug)]
pub enum RebuildReason {
NoCacheEntry,
OutputMissing(String),
Force,
}
impl std::fmt::Display for RebuildReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RebuildReason::NoCacheEntry => write!(f, "no cache entry"),
RebuildReason::OutputMissing(path) => write!(f, "output missing: {}", path),
RebuildReason::Force => write!(f, "forced"),
}
}
}
#[derive(Debug)]
pub enum ExplainAction {
Skip,
Restore(RebuildReason),
Rebuild(RebuildReason),
}
impl std::fmt::Display for ExplainAction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ExplainAction::Skip => write!(f, "SKIP (inputs unchanged)"),
ExplainAction::Restore(reason) => write!(f, "RESTORE ({})", reason),
ExplainAction::Rebuild(reason) => write!(f, "BUILD ({})", reason),
}
}
}
pub struct ObjectStore {
objects_dir: PathBuf,
descriptors_dir: PathBuf,
db: Database,
restore_method: RestoreMethod,
compression: bool,
remote: Option<Box<dyn RemoteCache>>,
remote_push: bool,
#[allow(dead_code)]
remote_pull: bool,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "type")]
enum CacheDescriptor {
#[serde(rename = "marker")]
Marker,
#[serde(rename = "blob")]
Blob {
checksum: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
mode: Option<u32>,
},
#[serde(rename = "tree")]
Tree {
entries: Vec<TreeEntry>,
},
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct TreeEntry {
path: String,
checksum: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
mode: Option<u32>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct CacheEntry {
input_checksum: String,
outputs: Vec<OutputEntry>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
struct OutputEntry {
path: String,
checksum: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
mode: Option<u32>,
}
#[derive(Debug, Default, Serialize)]
pub struct ProcessorCacheStats {
pub entry_count: usize,
pub output_count: usize,
pub output_bytes: u64,
}
#[derive(Debug, Serialize)]
pub struct CacheListEntry {
pub cache_key: String,
pub outputs: Vec<CacheListOutput>,
}
#[derive(Debug, Serialize)]
pub struct CacheListOutput {
pub path: String,
pub exists: bool,
}
pub struct ObjectStoreOptions {
pub restore_method: RestoreMethod,
pub compression: bool,
pub remote: Option<Box<dyn RemoteCache>>,
pub remote_push: bool,
pub remote_pull: bool,
}
impl ObjectStore {
pub fn new(opts: ObjectStoreOptions) -> Result<Self> {
let rsconstruct_dir = PathBuf::from(RSBUILD_DIR);
let objects_dir = rsconstruct_dir.join(OBJECTS_DIR);
let db_path = rsconstruct_dir.join(DB_FILE);
fs::create_dir_all(&rsconstruct_dir)
.context("Failed to create .rsconstruct directory")?;
let db = crate::db::open_or_recreate(&db_path, "Cache database")?;
let descriptors_dir = rsconstruct_dir.join(DESCRIPTORS_DIR);
Ok(Self {
objects_dir,
descriptors_dir,
db,
restore_method: opts.restore_method,
compression: opts.compression,
remote: opts.remote,
remote_push: opts.remote_push,
remote_pull: opts.remote_pull,
})
}
pub fn has_cache_entry(&self, cache_key: &str) -> bool {
self.get_entry(cache_key).is_some()
}
fn get_entry(&self, cache_key: &str) -> Option<CacheEntry> {
let read_txn = self.db.begin_read().ok()?;
let table = read_txn.open_table(CACHE_TABLE).ok()?;
let data = table.get(cache_key).ok()??;
serde_json::from_slice(data.value()).ok()
}
fn descriptor_path(&self, descriptor_key: &str) -> PathBuf {
let (prefix, rest) = descriptor_key.split_at(CHECKSUM_PREFIX_LEN.min(descriptor_key.len()));
self.descriptors_dir.join(prefix).join(rest)
}
fn store_descriptor(&self, cache_key: &str, descriptor: &CacheDescriptor) -> Result<()> {
let path = self.descriptor_path(cache_key);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.context("Failed to create descriptor directory")?;
}
let data = serde_json::to_vec(descriptor)
.context("Failed to serialize cache descriptor")?;
if path.exists() {
let mut perms = fs::metadata(&path)
.with_context(|| format!("Failed to read metadata for descriptor: {}", path.display()))?.permissions();
perms.set_readonly(false);
fs::set_permissions(&path, perms)
.with_context(|| format!("Failed to make descriptor writable: {}", path.display()))?;
}
fs::write(&path, &data)
.with_context(|| format!("Failed to write cache descriptor: {}", path.display()))?;
let mut perms = fs::metadata(&path)
.with_context(|| format!("Failed to read metadata for descriptor: {}", path.display()))?.permissions();
perms.set_readonly(true);
fs::set_permissions(&path, perms)
.with_context(|| format!("Failed to make descriptor read-only: {}", path.display()))?;
Ok(())
}
fn get_descriptor(&self, cache_key: &str) -> Option<CacheDescriptor> {
let path = self.descriptor_path(cache_key);
let data = fs::read(&path).ok()?;
serde_json::from_slice(&data).ok()
}
pub fn previous_tree_paths(&self, cache_key: &str) -> Vec<PathBuf> {
match self.get_descriptor(cache_key) {
Some(CacheDescriptor::Tree { entries }) => {
entries.into_iter().map(|e| PathBuf::from(e.path)).collect()
}
_ => Vec::new(),
}
}
pub fn store_marker(&self, cache_key: &str) -> Result<()> {
self.store_descriptor(cache_key, &CacheDescriptor::Marker)
}
pub fn store_blob_descriptor(&self, cache_key: &str, output_path: &Path) -> Result<bool> {
let content = fs::read(output_path)
.with_context(|| format!("Failed to read output: {}", output_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(output_path).ok()
.and_then(|m| crate::platform::get_mode(&m));
let changed = match self.get_descriptor(cache_key) {
Some(CacheDescriptor::Blob { checksum: prev, .. }) => prev != checksum,
_ => true,
};
if self.remote_push {
self.try_push_object_to_remote(&checksum)?;
}
self.store_descriptor(cache_key, &CacheDescriptor::Blob {
checksum,
mode,
})?;
Ok(changed)
}
pub fn store_tree_descriptor(
&self,
cache_key: &str,
output_dirs: &[std::sync::Arc<PathBuf>],
output_files: &[PathBuf],
is_foreign: &dyn Fn(&Path) -> bool,
) -> Result<bool> {
let prev = self.get_descriptor(cache_key);
let mut entries = Vec::new();
for dir in output_dirs {
let dir: &Path = dir;
anyhow::ensure!(dir.exists() && dir.is_dir(),
"Expected output directory not produced: {}", dir.display());
for file_path in walk_files(dir) {
if is_foreign(&file_path) {
continue;
}
let content = fs::read(&file_path)
.with_context(|| format!("Failed to read: {}", file_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(&file_path).ok()
.and_then(|m| crate::platform::get_mode(&m));
if self.remote_push {
self.try_push_object_to_remote(&checksum)?;
}
entries.push(TreeEntry {
path: file_path.display().to_string(),
checksum,
mode,
});
}
}
for file_path in output_files {
anyhow::ensure!(file_path.exists(),
"Expected output file not produced: {}", file_path.display());
let content = fs::read(file_path)
.with_context(|| format!("Failed to read: {}", file_path.display()))?;
let checksum = self.store_object(&content)?;
let mode = fs::metadata(file_path).ok()
.and_then(|m| crate::platform::get_mode(&m));
if self.remote_push {
self.try_push_object_to_remote(&checksum)?;
}
entries.push(TreeEntry {
path: Self::path_string(file_path),
checksum,
mode,
});
}
let changed = match prev {
Some(CacheDescriptor::Tree { entries: ref prev_entries }) => {
entries.len() != prev_entries.len()
|| entries.iter().zip(prev_entries.iter()).any(|(a, b)| a.checksum != b.checksum || a.path != b.path)
}
_ => true,
};
self.store_descriptor(cache_key, &CacheDescriptor::Tree { entries })?;
Ok(changed)
}
pub fn restore_from_descriptor(&self, cache_key: &str, output_paths: &[PathBuf]) -> Result<bool> {
let descriptor = match self.get_descriptor(cache_key) {
Some(d) => d,
None => return Ok(false),
};
match descriptor {
CacheDescriptor::Marker => Ok(true),
CacheDescriptor::Blob { checksum, mode } => {
let output_path = match output_paths.first() {
Some(p) => p,
None => return Ok(true), };
if output_path.exists() {
return Ok(true);
}
if !self.has_object(&checksum) {
return Ok(false);
}
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create output directory: {}", parent.display()))?;
}
self.restore_file(&checksum, output_path)
.with_context(|| format!("Failed to restore blob to: {}", output_path.display()))?;
if let Some(m) = mode {
crate::platform::set_permissions_mode(output_path, m)
.with_context(|| format!("Failed to set permissions on: {}", output_path.display()))?;
}
Ok(true)
}
CacheDescriptor::Tree { entries } => {
for entry in &entries {
let file_path = Path::new(&entry.path);
if file_path.exists() {
if let Ok(existing) = Self::calculate_checksum(file_path) {
if existing == entry.checksum {
continue;
}
}
fs::remove_file(file_path)
.with_context(|| format!("Failed to remove stale cached file: {}", file_path.display()))?;
}
if !self.has_object(&entry.checksum) {
return Ok(false);
}
if let Some(parent) = file_path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create directory for tree restore: {}", parent.display()))?;
}
self.restore_file(&entry.checksum, file_path)
.with_context(|| format!("Failed to restore tree entry: {}", file_path.display()))?;
if let Some(m) = entry.mode {
crate::platform::set_permissions_mode(file_path, m)
.with_context(|| format!("Failed to set permissions on: {}", file_path.display()))?;
}
}
Ok(true)
}
}
}
pub fn needs_rebuild_descriptor(&self, cache_key: &str, output_paths: &[PathBuf]) -> bool {
let descriptor = match self.get_descriptor(cache_key) {
Some(d) => d,
None => return true,
};
match descriptor {
CacheDescriptor::Marker => false,
CacheDescriptor::Blob { .. } => {
output_paths.iter().any(|p| !p.exists())
}
CacheDescriptor::Tree { entries } => {
entries.iter().any(|e| {
let p = Path::new(&e.path);
!p.exists() || Self::calculate_checksum(p).ok().as_ref() != Some(&e.checksum)
})
}
}
}
pub fn can_restore_descriptor(&self, cache_key: &str) -> bool {
let descriptor = match self.get_descriptor(cache_key) {
Some(d) => d,
None => return false,
};
match descriptor {
CacheDescriptor::Marker => true,
CacheDescriptor::Blob { checksum, .. } => self.has_object(&checksum),
CacheDescriptor::Tree { entries } => {
entries.iter().all(|e| self.has_object(&e.checksum))
}
}
}
pub fn explain_descriptor(&self, descriptor_key: &str, output_paths: &[PathBuf], force: bool) -> ExplainAction {
if force {
return ExplainAction::Rebuild(RebuildReason::Force);
}
let descriptor = match self.get_descriptor(descriptor_key) {
Some(d) => d,
None => return ExplainAction::Rebuild(RebuildReason::NoCacheEntry),
};
match descriptor {
CacheDescriptor::Marker => ExplainAction::Skip,
CacheDescriptor::Blob { checksum, .. } => {
for p in output_paths {
if !p.exists() {
let display = p.display().to_string();
if self.has_object(&checksum) {
return ExplainAction::Restore(RebuildReason::OutputMissing(display));
} else {
return ExplainAction::Rebuild(RebuildReason::OutputMissing(display));
}
}
}
ExplainAction::Skip
}
CacheDescriptor::Tree { entries } => {
for entry in &entries {
let p = Path::new(&entry.path);
let needs_restore = !p.exists()
|| Self::calculate_checksum(p).ok().as_ref() != Some(&entry.checksum);
if needs_restore {
if self.has_object(&entry.checksum) {
return ExplainAction::Restore(RebuildReason::OutputMissing(entry.path.clone()));
} else {
return ExplainAction::Rebuild(RebuildReason::OutputMissing(entry.path.clone()));
}
}
}
ExplainAction::Skip
}
}
}
pub fn calculate_checksum(file_path: &Path) -> Result<String> {
checksum::file_checksum(file_path)
}
pub fn calculate_checksum_bytes(data: &[u8]) -> String {
checksum::bytes_checksum(data)
}
fn object_path(&self, checksum: &str) -> PathBuf {
let (prefix, rest) = checksum.split_at(CHECKSUM_PREFIX_LEN.min(checksum.len()));
self.objects_dir.join(prefix).join(rest)
}
fn store_object(&self, content: &[u8]) -> Result<String> {
let checksum = Self::calculate_checksum_bytes(content);
let object_path = self.object_path(&checksum);
if !object_path.exists() {
if let Some(parent) = object_path.parent() {
fs::create_dir_all(parent)
.context("Failed to create object directory")?;
}
let blob = if self.compression {
zstd::encode_all(content, 0)
.context("Failed to zstd-compress object")?
} else {
content.to_vec()
};
fs::write(&object_path, &blob)
.context("Failed to write object")?;
let mut perms = fs::metadata(&object_path)
.context("Failed to read object metadata")?
.permissions();
perms.set_readonly(true);
fs::set_permissions(&object_path, perms)
.context("Failed to set object read-only")?;
}
Ok(checksum)
}
fn has_object(&self, checksum: &str) -> bool {
self.object_path(checksum).exists()
}
fn restore_file(&self, checksum: &str, output_path: &Path) -> Result<()> {
let object_path = self.object_path(checksum);
if self.compression {
let content = self.read_object(checksum)
.with_context(|| format!("Failed to read cached object: {}", checksum))?;
fs::write(output_path, &content)
.with_context(|| format!("Failed to write decompressed output: {}", output_path.display()))?;
crate::platform::set_permissions_mode(output_path, 0o644)
.context("Failed to make restored file writable")?;
return Ok(());
}
match self.restore_method {
RestoreMethod::Hardlink => {
fs::hard_link(&object_path, output_path)
.with_context(|| format!("Failed to hard link from cache: {}. If on a cross-filesystem setup, set restore_method = \"copy\" in rsconstruct.toml.", checksum))?;
}
RestoreMethod::Copy => {
fs::copy(&object_path, output_path)
.with_context(|| format!("Failed to copy from cache: {}", checksum))?;
crate::platform::set_permissions_mode(output_path, 0o644)
.context("Failed to make restored file writable")?;
}
RestoreMethod::Auto => unreachable!("Auto should be resolved before use"),
}
Ok(())
}
pub(crate) fn read_object(&self, checksum: &str) -> Result<Vec<u8>> {
let object_path = self.object_path(checksum);
let raw = fs::read(&object_path)
.with_context(|| format!("Failed to read object: {}", checksum))?;
if self.compression {
zstd::decode_all(raw.as_slice())
.with_context(|| format!("Failed to decompress object: {}", checksum))
} else {
Ok(raw)
}
}
fn path_string(path: &Path) -> String {
path.display().to_string()
}
}