1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
use super::{CacheEntry, ObjectStore, OutputEntry, walk_files};
impl ObjectStore {
/// Try to restore outputs from cache (local first, then remote).
/// Returns true if all outputs were restored (or for checkers, if cache entry is valid).
///
/// # Behavior by processor type
///
/// - **Generators** (non-empty `output_paths`): Restores output files from cached objects
/// via hardlink or copy. Returns true only if all outputs were successfully restored.
///
/// - **Checkers** (empty `output_paths`): No files to restore. Returns true if a cache
/// entry exists with matching input checksum, indicating the check previously passed.
/// This allows checkers to skip re-running after `rsconstruct clean && rsconstruct build`.
pub fn restore_from_cache(&self, cache_key: &str, input_checksum: &str, output_paths: &[PathBuf]) -> Result<bool> {
// For checkers (empty outputs), just verify cache entry exists with matching checksum.
// The cache entry itself serves as the "success marker" - no files need restoration.
if output_paths.is_empty() {
return Ok(self.get_entry(cache_key)
.map(|e| e.input_checksum == input_checksum)
.unwrap_or(false));
}
// Check if we have a cache entry with matching input checksum
let entry = match self.get_entry(cache_key) {
Some(e) if e.input_checksum == input_checksum => Some(e),
_ => {
// Try to fetch from remote if enabled
if self.remote_pull {
self.try_fetch_from_remote(cache_key, input_checksum)?
} else {
None
}
}
};
let entry = match entry {
Some(e) => e,
None => return Ok(false),
};
// Try to restore each missing output
for output_path in output_paths {
if output_path.exists() {
continue;
}
let rel_path = Self::path_string(output_path);
let cached_output = entry.outputs.iter()
.find(|o| o.path == rel_path);
match cached_output {
Some(out) => {
// Ensure object is available locally (fetch from remote if needed)
if !self.has_object(&out.checksum)
&& (!self.remote_pull || !self.try_fetch_object_from_remote(&out.checksum)?)
{
return Ok(false);
}
if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)?;
}
self.restore_file(&out.checksum, output_path)?;
}
None => return Ok(false),
}
}
Ok(true)
}
/// Try to fetch a cache entry from remote cache
fn try_fetch_from_remote(&self, cache_key: &str, input_checksum: &str) -> Result<Option<CacheEntry>> {
let remote = match &self.remote {
Some(r) => r,
None => return Ok(None),
};
let remote_key = format!("index/{}", cache_key);
let data = match remote.download_bytes(&remote_key)? {
Some(d) => d,
None => return Ok(None),
};
let entry: CacheEntry = match serde_json::from_slice(&data) {
Ok(e) => e,
Err(_) => return Ok(None),
};
// Verify the input checksum matches what we expect
if entry.input_checksum != input_checksum {
return Ok(None);
}
// Store the entry locally for future use
self.insert_entry(cache_key, &entry)?;
Ok(Some(entry))
}
/// Try to fetch an object from remote cache
fn try_fetch_object_from_remote(&self, checksum: &str) -> Result<bool> {
let remote = match &self.remote {
Some(r) => r,
None => return Ok(false),
};
let object_path = self.object_path(checksum);
if object_path.exists() {
return Ok(true);
}
let (prefix, rest) = checksum.split_at(super::CHECKSUM_PREFIX_LEN.min(checksum.len()));
let remote_key = format!("objects/{}/{}", prefix, rest);
let fetched = remote.download(&remote_key, &object_path)?;
// Make fetched object read-only to prevent corruption via hardlinks
if fetched {
let mut perms = fs::metadata(&object_path)
.context("Failed to read fetched object metadata")?
.permissions();
perms.set_readonly(true);
fs::set_permissions(&object_path, perms)
.context("Failed to set fetched object read-only")?;
}
Ok(fetched)
}
/// Cache the outputs of a successful build.
/// Returns `Ok(true)` if any output content changed compared to the previous
/// cache entry, `Ok(false)` if all outputs are content-identical.
pub fn cache_outputs(&self, cache_key: &str, input_checksum: &str, output_paths: &[PathBuf]) -> Result<bool> {
// Get previous entry for comparison
let prev_entry = self.get_entry(cache_key);
let mut outputs = Vec::new();
let mut any_changed = false;
for output_path in output_paths {
anyhow::ensure!(output_path.exists(),
"Expected output file not produced: {}", output_path.display());
let content = fs::read(output_path)?;
let checksum = self.store_object(&content)?;
let rel_path = Self::path_string(output_path);
// Check if this output changed vs previous entry
if !any_changed {
let prev_checksum = prev_entry.as_ref().and_then(|e| {
e.outputs.iter().find(|o| o.path == rel_path).map(|o| &o.checksum)
});
if prev_checksum != Some(&checksum) {
any_changed = true;
}
}
// Push object to remote cache if enabled
if self.remote_push {
self.try_push_object_to_remote(&checksum)?;
}
outputs.push(OutputEntry {
path: rel_path,
checksum,
mode: None,
});
}
// For checkers (empty outputs), nothing changed
if output_paths.is_empty() {
any_changed = false;
}
// Check if number of outputs changed
if !any_changed {
if let Some(ref prev) = prev_entry {
if prev.outputs.len() != outputs.len() {
any_changed = true;
}
} else {
// No previous entry means this is new
any_changed = true;
}
}
let entry = CacheEntry {
input_checksum: input_checksum.to_string(),
outputs,
};
self.insert_entry(cache_key, &entry)?;
// Push index entry to remote cache if enabled
if self.remote_push {
self.try_push_entry_to_remote(cache_key, &entry)?;
}
Ok(any_changed)
}
/// Try to push an object to remote cache (ignores errors)
fn try_push_object_to_remote(&self, checksum: &str) -> Result<()> {
let remote = match &self.remote {
Some(r) => r,
None => return Ok(()),
};
let object_path = self.object_path(checksum);
if !object_path.exists() {
return Ok(());
}
let (prefix, rest) = checksum.split_at(super::CHECKSUM_PREFIX_LEN.min(checksum.len()));
let remote_key = format!("objects/{}/{}", prefix, rest);
// Check if already exists remotely (avoid redundant uploads)
if remote.exists(&remote_key).unwrap_or(false) {
return Ok(());
}
// Upload (ignore errors - remote cache is best-effort)
if let Err(e) = remote.upload(&remote_key, &object_path) {
eprintln!("Warning: failed to push to remote cache: {}", e);
}
Ok(())
}
/// Try to push a cache entry to remote cache (ignores errors)
fn try_push_entry_to_remote(&self, cache_key: &str, entry: &CacheEntry) -> Result<()> {
let remote = match &self.remote {
Some(r) => r,
None => return Ok(()),
};
let remote_key = format!("index/{}", cache_key);
let data = serde_json::to_vec(entry)
.context("Failed to serialize cache entry for remote")?;
// Upload (ignore errors - remote cache is best-effort)
if let Err(e) = remote.upload_bytes(&remote_key, &data) {
eprintln!("Warning: failed to push index to remote cache: {}", e);
}
Ok(())
}
/// Cache all files under an output directory.
/// Walks the directory, stores each file as an object, and records the manifest.
/// Returns `Ok(true)` if any output content changed compared to the previous cache entry.
pub fn cache_output_dir(&self, cache_key: &str, input_checksum: &str, dir: &Path) -> Result<bool> {
anyhow::ensure!(dir.exists() && dir.is_dir(),
"Expected output directory not produced: {}", dir.display());
let prev_entry = self.get_entry(cache_key);
let files = walk_files(dir);
let mut outputs = Vec::new();
let mut any_changed = false;
for file_path in files {
let content = fs::read(&file_path)
.with_context(|| format!("Failed to read output file: {}", file_path.display()))?;
let checksum = self.store_object(&content)?;
let rel_path = file_path.strip_prefix(dir)
.with_context(|| format!("File {} is not under dir {}", file_path.display(), dir.display()))?
.display().to_string();
let mode = fs::metadata(&file_path)
.ok()
.and_then(|m| crate::platform::get_mode(&m));
if !any_changed {
let prev_checksum = prev_entry.as_ref().and_then(|e| {
e.outputs.iter().find(|o| o.path == rel_path).map(|o| &o.checksum)
});
if prev_checksum != Some(&checksum) {
any_changed = true;
}
}
if self.remote_push {
self.try_push_object_to_remote(&checksum)?;
}
outputs.push(OutputEntry {
path: rel_path,
checksum,
mode,
});
}
// Check if number of outputs changed
if !any_changed {
if let Some(ref prev) = prev_entry {
if prev.outputs.len() != outputs.len() {
any_changed = true;
}
} else {
any_changed = true;
}
}
let entry = CacheEntry {
input_checksum: input_checksum.to_string(),
outputs,
};
self.insert_entry(cache_key, &entry)?;
if self.remote_push {
self.try_push_entry_to_remote(cache_key, &entry)?;
}
Ok(any_changed)
}
/// Restore an output directory from cache.
/// Recreates the entire directory structure from cached objects.
/// Returns `Ok(true)` if restore succeeded, `Ok(false)` if no matching cache entry.
pub fn restore_output_dir(&self, cache_key: &str, input_checksum: &str, dir: &Path) -> Result<bool> {
let entry = match self.get_entry(cache_key) {
Some(e) if e.input_checksum == input_checksum => Some(e),
_ => {
if self.remote_pull {
self.try_fetch_from_remote(cache_key, input_checksum)?
} else {
None
}
}
};
let entry = match entry {
Some(e) => e,
None => return Ok(false),
};
// Clean existing directory for a fresh restore
if dir.exists() {
fs::remove_dir_all(dir)
.with_context(|| format!("Failed to remove existing output dir: {}", dir.display()))?;
}
for output in &entry.outputs {
// Ensure object is available locally
if !self.has_object(&output.checksum)
&& (!self.remote_pull || !self.try_fetch_object_from_remote(&output.checksum)?)
{
return Ok(false);
}
let file_path = dir.join(&output.path);
if let Some(parent) = file_path.parent() {
fs::create_dir_all(parent)?;
}
self.restore_file(&output.checksum, &file_path)?;
// Restore file permissions if stored
if let Some(mode) = output.mode {
crate::platform::set_permissions_mode(&file_path, mode)
.with_context(|| format!("Failed to set permissions on {}", file_path.display()))?;
}
}
Ok(true)
}
}