1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
use anyhow::Result;
use std::path::{Path, PathBuf};
use crate::config::StandardConfig;
#[derive(Debug, Clone)]
pub struct FileIndex {
files: Vec<PathBuf>,
}
#[cfg(test)]
impl FileIndex {
/// Create a `FileIndex` from an explicit list of paths (for testing).
fn from_paths(mut files: Vec<PathBuf>) -> Self {
files.sort();
Self { files }
}
}
impl FileIndex {
/// Build a file index by walking the current directory once.
/// Uses `ignore::WalkBuilder` which natively handles `.gitignore` and
/// `.rsconstructignore` (via `add_custom_ignore_filename`).
/// All paths are stored relative to project root (cwd).
pub fn build() -> Result<Self> {
Self::build_with_force_dirs(&[])
}
/// Build a file index, but also walk the listed directories ignoring
/// gitignore/rsconstructignore. Use this when the user has explicitly
/// listed a directory in a processor's `src_dirs` — that listing is an
/// opt-in to scan it even if it would otherwise be ignored.
///
/// Common case: `src_dirs = ["out/generator"]` where `out/` is in
/// `.gitignore` because it holds generated files. The user explicitly
/// wants those generated files scanned (e.g. for terms checking).
pub fn build_with_force_dirs(force_dirs: &[&str]) -> Result<Self> {
let walker = ignore::WalkBuilder::new(".")
.add_custom_ignore_filename(".rsconstructignore")
.hidden(false) // don't skip hidden files by default (let .gitignore handle it)
.build();
let mut files: Vec<PathBuf> = Vec::new();
for entry in walker {
let entry = crate::errors::ctx(entry, "Failed to read directory entry during file indexing")?;
if entry.file_type().is_some_and(|ft| ft.is_file()) {
let path = entry.into_path();
// Store relative paths (strip "./" prefix)
let relative = path.strip_prefix(".")
.unwrap_or(&path)
.to_path_buf();
files.push(relative);
}
}
// Walk force_dirs unconditionally — these are user-declared and
// opt-in, so gitignore should not filter them out.
for dir in force_dirs {
if dir.is_empty() {
continue;
}
let path = Path::new(dir);
if !path.is_dir() {
continue;
}
let walker = ignore::WalkBuilder::new(path)
.standard_filters(false) // ignore .gitignore, .ignore, hidden, etc.
.build();
for entry in walker {
let entry = crate::errors::ctx(entry, &format!("Failed to read directory entry under forced dir '{dir}'"))?;
if entry.file_type().is_some_and(|ft| ft.is_file()) {
let path = entry.into_path();
let relative = path.strip_prefix(".")
.unwrap_or(&path)
.to_path_buf();
files.push(relative);
}
}
}
files.sort();
files.dedup();
Ok(Self { files })
}
/// Query the index for files matching the given criteria.
/// All paths in the index are relative to project root.
///
/// - `root`: only include files under this directory (relative path, e.g., "src" or "")
/// - `extensions`: file extensions to match (e.g., `[".py", ".pyi"]`)
/// - `src_exclude_dirs`: directory path segments to skip (e.g., `["/.git/", "/out/"]`)
/// - `src_exclude_files`: file names to skip (e.g., `["setup.py"]`)
/// - `src_exclude_paths`: paths relative to project root to skip (e.g., `["Makefile"]`)
/// - `src_files`: if non-empty, only these paths are matched (allowlist)
pub fn query(
&self,
root: &Path,
extensions: &[&str],
src_exclude_dirs: &[&str],
src_exclude_files: &[&str],
src_exclude_paths: &[&str],
src_files: &[&str],
) -> Vec<PathBuf> {
self.files
.iter()
.filter(|path| {
// src_files: additional explicit files included alongside normal scanning
// Checked first — these bypass root and extension checks
if !src_files.is_empty() {
let path_str = path.to_string_lossy();
if src_files.iter().any(|p| *p == path_str) {
return true;
}
}
// Must be under root (root is relative, e.g., "src" or "")
// Empty root or "." means match all
let root_str = root.to_string_lossy();
if !root_str.is_empty() && root_str != "."
&& !path.starts_with(root) {
return false;
}
// Check exclude dirs
if !src_exclude_dirs.is_empty() {
let path_str = path.to_string_lossy();
if src_exclude_dirs.iter().any(|dir| path_str.contains(dir)) {
return false;
}
}
// Check extension match
// Extensions starting with "." match suffixes (e.g., ".py" matches "foo.py").
// Extensions without a leading "." are exact filenames (e.g., "Makefile", "requirements.txt").
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
if !extensions.iter().any(|ext| {
if ext.starts_with('.') { name.ends_with(ext) } else { name == *ext }
}) {
return false;
}
// Check exclude files
if !src_exclude_files.is_empty() && src_exclude_files.contains(&name) {
return false;
}
// Check exclude paths (paths are already relative)
if !src_exclude_paths.is_empty() {
let path_str = path.to_string_lossy();
if src_exclude_paths.iter().any(|p| *p == path_str) {
return false;
}
}
true
})
.cloned()
.collect()
}
/// Convenience wrapper using `StandardConfig` scan fields.
/// Returns relative paths.
///
/// - `scan`: processor scan configuration
/// - `recursive`: if false, only include files at depth 1 from the scan root
pub fn scan(
&self,
scan: &StandardConfig,
recursive: bool,
) -> Vec<PathBuf> {
let ext_refs: Vec<&str> = scan.src_extensions().iter().map(std::string::String::as_str).collect();
let exclude_dir_refs: Vec<&str> = scan.src_exclude_dirs().iter().map(std::string::String::as_str).collect();
let exclude_file_refs: Vec<&str> = scan.src_exclude_files().iter().map(std::string::String::as_str).collect();
let exclude_path_refs: Vec<&str> = scan.src_exclude_paths().iter().map(std::string::String::as_str).collect();
let include_path_refs: Vec<&str> = scan.src_files().iter().map(std::string::String::as_str).collect();
let mut results = Vec::new();
let src_dirs = scan.src_dirs();
// When src_dirs is empty but src_files is set, scan from project root
// so that query() can match the explicit file paths
let effective_dirs: Vec<&str> = if src_dirs.is_empty() && !include_path_refs.is_empty() {
vec![""]
} else {
src_dirs.iter().map(std::string::String::as_str).collect()
};
for dir in &effective_dirs {
// Normalize "." to "" so depth calculations work correctly
// (files in the index are stored as relative paths without "./" prefix)
let root = if *dir == "." || dir.is_empty() { PathBuf::new() } else { PathBuf::from(dir) };
let mut dir_results = self.query(&root, &ext_refs, &exclude_dir_refs, &exclude_file_refs, &exclude_path_refs, &include_path_refs);
if !recursive {
// Filter to depth 1 from scan root: keep only files whose path has
// exactly one more component than the root.
let root_depth = root.components().count();
dir_results.retain(|path| {
path.components().count() == root_depth + 1
});
}
results.append(&mut dir_results);
}
results.sort();
results.dedup();
results
}
/// Add virtual files (declared outputs from generators) to the index.
/// Used by the fixed-point discovery loop so downstream processors can
/// discover products for files that don't exist on disk yet.
/// Returns the number of files actually added (not already present).
pub fn add_virtual_files(&mut self, paths: &[PathBuf]) -> usize {
// Collect first, insert after: pushing while binary-searching would
// unsort the vec and make later searches (including duplicates within
// `paths`) unreliable.
let mut to_add: Vec<PathBuf> = paths.iter()
.filter(|p| self.files.binary_search(p).is_err())
.cloned()
.collect();
to_add.sort();
to_add.dedup();
let added = to_add.len();
if added > 0 {
self.files.extend(to_add);
self.files.sort();
self.files.dedup();
}
added
}
/// Return all files in the index.
pub fn files(&self) -> &[PathBuf] {
&self.files
}
/// Check if the index contains any file with the given extension.
/// Extension should include the dot, e.g., ".py", ".c".
pub fn has_extension(&self, ext: &str) -> bool {
self.files.iter().any(|path| {
path.file_name()
.and_then(|n| n.to_str())
.is_some_and(|name| name.ends_with(ext))
})
}
/// Check if a specific path exists in the index.
/// Uses binary search since the file list is sorted.
pub fn contains(&self, path: &Path) -> bool {
self.files.binary_search_by(|p| p.as_path().cmp(path)).is_ok()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_index() -> FileIndex {
FileIndex::from_paths(vec![
"src/main.c".into(),
"src/lib.c".into(),
"src/util/helper.c".into(),
"tests/test_main.py".into(),
"tests/test_lib.py".into(),
"README.md".into(),
"Makefile".into(),
"out/build/app.o".into(),
])
}
#[test]
fn contains_finds_existing_path() {
let idx = sample_index();
assert!(idx.contains(Path::new("src/main.c")));
assert!(idx.contains(Path::new("README.md")));
}
#[test]
fn contains_rejects_missing_path() {
let idx = sample_index();
assert!(!idx.contains(Path::new("nonexistent.c")));
assert!(!idx.contains(Path::new("src/missing.c")));
}
#[test]
fn has_extension_finds_existing() {
let idx = sample_index();
assert!(idx.has_extension(".c"));
assert!(idx.has_extension(".py"));
assert!(idx.has_extension(".md"));
}
#[test]
fn has_extension_rejects_missing() {
let idx = sample_index();
assert!(!idx.has_extension(".rs"));
assert!(!idx.has_extension(".java"));
}
#[test]
fn query_filters_by_extension() {
let idx = sample_index();
let results = idx.query(Path::new(""), &[".c"], &[], &[], &[], &[]);
assert_eq!(results.len(), 3); // main.c, lib.c, helper.c
assert!(results.iter().all(|p| p.to_string_lossy().ends_with(".c")));
}
#[test]
fn query_filters_by_root() {
let idx = sample_index();
let results = idx.query(Path::new("tests"), &[".py"], &[], &[], &[], &[]);
assert_eq!(results.len(), 2);
assert!(results.iter().all(|p| p.starts_with("tests")));
}
#[test]
fn query_excludes_dirs() {
let idx = sample_index();
let results = idx.query(Path::new(""), &[".c", ".o"], &["/util/"], &[], &[], &[]);
assert!(!results.iter().any(|p| p.to_string_lossy().contains("/util/")));
}
#[test]
fn query_excludes_files() {
let idx = sample_index();
let results = idx.query(Path::new(""), &[".c"], &[], &["lib.c"], &[], &[]);
assert!(!results.iter().any(|p| p.file_name().unwrap() == "lib.c"));
assert!(results.iter().any(|p| p.file_name().unwrap() == "main.c"));
}
#[test]
fn query_excludes_paths() {
let idx = sample_index();
let results = idx.query(Path::new(""), &[".c"], &[], &[], &["src/main.c"], &[]);
assert!(!results.contains(&PathBuf::from("src/main.c")));
assert!(results.contains(&PathBuf::from("src/lib.c")));
}
#[test]
fn query_empty_root_matches_all() {
let idx = sample_index();
let results = idx.query(Path::new(""), &[".md"], &[], &[], &[], &[]);
assert_eq!(results, vec![PathBuf::from("README.md")]);
}
}