use anyhow::{Result, bail};
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use crate::config::TermsConfig;
use crate::file_index::FileIndex;
use crate::graph::{BuildGraph, Product};
use crate::processors::{discover_checker_products, execute_checker_batch};
pub struct TermsProcessor {
config: TermsConfig,
}
impl TermsProcessor {
pub fn new(config: TermsConfig) -> Self {
Self { config }
}
fn execute_product(&self, product: &Product) -> Result<()> {
self.check_files(&[product.primary_input()])
}
fn check_files(&self, files: &[&Path]) -> Result<()> {
let terms = load_terms(&self.config.terms_dir)?;
if terms.is_empty() {
return Ok(());
}
let sorted = sorted_terms(&terms);
let mut bad_files: Vec<String> = Vec::new();
for file in files {
let unquoted = check_file_detail(file, &terms, &sorted)?;
if !unquoted.is_empty() {
let mut entry = file.display().to_string();
let mut unique_terms: Vec<&str> = unquoted.iter().map(|s| s.as_str()).collect();
unique_terms.sort();
unique_terms.dedup();
entry.push_str(&format!(": {}", unique_terms.join(", ")));
bad_files.push(entry);
}
}
if bad_files.is_empty() {
Ok(())
} else {
bail!(
"{} file(s) have term issues (run `rsconstruct terms fix` to fix):\n{}",
bad_files.len(),
bad_files.join("\n"),
)
}
}
}
impl crate::processors::Processor for TermsProcessor {
fn scan_config(&self) -> &crate::config::StandardConfig {
&self.config.standard
}
fn standard_config(&self) -> Option<&crate::config::StandardConfig> {
Some(&self.config.standard)
}
fn description(&self) -> &str {
"Check that technical terms are backtick-quoted in markdown files"
}
fn is_native(&self) -> bool { true }
fn auto_detect(&self, file_index: &FileIndex) -> bool {
Path::new(&self.config.terms_dir).is_dir()
&& !file_index.scan(&self.config.standard, true).is_empty()
}
fn discover(
&self,
graph: &mut BuildGraph,
file_index: &FileIndex,
instance_name: &str,
) -> Result<()> {
if !Path::new(&self.config.terms_dir).is_dir() {
return Ok(());
}
let mut dep_inputs = self.config.standard.dep_inputs.clone();
for entry in crate::errors::ctx(fs::read_dir(&self.config.terms_dir), &format!("Failed to read terms directory {}", self.config.terms_dir))? {
let entry = entry?;
let path = entry.path();
if path.extension().is_some_and(|e| e == "txt") {
dep_inputs.push(path.to_string_lossy().into_owned());
}
}
discover_checker_products(
graph, &self.config.standard, file_index, &dep_inputs,
&self.config.standard.dep_auto, &self.config, instance_name,
)
}
fn execute(&self, _ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()> {
self.execute_product(product)
}
fn supports_batch(&self) -> bool {
self.config.standard.batch
}
fn execute_batch(&self, ctx: &crate::build_context::BuildContext, products: &[&Product]) -> Vec<Result<()>> {
execute_checker_batch(ctx, products, |_ctx, files| self.check_files(files))
}
}
pub fn load_terms(terms_dir: &str) -> Result<HashSet<String>> {
let dir = Path::new(terms_dir);
if !dir.is_dir() {
bail!("terms_dir `{}` does not exist or is not a directory", terms_dir);
}
let mut seen: std::collections::HashMap<String, (String, usize)> = std::collections::HashMap::new();
let mut duplicates = Vec::new();
let mut entries: Vec<_> = crate::errors::ctx(fs::read_dir(dir), &format!("Failed to read terms directory {}", dir.display()))?
.filter_map(|e| e.ok())
.collect();
entries.sort_by_key(|e| e.path());
for entry in &entries {
let path = entry.path();
if path.extension().is_some_and(|e| e == "txt") {
let filename = path.file_name().unwrap().to_string_lossy().to_string();
let content = crate::errors::ctx(fs::read_to_string(&path), &format!("Failed to read terms file: {}", path.display()))?;
for (line_idx, line) in content.lines().enumerate() {
let term = line.trim();
if term.is_empty() {
continue;
}
if let Some((prev_file, prev_line)) = seen.get(term) {
duplicates.push(format!(
" `{}` in {}:{} (first seen in {}:{})",
term, filename, line_idx + 1, prev_file, prev_line,
));
} else {
seen.insert(term.to_string(), (filename.clone(), line_idx + 1));
}
}
}
}
if !duplicates.is_empty() {
bail!("Duplicate terms in {}:\n{}", terms_dir, duplicates.join("\n"));
}
Ok(seen.into_keys().collect())
}
fn sorted_terms(terms: &HashSet<String>) -> Vec<&str> {
let mut sorted: Vec<&str> = terms.iter().map(|s| s.as_str()).collect();
sorted.sort_by_key(|b| std::cmp::Reverse(b.len()));
sorted
}
fn excluded_ranges(text: &str) -> Vec<(usize, usize)> {
let mut ranges = Vec::new();
if text.starts_with("---\n") || text.starts_with("---\r\n") {
let skip = if text.as_bytes().get(3) == Some(&b'\r') { 5 } else { 4 };
if let Some(end_idx) = text[skip..].find("\n---") {
let mut end = skip + end_idx + 4; while end < text.len() && text.as_bytes()[end] != b'\n' {
end += 1;
}
if end < text.len() {
end += 1;
}
ranges.push((0, end));
}
}
let mut pos = 0;
let bytes = text.as_bytes();
while pos < bytes.len() {
if bytes[pos] == b'`' && pos + 2 < bytes.len() && bytes[pos + 1] == b'`' && bytes[pos + 2] == b'`' {
let start = pos;
pos += 3;
while pos < bytes.len() && bytes[pos] == b'`' {
pos += 1;
}
while pos < bytes.len() && bytes[pos] != b'\n' {
pos += 1;
}
loop {
if pos >= bytes.len() {
ranges.push((start, bytes.len()));
break;
}
if bytes[pos] == b'\n' || pos == 0 {
let line_start = if bytes[pos] == b'\n' { pos + 1 } else { pos };
if line_start + 2 < bytes.len()
&& bytes[line_start] == b'`'
&& bytes[line_start + 1] == b'`'
&& bytes[line_start + 2] == b'`'
{
let mut end = line_start + 3;
while end < bytes.len() && bytes[end] != b'\n' {
end += 1;
}
if end < bytes.len() {
end += 1;
}
ranges.push((start, end));
pos = end;
break;
}
}
pos += 1;
}
} else {
pos += 1;
}
}
let comment_open = b"<!--";
let comment_close = b"-->";
pos = 0;
while pos + 4 <= bytes.len() {
if &bytes[pos..pos + 4] == comment_open {
let start = pos;
pos += 4;
while pos + 3 <= bytes.len() {
if &bytes[pos..pos + 3] == comment_close {
pos += 3;
ranges.push((start, pos));
break;
}
pos += 1;
}
} else {
pos += 1;
}
}
ranges
}
fn backtick_span_ranges(text: &str, fenced: &[(usize, usize)]) -> Vec<(usize, usize)> {
let mut spans = Vec::new();
let bytes = text.as_bytes();
let mut i = 0;
while i < bytes.len() {
let mut in_fenced = false;
for &(fs, fe) in fenced {
if i >= fs && i < fe {
i = fe;
in_fenced = true;
break;
}
}
if in_fenced {
continue;
}
if bytes[i] == b'`' {
let open = i;
i += 1;
while i < bytes.len() && bytes[i] != b'`' && bytes[i] != b'\n' {
i += 1;
}
if i < bytes.len() && bytes[i] == b'`' && i > open {
spans.push((open, i + 1)); }
if i < bytes.len() {
i += 1;
}
} else {
i += 1;
}
}
spans
}
fn inside_ranges(pos: usize, end: usize, ranges: &[(usize, usize)]) -> bool {
ranges.iter().any(|&(s, e)| pos >= s && end <= e)
}
fn is_word_boundary(text: &[u8], pos: usize) -> bool {
if pos >= text.len() {
return true;
}
let ch = text[pos];
!ch.is_ascii_alphanumeric() && ch != b'_'
}
fn find_case_insensitive(haystack: &str, needle: &str, start: usize) -> Option<usize> {
let h = haystack.as_bytes();
let n = needle.as_bytes();
if n.is_empty() || start + n.len() > h.len() {
return None;
}
let mut i = start;
let limit = h.len() - n.len();
while i <= limit {
if h[i..i + n.len()].eq_ignore_ascii_case(n) {
return Some(i);
}
i += 1;
while i <= limit && !haystack.is_char_boundary(i) {
i += 1;
}
}
None
}
fn find_term_occurrences(text: &str, term: &str) -> Vec<(usize, usize)> {
let bytes = text.as_bytes();
let mut results = Vec::new();
let mut pos = 0;
while let Some(start) = find_case_insensitive(text, term, pos) {
let end = start + term.len();
let before_ok = start == 0 || is_word_boundary(bytes, start - 1);
let after_ok = end >= bytes.len() || is_word_boundary(bytes, end);
if before_ok && after_ok {
results.push((start, end));
pos = end;
} else {
pos = start + 1;
while pos < text.len() && !text.is_char_boundary(pos) {
pos += 1;
}
}
}
results
}
fn split_backticked(inner: &str) -> Vec<String> {
let mut results = Vec::new();
for part in inner.split(',') {
for tok in part.split(" and ").flat_map(|s| s.split(" or ")) {
let trimmed = tok.trim();
if !trimmed.is_empty() {
results.push(trimmed.to_string());
}
}
}
results
}
fn is_file_path(s: &str) -> bool {
if !s.contains('/') {
return false;
}
if let Some(last_slash) = s.rfind('/')
&& s[last_slash..].contains('.')
{
return true;
}
s.matches('/').count() > 1
}
fn looks_like_term_reference(inner: &str) -> bool {
let parts = split_backticked(inner);
if parts.is_empty() {
return false;
}
if is_file_path(inner) {
return false;
}
let code_chars = ['(', ')', '{', '}', '[', ']', ';', '=', '>', '<', '|', '\\', '"', '\'', '~'];
for part in &parts {
if part.contains(' ') || part.chars().any(|c| code_chars.contains(&c)) {
return false;
}
}
true
}
fn find_unquoted_positions(content: &str, sorted_terms: &[&str]) -> Vec<(usize, usize, String)> {
let fenced = excluded_ranges(content);
let backtick_spans = backtick_span_ranges(content, &fenced);
let mut claimed: Vec<(usize, usize)> = Vec::new();
let mut results = Vec::new();
for &term in sorted_terms {
for (start, end) in find_term_occurrences(content, term) {
if inside_ranges(start, end, &fenced) {
continue;
}
if inside_ranges(start, end, &backtick_spans) {
continue;
}
if claimed.iter().any(|&(cs, ce)| start < ce && end > cs) {
continue;
}
claimed.push((start, end));
results.push((start, end, content[start..end].to_string()));
}
}
results
}
fn find_non_tech_backticked_positions(content: &str, terms: &HashSet<String>) -> Vec<(usize, usize)> {
let fenced = excluded_ranges(content);
let spans = backtick_span_ranges(content, &fenced);
let mut results = Vec::new();
for &(start, end) in &spans {
let inner = &content[start + 1..end - 1];
if !looks_like_term_reference(inner) {
continue;
}
let parts = split_backticked(inner);
let all_non_tech = parts.iter().all(|p| {
!terms.iter().any(|t| t.eq_ignore_ascii_case(p))
});
if all_non_tech {
results.push((start, end));
}
}
results
}
fn apply_edits(content: &str, edits: &mut Vec<(usize, usize, String)>) -> String {
edits.sort_by(|a, b| b.0.cmp(&a.0));
edits.dedup_by(|a, b| a.1 > b.0);
let mut result = content.to_string();
for (start, end, replacement) in edits.iter() {
result = format!("{}{}{}", &result[..*start], replacement, &result[*end..]);
}
result
}
fn fix_content(original: &str, terms: &HashSet<String>, sorted_terms: &[&str], remove_non_terms: bool) -> String {
let cleaned = if remove_non_terms {
let mut removals: Vec<(usize, usize, String)> = find_non_tech_backticked_positions(original, terms)
.into_iter()
.map(|(s, e)| (s, e, original[s + 1..e - 1].to_string()))
.collect();
if removals.is_empty() {
original.to_string()
} else {
apply_edits(original, &mut removals)
}
} else {
original.to_string()
};
let mut additions: Vec<(usize, usize, String)> = find_unquoted_positions(&cleaned, sorted_terms)
.into_iter()
.map(|(s, e, m)| (s, e, format!("`{}`", m)))
.collect();
if additions.is_empty() {
cleaned
} else {
apply_edits(&cleaned, &mut additions)
}
}
fn check_file_detail(path: &Path, _terms: &HashSet<String>, sorted_terms: &[&str]) -> Result<Vec<String>> {
let content = crate::errors::ctx(fs::read_to_string(path), &format!("Failed to read {}", path.display()))?;
let matches = find_unquoted_positions(&content, sorted_terms);
Ok(matches.into_iter().map(|(_, _, term)| term).collect())
}
pub fn fix_file(path: &Path, terms: &HashSet<String>, sorted_terms: &[&str], remove_non_terms: bool) -> Result<bool> {
let original = crate::errors::ctx(fs::read_to_string(path), &format!("Failed to read {}", path.display()))?;
let fixed = fix_content(&original, terms, sorted_terms, remove_non_terms);
if fixed != original {
crate::errors::ctx(fs::write(path, &fixed), &format!("Failed to write {}", path.display()))?;
Ok(true)
} else {
Ok(false)
}
}
pub fn fix_all(config: &TermsConfig, remove_non_terms: bool) -> Result<()> {
let terms = load_terms(&config.terms_dir)?;
if terms.is_empty() {
println!("No technical terms found in {}", config.terms_dir);
return Ok(());
}
let sorted = sorted_terms(&terms);
let file_index = FileIndex::build()?;
let md_files = file_index.scan(&config.standard, true);
if md_files.is_empty() {
println!("No markdown files found");
return Ok(());
}
println!("Checking {} markdown files against {} terms...", md_files.len(), terms.len());
let mut modified_count = 0;
for file in &md_files {
if fix_file(file, &terms, &sorted, remove_non_terms)? {
modified_count += 1;
println!(" Fixed: {}", file.display());
}
}
println!("Done. Modified {} of {} files.", modified_count, md_files.len());
Ok(())
}
pub fn merge_terms(config: &TermsConfig, source_dir: &str) -> Result<()> {
let src = Path::new(source_dir);
if !src.is_dir() {
bail!("Source directory `{}` does not exist or is not a directory", source_dir);
}
let dest = Path::new(&config.terms_dir);
if !dest.is_dir() {
bail!("Terms directory `{}` does not exist or is not a directory", config.terms_dir);
}
let mut merged_count = 0;
let mut copied_count = 0;
for entry in crate::errors::ctx(fs::read_dir(src), &format!("Failed to read source directory {}", src.display()))? {
let entry = entry?;
let path = entry.path();
if path.extension().is_none_or(|e| e != "txt") {
continue;
}
let filename = path.file_name().unwrap();
let dest_path = dest.join(filename);
let source_content = crate::errors::ctx(fs::read_to_string(&path), &format!("Failed to read terms source: {}", path.display()))?;
let source_terms: HashSet<String> = source_content
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect();
if dest_path.exists() {
let dest_content = crate::errors::ctx(fs::read_to_string(&dest_path), &format!("Failed to read terms dest: {}", dest_path.display()))?;
let dest_terms: HashSet<String> = dest_content
.lines()
.map(|l| l.trim().to_string())
.filter(|l| !l.is_empty())
.collect();
let mut all_terms = dest_terms.clone();
all_terms.extend(source_terms.clone());
let mut sorted: Vec<String> = all_terms.into_iter().collect();
sorted.sort();
let content = sorted.join("\n") + "\n";
if content != source_content || content != dest_content {
crate::errors::ctx(fs::write(&dest_path, &content), &format!("Failed to write {}", dest_path.display()))?;
crate::errors::ctx(fs::write(&path, &content), &format!("Failed to write {}", path.display()))?;
merged_count += 1;
let added_to_dest = sorted.len() - dest_terms.len();
let added_to_src = sorted.len() - source_terms.len();
println!(" Merged: {} (+{} to dest, +{} to source)",
filename.to_string_lossy(), added_to_dest, added_to_src);
}
} else {
let mut sorted: Vec<String> = source_terms.into_iter().collect();
sorted.sort();
crate::errors::ctx(fs::write(&dest_path, sorted.join("\n") + "\n"), &format!("Failed to write {}", dest_path.display()))?;
copied_count += 1;
println!(" Copied to dest: {}", filename.to_string_lossy());
}
}
for entry in crate::errors::ctx(fs::read_dir(dest), &format!("Failed to read destination directory {}", dest.display()))? {
let entry = entry?;
let path = entry.path();
if path.extension().is_none_or(|e| e != "txt") {
continue;
}
let filename = path.file_name().unwrap();
let src_path = src.join(filename);
if !src_path.exists() {
crate::errors::ctx(fs::copy(&path, &src_path), &format!("Failed to copy {} to {}", path.display(), src_path.display()))?;
copied_count += 1;
println!(" Copied to source: {}", filename.to_string_lossy());
}
}
println!("Done. Merged {} file(s), copied {} new file(s).", merged_count, copied_count);
Ok(())
}
pub fn stats(config: &TermsConfig) -> Result<()> {
let dir = Path::new(&config.terms_dir);
if !dir.is_dir() {
bail!("terms_dir `{}` does not exist or is not a directory", config.terms_dir);
}
let mut file_count = 0;
let mut total_terms = 0;
for entry in crate::errors::ctx(fs::read_dir(dir), &format!("Failed to read terms directory {}", dir.display()))? {
let entry = entry?;
if entry.path().extension().is_some_and(|e| e == "txt") {
file_count += 1;
let content = crate::errors::ctx(fs::read_to_string(entry.path()), &format!("Failed to read {}", entry.path().display()))?;
total_terms += content.lines().filter(|l| !l.trim().is_empty()).count();
}
}
println!("{} term file(s), {} total terms", file_count, total_terms);
Ok(())
}
fn plugin_create(toml: &toml::Value) -> anyhow::Result<Box<dyn crate::processors::Processor>> {
crate::registries::deserialize_and_create(toml, |cfg| Box::new(TermsProcessor::new(cfg)))
}
inventory::submit! {
crate::registries::ProcessorPlugin {
version: 1,
name: "terms",
processor_type: crate::processors::ProcessorType::Checker,
create: plugin_create,
defconfig_json: crate::registries::default_config_json::<crate::config::TermsConfig>,
known_fields: crate::registries::typed_known_fields::<crate::config::TermsConfig>,
output_fields: crate::registries::typed_output_fields::<crate::config::TermsConfig>,
must_fields: crate::registries::typed_must_fields::<crate::config::TermsConfig>,
field_descriptions: crate::registries::typed_field_descriptions::<crate::config::TermsConfig>,
keywords: &["checker", "terminology", "text", "words"],
description: "Check that technical terms are backtick-quoted in markdown files",
is_native: true,
can_fix: false,
}
}