Skip to main content

kaniop_radius/reconcile/
cleanup.rs

1//! Finalizer for a `KanidmRadiusServer`.
2//!
3//! Order:
4//!
5//! 1. Delete our Deployment in the foreground.
6//! 2. Wait until no ReplicaSet or pod uses our config Secrets or tokens.
7//! 3. Remove our `idm_radius_servers` membership edge and delete the account
8//!    (which removes every token on it).
9//! 4. Remove the finalizer; owner-reference GC removes the config Secrets and
10//!    the Service.
11//!
12//! Skips: when the referenced Kanidm CR no longer exists, or the account
13//! exists without our ownership marker, the Kanidm cleanup is skipped with a
14//! Warning event and the finalizer is removed: deletion never blocks forever
15//! on something the CR does not own or cannot reach by design. A Kanidm CR
16//! that exists but is unreachable is a transient error and retries with
17//! backoff.
18//!
19//! [`next_step`] and [`identity_cleanup`] are pure; the reconciler performs
20//! one step per pass and requeues while waiting.
21
22use std::collections::HashMap;
23use std::sync::{Mutex, PoisonError};
24use std::time::{Duration, Instant};
25
26use crate::reconcile::error::{ExecError, ExecResult, Failure};
27use crate::reconcile::identity::DeleteOutcome;
28use crate::reconcile::status::ConditionKind;
29
30/// What the finalizer sees in one pass.
31#[derive(Clone, Debug, PartialEq, Eq)]
32pub struct CleanupView {
33    /// Our Deployment, if it still exists.
34    pub deployment: Option<DeploymentPhase>,
35    /// `replicaset/<name>` or `pod/<name>` still using our Secrets or tokens.
36    pub consumers: Vec<String>,
37    /// Whether the referenced Kanidm CR exists.
38    pub kanidm_present: bool,
39}
40
41#[derive(Clone, Copy, Debug, PartialEq, Eq)]
42pub enum DeploymentPhase {
43    Live,
44    /// Deletion started (foreground: it stays until its pods are gone).
45    Deleting,
46}
47
48/// Why the Kanidm cleanup was skipped.
49#[derive(Clone, Copy, Debug, PartialEq, Eq)]
50pub enum SkipReason {
51    KanidmGone,
52    AccountNotOwned,
53}
54
55impl SkipReason {
56    /// Event reason.
57    pub fn reason(self) -> &'static str {
58        match self {
59            Self::KanidmGone => "KanidmNotFound",
60            Self::AccountNotOwned => "AccountNotOwned",
61        }
62    }
63
64    /// Event note; names no credential.
65    pub fn note(self) -> &'static str {
66        match self {
67            Self::KanidmGone => {
68                "referenced Kanidm no longer exists; its RADIUS service account was not cleaned up"
69            }
70            Self::AccountNotOwned => {
71                "Kanidm account for this resource exists without its ownership marker; left untouched"
72            }
73        }
74    }
75}
76
77/// The next thing the finalizer does.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub enum CleanupStep {
80    DeleteDeployment,
81    AwaitDrain { consumers: Vec<String> },
82    DeleteIdentity,
83    Skip(SkipReason),
84}
85
86/// How the finalizer ends after the identity step.
87#[derive(Clone, Copy, Debug, PartialEq, Eq)]
88pub enum Finish {
89    RemoveFinalizer,
90    /// Emit a Warning event, then remove the finalizer.
91    SkipAndRemove(SkipReason),
92}
93
94/// Decide the next finalizer step. Pure.
95pub fn next_step(view: &CleanupView) -> CleanupStep {
96    match view.deployment {
97        Some(DeploymentPhase::Live) => CleanupStep::DeleteDeployment,
98        Some(DeploymentPhase::Deleting) => CleanupStep::AwaitDrain {
99            consumers: view.consumers.clone(),
100        },
101        None if !view.consumers.is_empty() => CleanupStep::AwaitDrain {
102            consumers: view.consumers.clone(),
103        },
104        None if view.kanidm_present => CleanupStep::DeleteIdentity,
105        None => CleanupStep::Skip(SkipReason::KanidmGone),
106    }
107}
108
109/// Event reason while deletion waits for consumers to stop.
110pub const DRAIN_WARNING_REASON: &str = "AwaitingConsumers";
111/// An unchanged set of consumers is warned about at most this often.
112pub const DRAIN_WARNING_INTERVAL: Duration = Duration::from_secs(10 * 60);
113/// Consumers named in one event note; the rest are counted.
114const DRAIN_NOTE_MAX_CONSUMERS: usize = 10;
115
116/// Event note naming the consumers deletion waits for. Bounded: at most
117/// [`DRAIN_NOTE_MAX_CONSUMERS`] names, the rest counted.
118pub fn drain_note(consumers: &[String]) -> String {
119    let mut named = consumers
120        .iter()
121        .take(DRAIN_NOTE_MAX_CONSUMERS)
122        .map(String::as_str)
123        .collect::<Vec<_>>()
124        .join(", ");
125    let rest = consumers.len().saturating_sub(DRAIN_NOTE_MAX_CONSUMERS);
126    if rest > 0 {
127        named.push_str(&format!(" and {rest} more"));
128    }
129    format!("deletion waits until these stop using the config Secrets or tokens: {named}")
130}
131
132/// The consumer set each deleting CR (by UID) was last warned about, and
133/// when. Kept in memory: the finalizer requeues every few seconds and an
134/// event per pass would flood the namespace, while an operator restart only
135/// repeats one warning.
136#[derive(Debug, Default)]
137pub struct DrainWarnings {
138    last: Mutex<HashMap<String, (Vec<String>, Instant)>>,
139}
140
141impl DrainWarnings {
142    /// Whether to warn about `consumers` now: the set changed since the last
143    /// warning or [`DRAIN_WARNING_INTERVAL`] passed. Records the warning when
144    /// it returns true. An empty set never warns.
145    pub fn should_warn(&self, uid: &str, consumers: &[String], now: Instant) -> bool {
146        if consumers.is_empty() {
147            return false;
148        }
149        let mut set = consumers.to_vec();
150        set.sort();
151        let mut last = self.last.lock().unwrap_or_else(PoisonError::into_inner);
152        // Entries past the interval decide nothing (they would warn anyway);
153        // dropping them bounds the map when a finalizer is removed by hand.
154        last.retain(|_, (_, at)| now.saturating_duration_since(*at) < DRAIN_WARNING_INTERVAL);
155        let due = last.get(uid).is_none_or(|(previous, at)| {
156            *previous != set || now.saturating_duration_since(*at) >= DRAIN_WARNING_INTERVAL
157        });
158        if due {
159            last.insert(uid.to_string(), (set, now));
160        }
161        due
162    }
163
164    /// Drop the record of a CR whose finalizer is done.
165    pub fn forget(&self, uid: &str) {
166        self.last
167            .lock()
168            .unwrap_or_else(PoisonError::into_inner)
169            .remove(uid);
170    }
171}
172
173/// Map the result of `delete_identity` to how the finalizer ends.
174/// Transient errors keep their source and retry with backoff.
175pub fn identity_cleanup(result: ExecResult<DeleteOutcome>) -> Result<Finish, Failure> {
176    match result {
177        Ok(DeleteOutcome::Deleted | DeleteOutcome::AlreadyAbsent) => Ok(Finish::RemoveFinalizer),
178        Err(ExecError::AccountNotOwned { .. }) => {
179            Ok(Finish::SkipAndRemove(SkipReason::AccountNotOwned))
180        }
181        Err(error) => Err(Failure::exec(ConditionKind::IdentityReady)(error)),
182    }
183}
184
185#[cfg(test)]
186mod tests {
187    use kaniop_k8s_util::error::Error;
188
189    use super::*;
190
191    fn view(deployment: Option<DeploymentPhase>, consumers: &[&str], kanidm: bool) -> CleanupView {
192        CleanupView {
193            deployment,
194            consumers: consumers.iter().map(|c| c.to_string()).collect(),
195            kanidm_present: kanidm,
196        }
197    }
198
199    #[test]
200    fn steps_run_in_order() {
201        use DeploymentPhase::*;
202        let cases = [
203            // 1. A live Deployment is deleted first, whatever else exists.
204            (
205                view(Some(Live), &["pod/a"], true),
206                CleanupStep::DeleteDeployment,
207            ),
208            (view(Some(Live), &[], false), CleanupStep::DeleteDeployment),
209            // 2. Then wait for the Deployment and every consumer to go.
210            (
211                view(Some(Deleting), &[], true),
212                CleanupStep::AwaitDrain { consumers: vec![] },
213            ),
214            (
215                view(None, &["replicaset/b", "pod/a"], true),
216                CleanupStep::AwaitDrain {
217                    consumers: vec!["replicaset/b".into(), "pod/a".into()],
218                },
219            ),
220            // Draining comes first even when Kanidm is gone.
221            (
222                view(None, &["pod/a"], false),
223                CleanupStep::AwaitDrain {
224                    consumers: vec!["pod/a".into()],
225                },
226            ),
227            // 3. Drained: clean up the Kanidm identity.
228            (view(None, &[], true), CleanupStep::DeleteIdentity),
229            // Kanidm CR gone: skip with a warning.
230            (
231                view(None, &[], false),
232                CleanupStep::Skip(SkipReason::KanidmGone),
233            ),
234        ];
235        for (view, expected) in cases {
236            assert_eq!(next_step(&view), expected, "{view:?}");
237        }
238    }
239
240    fn names(names: &[&str]) -> Vec<String> {
241        names.iter().map(|n| n.to_string()).collect()
242    }
243
244    #[test]
245    fn drain_warnings_only_on_change_or_after_the_interval() {
246        let warnings = DrainWarnings::default();
247        let t0 = Instant::now();
248        let set = names(&["pod/a", "replicaset/b"]);
249        assert!(!warnings.should_warn("uid-1", &[], t0), "nothing to name");
250        assert!(warnings.should_warn("uid-1", &set, t0), "first time");
251        assert!(!warnings.should_warn("uid-1", &set, t0 + Duration::from_secs(5)));
252        // Same set in another order is unchanged.
253        let reordered = names(&["replicaset/b", "pod/a"]);
254        assert!(!warnings.should_warn("uid-1", &reordered, t0 + Duration::from_secs(10)));
255        // Another CR is independent.
256        assert!(warnings.should_warn("uid-2", &set, t0 + Duration::from_secs(10)));
257        // A changed set warns at once.
258        let shrunk = names(&["pod/a"]);
259        assert!(warnings.should_warn("uid-1", &shrunk, t0 + Duration::from_secs(15)));
260        assert!(!warnings.should_warn("uid-1", &shrunk, t0 + Duration::from_secs(20)));
261        // An unchanged set warns again once the interval has passed.
262        let later = t0 + Duration::from_secs(15) + DRAIN_WARNING_INTERVAL;
263        assert!(warnings.should_warn("uid-1", &shrunk, later));
264        assert!(!warnings.should_warn("uid-1", &shrunk, later + Duration::from_secs(5)));
265        // Forgotten after the finalizer is done.
266        warnings.forget("uid-1");
267        assert!(warnings.should_warn("uid-1", &shrunk, later + Duration::from_secs(10)));
268    }
269
270    #[test]
271    fn drain_note_names_consumers_and_is_bounded() {
272        let note = drain_note(&names(&["pod/a", "replicaset/b"]));
273        assert!(note.contains("pod/a, replicaset/b"), "{note}");
274        let many: Vec<String> = (0..25).map(|i| format!("pod/p{i}")).collect();
275        let note = drain_note(&many);
276        assert!(note.contains("pod/p9"), "{note}");
277        assert!(!note.contains("pod/p10"), "{note}");
278        assert!(note.contains("and 15 more"), "{note}");
279    }
280
281    #[test]
282    fn identity_cleanup_outcomes() {
283        assert_eq!(
284            identity_cleanup(Ok(DeleteOutcome::Deleted)).unwrap(),
285            Finish::RemoveFinalizer
286        );
287        assert_eq!(
288            identity_cleanup(Ok(DeleteOutcome::AlreadyAbsent)).unwrap(),
289            Finish::RemoveFinalizer
290        );
291        // An account that is not ours is skipped, never deleted, and does
292        // not block deletion.
293        assert_eq!(
294            identity_cleanup(Err(ExecError::AccountNotOwned {
295                account: "kaniop_radius_x".into()
296            }))
297            .unwrap(),
298            Finish::SkipAndRemove(SkipReason::AccountNotOwned)
299        );
300        // An unreachable Kanidm retries with backoff and keeps its source.
301        match identity_cleanup(Err(ExecError::Api(Error::MissingData("down".into())))) {
302            Err(Failure::Transient { condition, error }) => {
303                assert_eq!(condition, ConditionKind::IdentityReady);
304                assert_eq!(error.to_string(), "down");
305            }
306            other => panic!("expected transient, got {other:?}"),
307        }
308    }
309}