Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::{CLUSTER_LABEL, statefulset::StatefulSetExt};
3
4use kaniop_k8s_util::client::get_output;
5use kaniop_k8s_util::error::{Error, Result};
6
7use std::collections::BTreeSet;
8use std::sync::Arc;
9use std::time::Duration;
10
11use futures::StreamExt;
12use k8s_openapi::api::apps::v1::StatefulSet;
13use k8s_openapi::api::batch::v1::{Job, JobSpec};
14use k8s_openapi::api::core::v1::{
15    ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
16    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, Volume, VolumeMount,
17};
18use k8s_openapi::api::storage::v1::VolumeAttachment;
19use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
20use k8s_openapi::jiff::Timestamp;
21use kube::api::{
22    AttachParams, DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams,
23};
24use kube::runtime::controller::{Action, Controller};
25use kube::runtime::events::{Event, EventType, Recorder, Reporter};
26use kube::runtime::finalizer::{Event as Finalizer, finalizer};
27use kube::runtime::watcher;
28use kube::{Api, Client, CustomResource, Resource, ResourceExt};
29use opentelemetry::metrics::{Counter, Histogram};
30use opentelemetry::{KeyValue, global};
31#[cfg(feature = "schemars")]
32use schemars::JsonSchema;
33use serde::{Deserialize, Serialize};
34use serde_json::json;
35use tracing::{debug, error, info, warn};
36
37pub const CONTROLLER_ID: &str = "kanidm-restore";
38pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
39const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
40const DATA_VOLUME: &str = "kanidm-data";
41const CONFIG_VOLUME: &str = "kanidm-config";
42const DATA_PATH: &str = "/data";
43const CONFIG_PATH: &str = "/run/kanidm";
44const BACKUP_PATH: &str = "/data/backups";
45const REQUEUE: Duration = Duration::from_secs(2);
46const CONDITION_TRUE: &str = "True";
47const CONDITION_FALSE: &str = "False";
48const CONDITION_PROGRESSING: &str = "Progressing";
49const CONDITION_READY: &str = "Ready";
50const CONDITION_FAILED: &str = "Failed";
51
52#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
53#[cfg_attr(feature = "schemars", derive(JsonSchema))]
54#[cfg_attr(
55    feature = "schemars",
56    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
57)]
58#[kube(
59    category = "kaniop",
60    group = "kaniop.rs",
61    version = "v1beta1",
62    kind = "KanidmRestore",
63    plural = "kanidmrestores",
64    singular = "kanidmrestore",
65    shortname = "idmrestore",
66    namespaced,
67    status = "KanidmRestoreStatus",
68    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
69    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
70    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
71)]
72#[serde(rename_all = "camelCase")]
73pub struct KanidmRestoreSpec {
74    pub target_ref: KanidmRestoreTargetRef,
75    pub source: KanidmRestoreSource,
76    /// Exact Kanidm image used for restore. It must equal the target image and may not use latest.
77    pub restore_image: String,
78}
79
80#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
81#[cfg_attr(feature = "schemars", derive(JsonSchema))]
82#[serde(rename_all = "camelCase")]
83pub struct KanidmRestoreTargetRef {
84    pub name: String,
85    pub uid: String,
86}
87
88#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
89#[cfg_attr(feature = "schemars", derive(JsonSchema))]
90#[serde(rename_all = "camelCase")]
91pub struct KanidmRestoreSource {
92    pub local: KanidmRestoreLocalSource,
93}
94
95#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
96#[cfg_attr(feature = "schemars", derive(JsonSchema))]
97#[serde(rename_all = "camelCase")]
98pub struct KanidmRestoreLocalSource {
99    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
100    pub file_name: String,
101}
102
103#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
104#[cfg_attr(feature = "schemars", derive(JsonSchema))]
105pub enum KanidmRestorePhase {
106    #[default]
107    Pending,
108    Validating,
109    Quiescing,
110    RestoringPrimary,
111    Verifying,
112    RebuildingReplicas,
113    Resuming,
114    Completed,
115    Failed,
116}
117
118#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
119#[cfg_attr(feature = "schemars", derive(JsonSchema))]
120#[serde(rename_all = "camelCase")]
121pub struct KanidmRestoreStatus {
122    #[serde(skip_serializing_if = "Option::is_none")]
123    pub observed_generation: Option<i64>,
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub observed_target_uid: Option<String>,
126    #[serde(default)]
127    pub phase: KanidmRestorePhase,
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub restore_job_name: Option<String>,
130    #[serde(skip_serializing_if = "Option::is_none")]
131    pub verify_job_name: Option<String>,
132    #[serde(default)]
133    pub replicas_cleared: bool,
134    /// Persisted before the restore Job may be created. Once true, finalizer cleanup
135    /// fails closed until the restore has completed.
136    #[serde(default)]
137    pub database_mutation_started: bool,
138    #[serde(default, skip_serializing_if = "Vec::is_empty")]
139    pub conditions: Vec<Condition>,
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub message: Option<String>,
142}
143
144#[derive(Clone)]
145struct RestoreMetrics {
146    attempts: Counter<u64>,
147    outcomes: Counter<u64>,
148    duration_seconds: Histogram<f64>,
149}
150
151impl RestoreMetrics {
152    fn new() -> Self {
153        let meter = global::meter("kaniop");
154        Self {
155            attempts: meter
156                .u64_counter("kanidm_restore_attempts")
157                .with_description("Number of Kanidm restore attempts started")
158                .build(),
159            outcomes: meter
160                .u64_counter("kanidm_restore_outcomes")
161                .with_description("Number of terminal Kanidm restore outcomes")
162                .build(),
163            duration_seconds: meter
164                .f64_histogram("kanidm_restore_duration_seconds")
165                .with_description("Kanidm restore duration from object creation to terminal phase")
166                .with_unit("s")
167                .build(),
168        }
169    }
170}
171
172#[derive(Clone)]
173struct RestoreContext {
174    client: Client,
175    recorder: Recorder,
176    metrics: RestoreMetrics,
177}
178
179pub async fn run(client: Client) {
180    let api = Api::<KanidmRestore>::all(client.clone());
181    let recorder = Recorder::new(
182        client.clone(),
183        Reporter {
184            controller: CONTROLLER_ID.into(),
185            instance: None,
186        },
187    );
188    let ctx = Arc::new(RestoreContext {
189        client,
190        recorder,
191        metrics: RestoreMetrics::new(),
192    });
193    info!("starting {CONTROLLER_ID} controller");
194    Controller::new(api, watcher::Config::default().any_semantic())
195        .shutdown_on_signal()
196        .run(reconcile_restore, error_policy, ctx)
197        .for_each(|result| async move {
198            if let Err(error) = result {
199                error!(%error, "KanidmRestore reconciliation failed");
200            }
201        })
202        .await;
203}
204
205fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
206    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
207    Action::requeue(Duration::from_secs(5))
208}
209
210async fn reconcile_restore(
211    restore: Arc<KanidmRestore>,
212    ctx: Arc<RestoreContext>,
213) -> Result<Action> {
214    let namespace = restore
215        .namespace()
216        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
217    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
218    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
219        let ctx = ctx.clone();
220        async move {
221            match event {
222                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
223                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
224            }
225        }
226    })
227    .await
228    .map_err(|error| {
229        Error::FinalizerError(
230            "failed on KanidmRestore finalizer".to_string(),
231            Box::new(error),
232        )
233    })
234}
235
236async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
237    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
238    match phase {
239        KanidmRestorePhase::Pending => {
240            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
241            Ok(Action::requeue(REQUEUE))
242        }
243        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
244            Ok(target) => {
245                mark_restoring(&restore, &target, &ctx).await?;
246                let mut status = restore.status.clone().unwrap_or_default();
247                status.observed_target_uid = target.uid();
248                status.phase = KanidmRestorePhase::Quiescing;
249                status.message = None;
250                patch_status(&restore, &ctx, status).await?;
251                Ok(Action::requeue(REQUEUE))
252            }
253            Err(error) => {
254                set_phase(
255                    &restore,
256                    &ctx,
257                    KanidmRestorePhase::Failed,
258                    Some(error.to_string()),
259                )
260                .await?;
261                Ok(Action::requeue(Duration::from_secs(300)))
262            }
263        },
264        KanidmRestorePhase::Quiescing => {
265            let target = get_target(&restore, &ctx).await?;
266            scale_all(&target, &ctx, 0).await?;
267            if target_pods_stopped(&target, &ctx).await?
268                && target_volumes_detached(&target, &ctx).await?
269            {
270                set_phase(&restore, &ctx, KanidmRestorePhase::RestoringPrimary, None).await?;
271            }
272            Ok(Action::requeue(REQUEUE))
273        }
274        KanidmRestorePhase::RestoringPrimary => {
275            let target = get_target(&restore, &ctx).await?;
276            ensure_restore_config(&restore, &target, &ctx).await?;
277            if !restore
278                .status
279                .as_ref()
280                .is_some_and(|status| status.database_mutation_started)
281            {
282                let mut status = restore.status.clone().unwrap_or_default();
283                status.database_mutation_started = true;
284                status.message = Some(
285                    "database mutation boundary persisted; starting primary restore".to_string(),
286                );
287                patch_status(&restore, &ctx, status).await?;
288                return Ok(Action::requeue(REQUEUE));
289            }
290            let name = restore_job_name(&restore);
291            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
292            match job_state(&restore, &ctx, &name).await? {
293                JobState::Complete => {
294                    let mut status = restore.status.clone().unwrap_or_default();
295                    status.restore_job_name = Some(name);
296                    status.phase = KanidmRestorePhase::Verifying;
297                    status.message = None;
298                    patch_status(&restore, &ctx, status).await?;
299                }
300                JobState::Failed => {
301                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
302                }
303                JobState::Running => {}
304            }
305            Ok(Action::requeue(REQUEUE))
306        }
307        KanidmRestorePhase::Verifying => {
308            let target = get_target(&restore, &ctx).await?;
309            let name = verify_job_name(&restore);
310            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
311            match job_state(&restore, &ctx, &name).await? {
312                JobState::Complete => {
313                    let mut status = restore.status.clone().unwrap_or_default();
314                    status.verify_job_name = Some(name);
315                    status.phase = KanidmRestorePhase::RebuildingReplicas;
316                    status.message = None;
317                    patch_status(&restore, &ctx, status).await?;
318                }
319                JobState::Failed => {
320                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
321                }
322                JobState::Running => {}
323            }
324            Ok(Action::requeue(REQUEUE))
325        }
326        KanidmRestorePhase::RebuildingReplicas => {
327            let target = get_target(&restore, &ctx).await?;
328            let mut status = restore.status.clone().unwrap_or_default();
329            if !status.replicas_cleared {
330                if delete_secondary_pvcs(&target, &ctx).await? {
331                    status.replicas_cleared = true;
332                    status.message = Some("secondary database state cleared".to_string());
333                    patch_status(&restore, &ctx, status).await?;
334                }
335                return Ok(Action::requeue(REQUEUE));
336            }
337
338            scale_primary(&target, &ctx, 1).await?;
339            if !primary_ready(&target, &ctx).await? {
340                return Ok(Action::requeue(REQUEUE));
341            }
342            scale_desired(&target, &ctx).await?;
343            if all_desired_ready(&target, &ctx).await? {
344                set_phase(&restore, &ctx, KanidmRestorePhase::Resuming, None).await?;
345            }
346            Ok(Action::requeue(REQUEUE))
347        }
348        KanidmRestorePhase::Resuming => {
349            let target = get_target(&restore, &ctx).await?;
350            clear_restoring(&restore, &target, &ctx).await?;
351            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
352            Ok(Action::requeue(Duration::from_secs(3600)))
353        }
354        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
355            Ok(Action::requeue(Duration::from_secs(3600)))
356        }
357    }
358}
359
360async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
361    let status = restore.status.clone().unwrap_or_default();
362    if status.database_mutation_started && status.phase != KanidmRestorePhase::Completed {
363        return Err(Error::MissingData(format!(
364            "refusing to remove KanidmRestore after database mutation started in phase {:?}; recover the target or remove the finalizer explicitly",
365            status.phase
366        )));
367    }
368    if let Ok(target) = get_target(&restore, &ctx).await {
369        let owns_maintenance =
370            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
371        if owns_maintenance && !status.database_mutation_started {
372            scale_desired(&target, &ctx).await?;
373        }
374        clear_restoring(&restore, &target, &ctx).await?;
375    }
376    Ok(Action::await_change())
377}
378
379async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
380    let target = get_target(restore, ctx).await?;
381    let actual_uid = target
382        .uid()
383        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
384    if actual_uid != restore.spec.target_ref.uid {
385        return Err(Error::MissingData(format!(
386            "target UID mismatch: expected {}, got {}",
387            restore.spec.target_ref.uid, actual_uid
388        )));
389    }
390    if !safe_basename(&restore.spec.source.local.file_name) {
391        return Err(Error::MissingData(
392            "restore source fileName must be a safe basename".to_string(),
393        ));
394    }
395    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
396    {
397        return Err(Error::MissingData(format!(
398            "restoreImage must be the target's pinned Kanidm image (target image is {})",
399            target.spec.image
400        )));
401    }
402    let storage = target
403        .spec
404        .storage
405        .as_ref()
406        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
407    if storage.empty_dir.is_some()
408        || storage.ephemeral.is_some()
409        || storage.volume_claim_template.is_none()
410    {
411        return Err(Error::MissingData(
412            "restore requires PVC-backed Kanidm storage".to_string(),
413        ));
414    }
415    let primaries = target
416        .spec
417        .replica_groups
418        .iter()
419        .filter(|rg| rg.primary_node)
420        .count();
421    if primaries != 1 {
422        return Err(Error::MissingData(
423            "backup/restore requires exactly one primary replica group".to_string(),
424        ));
425    }
426    validate_backup_source(restore, &target, ctx).await?;
427
428    let ns = restore.namespace().unwrap();
429    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
430        .list(&ListParams::default())
431        .await
432        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
433    if restores.items.iter().any(|other| {
434        other.name_any() != restore.name_any()
435            && other.spec.target_ref.name == restore.spec.target_ref.name
436            && !matches!(
437                other.status.as_ref().map(|s| s.phase),
438                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
439            )
440    }) {
441        return Err(Error::MissingData(
442            "another active restore targets this Kanidm".to_string(),
443        ));
444    }
445    Ok(target)
446}
447
448fn safe_basename(name: &str) -> bool {
449    !name.is_empty()
450        && name != "."
451        && name != ".."
452        && !name.contains('/')
453        && !name.contains('\\')
454        && !name.contains('\0')
455}
456
457fn mutable_image(image: &str) -> bool {
458    image == "kanidm/server:latest"
459        || image.ends_with(":latest")
460        || (!image.contains('@')
461            && !image
462                .rsplit('/')
463                .next()
464                .is_some_and(|part| part.contains(':')))
465}
466
467async fn validate_backup_source(
468    restore: &KanidmRestore,
469    target: &Kanidm,
470    ctx: &RestoreContext,
471) -> Result<()> {
472    let ns = target.namespace().unwrap();
473    let primary = primary_group(target)?;
474    let pod_name = format!("{}-0", target.statefulset_name(&primary.name));
475    let backup_file = format!("{BACKUP_PATH}/{}", restore.spec.source.local.file_name);
476    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
477    let attached = pods
478        .exec(
479            &pod_name,
480            vec![
481                "/bin/sh".to_string(),
482                "-c".to_string(),
483                r#"test -f "$1""#.to_string(),
484                "kaniop-backup-preflight".to_string(),
485                backup_file.clone(),
486            ],
487            &AttachParams::default().container("kanidm"),
488        )
489        .await
490        .map_err(|e| Error::kube_error("exec backup preflight in", "Pod", &ns, &pod_name, e))?;
491    get_output(attached).await.map(|_| ()).map_err(|error| {
492        Error::MissingData(format!(
493            "restore source {backup_file} is not accessible on primary pod {pod_name}: {error}"
494        ))
495    })
496}
497
498async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
499    let ns = restore
500        .namespace()
501        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
502    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
503        .get(&restore.spec.target_ref.name)
504        .await
505        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
506}
507
508async fn mark_restoring(
509    restore: &KanidmRestore,
510    target: &Kanidm,
511    ctx: &RestoreContext,
512) -> Result<()> {
513    let ns = target.namespace().unwrap();
514    let uid = restore
515        .uid()
516        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
517    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
518        .patch(
519            &target.name_any(),
520            &PatchParams::default(),
521            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
522        )
523        .await
524        .map(|_| ())
525        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
526}
527
528async fn clear_restoring(
529    restore: &KanidmRestore,
530    target: &Kanidm,
531    ctx: &RestoreContext,
532) -> Result<()> {
533    let ns = target.namespace().unwrap();
534    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
535        return Ok(());
536    }
537    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
538        .patch(
539            &target.name_any(),
540            &PatchParams::default(),
541            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
542        )
543        .await
544        .map(|_| ())
545        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
546}
547
548async fn patch_status(
549    restore: &KanidmRestore,
550    ctx: &RestoreContext,
551    mut status: KanidmRestoreStatus,
552) -> Result<()> {
553    let ns = restore.namespace().unwrap();
554    let previous_phase = restore.status.as_ref().map(|current| current.phase);
555    let next_phase = status.phase;
556    status.observed_generation = restore.metadata.generation;
557    update_restore_conditions(&mut status, restore.metadata.generation);
558
559    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
560        .patch_status(
561            &restore.name_any(),
562            &PatchParams::default(),
563            &Patch::Merge(json!({"status": &status})),
564        )
565        .await
566        .map_err(|e| {
567            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
568        })?;
569
570    if previous_phase != Some(next_phase) {
571        record_restore_transition(
572            restore,
573            ctx,
574            previous_phase,
575            next_phase,
576            status.message.as_deref(),
577        )
578        .await;
579    }
580    Ok(())
581}
582
583fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
584    let previous = status.conditions.clone();
585    let phase = status.phase;
586    let phase_reason = format!("{phase:?}");
587    let message = status
588        .message
589        .clone()
590        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
591    let terminal_success = phase == KanidmRestorePhase::Completed;
592    let terminal_failure = phase == KanidmRestorePhase::Failed;
593    let progressing = !terminal_success && !terminal_failure;
594
595    status.conditions = vec![
596        restore_condition(
597            &previous,
598            CONDITION_PROGRESSING,
599            if progressing {
600                CONDITION_TRUE
601            } else {
602                CONDITION_FALSE
603            },
604            &phase_reason,
605            &message,
606            generation,
607        ),
608        restore_condition(
609            &previous,
610            CONDITION_READY,
611            if terminal_success {
612                CONDITION_TRUE
613            } else {
614                CONDITION_FALSE
615            },
616            if terminal_success {
617                "RestoreCompleted"
618            } else {
619                "RestoreNotCompleted"
620            },
621            if terminal_success {
622                "Kanidm restore completed successfully."
623            } else {
624                "Kanidm restore has not completed successfully."
625            },
626            generation,
627        ),
628        restore_condition(
629            &previous,
630            CONDITION_FAILED,
631            if terminal_failure {
632                CONDITION_TRUE
633            } else {
634                CONDITION_FALSE
635            },
636            if terminal_failure {
637                "RestoreFailed"
638            } else {
639                "NoRestoreFailure"
640            },
641            if terminal_failure {
642                status
643                    .message
644                    .as_deref()
645                    .unwrap_or("Kanidm restore failed.")
646            } else {
647                "No terminal restore failure has been recorded."
648            },
649            generation,
650        ),
651    ];
652}
653
654fn restore_condition(
655    previous: &[Condition],
656    condition_type: &str,
657    condition_status: &str,
658    reason: &str,
659    message: &str,
660    generation: Option<i64>,
661) -> Condition {
662    let last_transition_time = previous
663        .iter()
664        .find(|condition| {
665            condition.type_ == condition_type
666                && condition.status == condition_status
667                && condition.reason == reason
668        })
669        .map(|condition| condition.last_transition_time.clone())
670        .unwrap_or_else(|| Time(Timestamp::now()));
671    Condition {
672        type_: condition_type.to_string(),
673        status: condition_status.to_string(),
674        reason: reason.to_string(),
675        message: message.to_string(),
676        last_transition_time,
677        observed_generation: generation,
678    }
679}
680
681async fn record_restore_transition(
682    restore: &KanidmRestore,
683    ctx: &RestoreContext,
684    previous_phase: Option<KanidmRestorePhase>,
685    phase: KanidmRestorePhase,
686    message: Option<&str>,
687) {
688    if phase == KanidmRestorePhase::Validating {
689        ctx.metrics.attempts.add(1, &[]);
690    }
691
692    let result = match phase {
693        KanidmRestorePhase::Completed => Some("success"),
694        KanidmRestorePhase::Failed => Some("failure"),
695        _ => None,
696    };
697    if let Some(result) = result {
698        let attributes = [KeyValue::new("result", result)];
699        ctx.metrics.outcomes.add(1, &attributes);
700        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
701            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
702            ctx.metrics.duration_seconds.record(elapsed, &attributes);
703        }
704    }
705
706    let reason = if phase == KanidmRestorePhase::Failed {
707        "RestoreFailed"
708    } else {
709        "RestorePhaseChanged"
710    };
711    let note = message.map(str::to_string).or_else(|| {
712        Some(format!(
713            "Kanidm restore phase changed from {} to {phase:?}.",
714            previous_phase
715                .map(|previous| format!("{previous:?}"))
716                .unwrap_or_else(|| "None".to_string())
717        ))
718    });
719    if let Err(error) = ctx
720        .recorder
721        .publish(
722            &Event {
723                type_: if phase == KanidmRestorePhase::Failed {
724                    EventType::Warning
725                } else {
726                    EventType::Normal
727                },
728                reason: reason.to_string(),
729                note,
730                action: "Restore".to_string(),
731                secondary: None,
732            },
733            &restore.object_ref(&()),
734        )
735        .await
736    {
737        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
738    }
739}
740
741async fn set_phase(
742    restore: &KanidmRestore,
743    ctx: &RestoreContext,
744    phase: KanidmRestorePhase,
745    message: Option<String>,
746) -> Result<()> {
747    let mut status = restore.status.clone().unwrap_or_default();
748    status.phase = phase;
749    status.message = message;
750    patch_status(restore, ctx, status).await
751}
752
753async fn fail_after_mutation(
754    restore: &KanidmRestore,
755    ctx: &RestoreContext,
756    message: &str,
757) -> Result<()> {
758    set_phase(
759        restore,
760        ctx,
761        KanidmRestorePhase::Failed,
762        Some(message.to_string()),
763    )
764    .await
765}
766
767fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
768    target
769        .spec
770        .replica_groups
771        .iter()
772        .find(|rg| rg.primary_node)
773        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
774}
775
776async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
777    let ns = target.namespace().unwrap();
778    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
779    for rg in &target.spec.replica_groups {
780        let name = target.statefulset_name(&rg.name);
781        api.patch(
782            &name,
783            &PatchParams::default(),
784            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
785        )
786        .await
787        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
788    }
789    Ok(())
790}
791
792async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
793    let ns = target.namespace().unwrap();
794    let rg = primary_group(target)?;
795    let name = target.statefulset_name(&rg.name);
796    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
797        .patch(
798            &name,
799            &PatchParams::default(),
800            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
801        )
802        .await
803        .map(|_| ())
804        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
805}
806
807async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
808    let ns = target.namespace().unwrap();
809    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
810    for rg in &target.spec.replica_groups {
811        let name = target.statefulset_name(&rg.name);
812        api.patch(
813            &name,
814            &PatchParams::default(),
815            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
816        )
817        .await
818        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
819    }
820    Ok(())
821}
822
823async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
824    let ns = target.namespace().unwrap();
825    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
826        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
827        .await
828        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
829    Ok(pods.items.is_empty())
830}
831
832async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
833    let ns = target.namespace().unwrap();
834    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
835    let mut pv_names = BTreeSet::new();
836    let primary_name = primary_pvc_name(target)?;
837    let primary_pvc = pvc_api
838        .get(&primary_name)
839        .await
840        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
841    let primary_pv = primary_pvc
842        .spec
843        .as_ref()
844        .and_then(|spec| spec.volume_name.clone())
845        .ok_or_else(|| {
846            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
847        })?;
848    pv_names.insert(primary_pv);
849
850    for rg in &target.spec.replica_groups {
851        let sts = target.statefulset_name(&rg.name);
852        for ordinal in 0..rg.replicas {
853            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
854            if name == primary_name {
855                continue;
856            }
857            if let Some(pvc) = pvc_api
858                .get_opt(&name)
859                .await
860                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
861                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
862            {
863                pv_names.insert(pv_name);
864            }
865        }
866    }
867
868    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
869        .list(&ListParams::default())
870        .await
871        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
872    Ok(!attachments.items.iter().any(|attachment| {
873        attachment
874            .spec
875            .source
876            .persistent_volume_name
877            .as_ref()
878            .is_some_and(|pv_name| pv_names.contains(pv_name))
879    }))
880}
881
882async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
883    let ns = target.namespace().unwrap();
884    let name = target.statefulset_name(&primary_group(target)?.name);
885    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
886        .get(&name)
887        .await
888        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
889    Ok(sts
890        .status
891        .as_ref()
892        .and_then(|s| s.ready_replicas)
893        .unwrap_or(0)
894        >= 1)
895}
896
897async fn all_desired_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
898    let ns = target.namespace().unwrap();
899    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
900    for rg in &target.spec.replica_groups {
901        let name = target.statefulset_name(&rg.name);
902        let sts = api
903            .get(&name)
904            .await
905            .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
906        if sts
907            .status
908            .as_ref()
909            .and_then(|s| s.ready_replicas)
910            .unwrap_or(0)
911            != rg.replicas
912        {
913            return Ok(false);
914        }
915    }
916    Ok(true)
917}
918
919fn primary_pvc_name(target: &Kanidm) -> Result<String> {
920    let rg = primary_group(target)?;
921    Ok(format!(
922        "{DATA_VOLUME}-{}-0",
923        target.statefulset_name(&rg.name)
924    ))
925}
926
927async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
928    let ns = target.namespace().unwrap();
929    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
930    let mut all_absent = true;
931    for rg in &target.spec.replica_groups {
932        let sts = target.statefulset_name(&rg.name);
933        for ordinal in 0..rg.replicas {
934            if rg.primary_node && ordinal == 0 {
935                continue;
936            }
937            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
938            match api
939                .get_opt(&name)
940                .await
941                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
942            {
943                None => {}
944                Some(pvc) => {
945                    all_absent = false;
946                    if pvc.metadata.deletion_timestamp.is_none() {
947                        match api.delete(&name, &DeleteParams::default()).await {
948                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
949                            Err(kube::Error::Api(status)) if status.code == 404 => {}
950                            Err(error) => {
951                                return Err(Error::kube_error(
952                                    "delete",
953                                    "PersistentVolumeClaim",
954                                    &ns,
955                                    &name,
956                                    error,
957                                ));
958                            }
959                        }
960                    }
961                }
962            }
963        }
964    }
965    Ok(all_absent)
966}
967
968fn config_map_name(restore: &KanidmRestore) -> String {
969    format!("{}-config", restore.name_any())
970}
971fn restore_job_name(restore: &KanidmRestore) -> String {
972    format!("{}-restore", restore.name_any())
973}
974fn verify_job_name(restore: &KanidmRestore) -> String {
975    format!("{}-verify", restore.name_any())
976}
977
978async fn ensure_restore_config(
979    restore: &KanidmRestore,
980    target: &Kanidm,
981    ctx: &RestoreContext,
982) -> Result<()> {
983    let ns = restore.namespace().unwrap();
984    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), &ns);
985    let name = config_map_name(restore);
986    if api
987        .get_opt(&name)
988        .await
989        .map_err(|e| Error::kube_error("get", "ConfigMap", &ns, &name, e))?
990        .is_some()
991    {
992        return Ok(());
993    }
994    let config = ConfigMap {
995        metadata: ObjectMeta {
996            name: Some(name.clone()),
997            namespace: Some(ns.clone()),
998            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
999            ..Default::default()
1000        },
1001        data: Some(std::collections::BTreeMap::from([(
1002            "server.toml".to_string(),
1003            "version = \"2\"\n".to_string(),
1004        )])),
1005        ..Default::default()
1006    };
1007    api.create(&PostParams::default(), &config)
1008        .await
1009        .map(|_| ())
1010        .map_err(|e| Error::kube_error("create", "ConfigMap", &ns, target.name_any(), e))
1011}
1012
1013async fn ensure_database_job(
1014    restore: &KanidmRestore,
1015    target: &Kanidm,
1016    ctx: &RestoreContext,
1017    name: &str,
1018    verify: bool,
1019) -> Result<()> {
1020    let ns = restore.namespace().unwrap();
1021    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1022    if api
1023        .get_opt(name)
1024        .await
1025        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1026        .is_some()
1027    {
1028        return Ok(());
1029    }
1030    let mut command = vec![
1031        "kanidmd".to_string(),
1032        "database".to_string(),
1033        if verify { "verify" } else { "restore" }.to_string(),
1034        "-c".to_string(),
1035        format!("{CONFIG_PATH}/server.toml"),
1036    ];
1037    if !verify {
1038        command.push(format!(
1039            "{BACKUP_PATH}/{}",
1040            restore.spec.source.local.file_name
1041        ));
1042    }
1043    let job = Job {
1044        metadata: ObjectMeta {
1045            name: Some(name.to_string()),
1046            namespace: Some(ns.clone()),
1047            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1048            ..Default::default()
1049        },
1050        spec: Some(JobSpec {
1051            backoff_limit: Some(0),
1052            template: PodTemplateSpec {
1053                metadata: Some(ObjectMeta::default()),
1054                spec: Some(PodSpec {
1055                    automount_service_account_token: Some(false),
1056                    restart_policy: Some("Never".to_string()),
1057                    containers: vec![Container {
1058                        name: if verify { "verify" } else { "restore" }.to_string(),
1059                        image: Some(restore.spec.restore_image.clone()),
1060                        command: Some(command),
1061                        env: Some(vec![
1062                            EnvVar {
1063                                name: "KANIDM_DB_PATH".to_string(),
1064                                value: Some("/data/kanidm.db".to_string()),
1065                                ..Default::default()
1066                            },
1067                            EnvVar {
1068                                name: "KANIDM_DOMAIN".to_string(),
1069                                value: Some(target.spec.domain.clone()),
1070                                ..Default::default()
1071                            },
1072                        ]),
1073                        volume_mounts: Some(vec![
1074                            VolumeMount {
1075                                name: DATA_VOLUME.to_string(),
1076                                mount_path: DATA_PATH.to_string(),
1077                                ..Default::default()
1078                            },
1079                            VolumeMount {
1080                                name: CONFIG_VOLUME.to_string(),
1081                                mount_path: CONFIG_PATH.to_string(),
1082                                read_only: Some(true),
1083                                ..Default::default()
1084                            },
1085                        ]),
1086                        ..Default::default()
1087                    }],
1088                    volumes: Some(vec![
1089                        Volume {
1090                            name: DATA_VOLUME.to_string(),
1091                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1092                                claim_name: primary_pvc_name(target)?,
1093                                read_only: Some(false),
1094                            }),
1095                            ..Default::default()
1096                        },
1097                        Volume {
1098                            name: CONFIG_VOLUME.to_string(),
1099                            config_map: Some(ConfigMapVolumeSource {
1100                                name: config_map_name(restore),
1101                                ..Default::default()
1102                            }),
1103                            ..Default::default()
1104                        },
1105                    ]),
1106                    ..Default::default()
1107                }),
1108            },
1109            ..Default::default()
1110        }),
1111        ..Default::default()
1112    };
1113    api.create(&PostParams::default(), &job)
1114        .await
1115        .map(|_| ())
1116        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1117}
1118
1119enum JobState {
1120    Running,
1121    Complete,
1122    Failed,
1123}
1124
1125async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1126    let ns = restore.namespace().unwrap();
1127    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1128        .get(name)
1129        .await
1130        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1131    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1132        return Ok(JobState::Failed);
1133    }
1134    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1135        return Ok(JobState::Complete);
1136    }
1137    Ok(JobState::Running)
1138}
1139
1140#[cfg(test)]
1141mod tests {
1142    use super::{
1143        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestorePhase, KanidmRestoreStatus,
1144        mutable_image, safe_basename, update_restore_conditions,
1145    };
1146
1147    #[test]
1148    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
1149        assert!(!KanidmRestoreStatus::default().database_mutation_started);
1150    }
1151
1152    #[test]
1153    fn completed_restore_sets_ready_condition() {
1154        let mut status = KanidmRestoreStatus {
1155            phase: KanidmRestorePhase::Completed,
1156            ..Default::default()
1157        };
1158        update_restore_conditions(&mut status, Some(7));
1159        assert!(status.conditions.iter().any(|condition| {
1160            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
1161        }));
1162        assert!(!status.conditions.iter().any(|condition| {
1163            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
1164        }));
1165    }
1166
1167    #[test]
1168    fn failed_restore_sets_failed_condition() {
1169        let mut status = KanidmRestoreStatus {
1170            phase: KanidmRestorePhase::Failed,
1171            message: Some("verification failed".to_string()),
1172            ..Default::default()
1173        };
1174        update_restore_conditions(&mut status, Some(8));
1175        assert!(status.conditions.iter().any(|condition| {
1176            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
1177        }));
1178    }
1179
1180    #[test]
1181    fn restore_filename_is_confined_to_backup_directory() {
1182        assert!(safe_basename("backup.json.gz"));
1183        assert!(!safe_basename("../backup.json.gz"));
1184        assert!(!safe_basename("a/b"));
1185        assert!(!safe_basename(""));
1186    }
1187
1188    #[test]
1189    fn restore_image_rejects_latest_and_unpinned_names() {
1190        assert!(mutable_image("kanidm/server:latest"));
1191        assert!(mutable_image("kanidm/server"));
1192        assert!(!mutable_image("kanidm/server:1.10.0"));
1193        assert!(!mutable_image("kanidm/server@sha256:abc"));
1194    }
1195}