Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, ca_bundle_env_var, ca_bundle_path,
9};
10use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
11use kaniop_backup_core::image::data_mover_image;
12use kaniop_backup_core::operation::{
13    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
14};
15use kaniop_backup_core::result::{ExitCode, parse_result_document};
16use kaniop_k8s_util::error::{Error, Result};
17
18use std::collections::BTreeSet;
19use std::sync::Arc;
20use std::time::Duration;
21
22use futures::StreamExt;
23use k8s_openapi::api::apps::v1::StatefulSet;
24use k8s_openapi::api::batch::v1::{Job, JobSpec};
25use k8s_openapi::api::core::v1::{
26    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
27    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
28    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
29};
30use k8s_openapi::api::storage::v1::VolumeAttachment;
31use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
32use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
33use k8s_openapi::jiff::Timestamp;
34use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
35use kube::runtime::controller::{Action, Controller};
36use kube::runtime::events::{Event, EventType, Recorder, Reporter};
37use kube::runtime::finalizer::{Event as Finalizer, finalizer};
38use kube::runtime::watcher;
39use kube::{Api, Client, CustomResource, Resource, ResourceExt};
40use opentelemetry::metrics::{Counter, Histogram};
41use opentelemetry::{KeyValue, global};
42#[cfg(feature = "schemars")]
43use schemars::JsonSchema;
44use serde::{Deserialize, Serialize};
45use serde_json::json;
46use tracing::{debug, error, info, warn};
47
48pub const CONTROLLER_ID: &str = "kanidm-restore";
49pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
50const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
51const DATA_VOLUME: &str = "kanidm-data";
52const STAGING_VOLUME: &str = "restore-staging";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const STAGING_PATH: &str = "/staging";
59const SHARED_VOL_PATH: &str = "/shared";
60const REQUEUE: Duration = Duration::from_secs(2);
61const CONDITION_TRUE: &str = "True";
62const CONDITION_FALSE: &str = "False";
63const CONDITION_PROGRESSING: &str = "Progressing";
64const CONDITION_READY: &str = "Ready";
65const CONDITION_FAILED: &str = "Failed";
66
67pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
68pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
71const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
72const SOURCE_PREP_RESULT_OPERATION: &str = "download";
73
74#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
75#[cfg_attr(feature = "schemars", derive(JsonSchema))]
76#[cfg_attr(
77    feature = "schemars",
78    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
79)]
80#[kube(
81    category = "kaniop",
82    group = "kaniop.rs",
83    version = "v1beta1",
84    kind = "KanidmRestore",
85    plural = "kanidmrestores",
86    singular = "kanidmrestore",
87    shortname = "idmrestore",
88    namespaced,
89    status = "KanidmRestoreStatus",
90    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
91    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
92    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
93)]
94#[serde(rename_all = "camelCase")]
95pub struct KanidmRestoreSpec {
96    pub target_ref: KanidmRestoreTargetRef,
97    pub source: KanidmRestoreSource,
98    pub restore_image: String,
99    #[serde(skip_serializing_if = "Option::is_none")]
100    pub safety_backup: Option<SafetyBackupConfig>,
101}
102
103#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
104#[cfg_attr(feature = "schemars", derive(JsonSchema))]
105#[serde(rename_all = "camelCase")]
106pub struct KanidmRestoreTargetRef {
107    pub name: String,
108    pub uid: String,
109}
110
111#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
112#[cfg_attr(feature = "schemars", derive(JsonSchema))]
113#[cfg_attr(
114    feature = "schemars",
115    schemars(extend("x-kubernetes-validations" = [
116        {
117            "message": "exactly one of local or backupRef must be set",
118            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
119        }
120    ]))
121)]
122#[serde(rename_all = "camelCase")]
123pub struct KanidmRestoreSource {
124    /// Local file source. Mutually exclusive with `backupRef`.
125    #[serde(skip_serializing_if = "Option::is_none")]
126    pub local: Option<KanidmRestoreLocalSource>,
127    /// Remote cataloged backup reference. Mutually exclusive with `local`.
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
130}
131
132#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
133#[cfg_attr(feature = "schemars", derive(JsonSchema))]
134#[serde(rename_all = "camelCase")]
135pub struct KanidmRestoreLocalSource {
136    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
137    pub file_name: String,
138}
139
140#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
141#[cfg_attr(feature = "schemars", derive(JsonSchema))]
142#[serde(rename_all = "camelCase")]
143pub struct KanidmRestoreBackupRefSource {
144    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
145    pub name: String,
146}
147
148#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
149#[cfg_attr(feature = "schemars", derive(JsonSchema))]
150#[serde(rename_all = "camelCase")]
151pub struct SafetyBackupConfig {
152    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
153    #[serde(skip_serializing_if = "Option::is_none")]
154    pub repository_ref: Option<SafetyBackupRepositoryRef>,
155    /// Skip the mandatory safety backup. Requires break-glass annotations.
156    #[serde(default)]
157    pub skip: bool,
158}
159
160#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
161#[cfg_attr(feature = "schemars", derive(JsonSchema))]
162#[serde(rename_all = "camelCase")]
163pub struct SafetyBackupRepositoryRef {
164    pub name: String,
165}
166
167#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
168#[cfg_attr(feature = "schemars", derive(JsonSchema))]
169pub enum KanidmRestorePhase {
170    #[default]
171    Pending,
172    Validating,
173    Quiescing,
174    SafetyBackup,
175    PreparingSource,
176    RestoringPrimary,
177    Verifying,
178    RebuildingReplicas,
179    Resuming,
180    Completed,
181    Failed,
182}
183
184#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
185#[cfg_attr(feature = "schemars", derive(JsonSchema))]
186#[serde(rename_all = "camelCase")]
187pub struct KanidmRestoreStatus {
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub observed_generation: Option<i64>,
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub observed_target_uid: Option<String>,
192    #[serde(default)]
193    pub phase: KanidmRestorePhase,
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub restore_job_name: Option<String>,
196    #[serde(skip_serializing_if = "Option::is_none")]
197    pub verify_job_name: Option<String>,
198    #[serde(skip_serializing_if = "Option::is_none")]
199    pub safety_backup_job_name: Option<String>,
200    #[serde(skip_serializing_if = "Option::is_none")]
201    pub source_prep_job_name: Option<String>,
202    #[serde(skip_serializing_if = "Option::is_none")]
203    pub safety_backup_ref: Option<String>,
204    #[serde(skip_serializing_if = "Option::is_none")]
205    pub safety_backup_expected_backup_id: Option<String>,
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub safety_backup_manifest_key: Option<String>,
208    #[serde(skip_serializing_if = "Option::is_none")]
209    pub safety_backup_payload_sha256: Option<String>,
210    #[serde(default)]
211    pub replicas_cleared: bool,
212    #[serde(default)]
213    pub certificates_cleared: bool,
214    #[serde(default)]
215    pub database_mutation_started: bool,
216    #[serde(default, skip_serializing_if = "Vec::is_empty")]
217    pub original_replicas: Vec<ReplicaCountEntry>,
218    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
219    pub phase_timestamps: std::collections::BTreeMap<String, String>,
220    #[serde(default, skip_serializing_if = "Vec::is_empty")]
221    #[cfg_attr(
222        feature = "schemars",
223        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
224    )]
225    pub conditions: Vec<Condition>,
226    #[serde(skip_serializing_if = "Option::is_none")]
227    pub message: Option<String>,
228}
229
230#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
231#[cfg_attr(feature = "schemars", derive(JsonSchema))]
232#[serde(rename_all = "camelCase")]
233pub struct ReplicaCountEntry {
234    pub group: String,
235    pub replicas: i32,
236}
237
238#[derive(Clone)]
239struct RestoreMetrics {
240    attempts: Counter<u64>,
241    outcomes: Counter<u64>,
242    duration_seconds: Histogram<f64>,
243    break_glass_total: Counter<u64>,
244    safety_backup_duration_seconds: Histogram<f64>,
245}
246
247impl RestoreMetrics {
248    fn new() -> Self {
249        let meter = global::meter("kaniop");
250        Self {
251            attempts: meter
252                .u64_counter("kaniop_restore_attempts_total")
253                .with_description("Number of Kanidm restore attempts started")
254                .build(),
255            outcomes: meter
256                .u64_counter("kaniop_restore_outcomes_total")
257                .with_description("Number of terminal Kanidm restore outcomes")
258                .build(),
259            duration_seconds: meter
260                .f64_histogram("kaniop_restore_duration_seconds")
261                .with_description("Kanidm restore duration from object creation to terminal phase")
262                .with_unit("s")
263                .build(),
264            break_glass_total: meter
265                .u64_counter("kaniop_restore_break_glass_total")
266                .with_description("Number of break-glass safety backup overrides")
267                .build(),
268            safety_backup_duration_seconds: meter
269                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
270                .with_description("Duration of pre-restore safety backup creation")
271                .with_unit("s")
272                .build(),
273        }
274    }
275}
276
277#[derive(Clone)]
278struct RestoreContext {
279    client: Client,
280    recorder: Recorder,
281    metrics: RestoreMetrics,
282}
283
284pub async fn run(client: Client) {
285    let api = Api::<KanidmRestore>::all(client.clone());
286    let recorder = Recorder::new(
287        client.clone(),
288        Reporter {
289            controller: CONTROLLER_ID.into(),
290            instance: None,
291        },
292    );
293    let ctx = Arc::new(RestoreContext {
294        client,
295        recorder,
296        metrics: RestoreMetrics::new(),
297    });
298    info!("starting {CONTROLLER_ID} controller");
299    Controller::new(api, watcher::Config::default().any_semantic())
300        .shutdown_on_signal()
301        .run(reconcile_restore, error_policy, ctx)
302        .for_each(|result| async move {
303            if let Err(error) = result {
304                error!(%error, "KanidmRestore reconciliation failed");
305            }
306        })
307        .await;
308}
309
310fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
311    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
312    Action::requeue(Duration::from_secs(5))
313}
314
315async fn reconcile_restore(
316    restore: Arc<KanidmRestore>,
317    ctx: Arc<RestoreContext>,
318) -> Result<Action> {
319    let namespace = restore
320        .namespace()
321        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
322    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
323    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
324        let ctx = ctx.clone();
325        async move {
326            match event {
327                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
328                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
329            }
330        }
331    })
332    .await
333    .map_err(|error| {
334        Error::FinalizerError(
335            "failed on KanidmRestore finalizer".to_string(),
336            Box::new(error),
337        )
338    })
339}
340
341async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
342    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
343    match phase {
344        KanidmRestorePhase::Pending => {
345            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
346            Ok(Action::requeue(REQUEUE))
347        }
348        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
349            Ok(target) => {
350                mark_restoring(&restore, &target, &ctx).await?;
351                let mut status = restore.status.clone().unwrap_or_default();
352                status.observed_target_uid = target.uid();
353                status.original_replicas = capture_original_replicas(&target);
354                status.phase = KanidmRestorePhase::Quiescing;
355                status.message = None;
356                patch_status(&restore, &ctx, status).await?;
357                Ok(Action::requeue(REQUEUE))
358            }
359            Err(error) => {
360                set_phase(
361                    &restore,
362                    &ctx,
363                    KanidmRestorePhase::Failed,
364                    Some(error.to_string()),
365                )
366                .await?;
367                Ok(Action::requeue(Duration::from_secs(300)))
368            }
369        },
370        KanidmRestorePhase::Quiescing => {
371            let target = get_target(&restore, &ctx).await?;
372            scale_all(&target, &ctx, 0).await?;
373            if target_pods_stopped(&target, &ctx).await?
374                && target_volumes_detached(&target, &ctx).await?
375            {
376                let next_phase = if requires_safety_backup(&restore) {
377                    KanidmRestorePhase::SafetyBackup
378                } else {
379                    record_break_glass(&restore, &ctx).await?;
380                    KanidmRestorePhase::PreparingSource
381                };
382                if next_phase == KanidmRestorePhase::PreparingSource {
383                    let refreshed = Api::<KanidmRestore>::namespaced(
384                        ctx.client.clone(),
385                        restore.namespace().as_deref().unwrap_or_default(),
386                    )
387                    .get(&restore.name_any())
388                    .await
389                    .map_err(|error| {
390                        Error::kube_error(
391                            "get",
392                            "KanidmRestore",
393                            restore.namespace().as_deref().unwrap_or_default(),
394                            restore.name_any(),
395                            error,
396                        )
397                    })?;
398                    set_phase(&refreshed, &ctx, next_phase, None).await?;
399                } else {
400                    set_phase(&restore, &ctx, next_phase, None).await?;
401                }
402            }
403            Ok(Action::requeue(REQUEUE))
404        }
405        KanidmRestorePhase::SafetyBackup => {
406            let target = get_target(&restore, &ctx).await?;
407            let name = safety_backup_job_name(&restore);
408            let expected_backup_id = compute_safety_backup_id(&restore);
409            let expected_manifest_key = compute_safety_manifest_key(&restore);
410            ensure_safety_backup_job(
411                &restore,
412                &target,
413                &ctx,
414                &name,
415                &expected_backup_id,
416                &expected_manifest_key,
417            )
418            .await?;
419            let mut status = restore.status.clone().unwrap_or_default();
420            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421            match job_state(&restore, &ctx, &name).await? {
422                JobState::Complete => {
423                    let result =
424                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425                    match result {
426                        Ok(verified) => {
427                            status.safety_backup_job_name = Some(name);
428                            status.safety_backup_ref =
429                                Some(format!("safety-{}", restore.name_any()));
430                            status.safety_backup_manifest_key = Some(verified.manifest_key);
431                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432                            record_safety_backup_duration(&restore, &ctx).await;
433                            status.phase = KanidmRestorePhase::PreparingSource;
434                            status.message = None;
435                            patch_status(&restore, &ctx, status).await?;
436                        }
437                        Err(error) => {
438                            resume_before_mutation(&restore, &ctx).await?;
439                            set_phase(
440                                &restore,
441                                &ctx,
442                                KanidmRestorePhase::Failed,
443                                Some(format!("safety backup result verification failed: {error}")),
444                            )
445                            .await?;
446                        }
447                    }
448                }
449                JobState::Failed => {
450                    resume_before_mutation(&restore, &ctx).await?;
451                    set_phase(
452                        &restore,
453                        &ctx,
454                        KanidmRestorePhase::Failed,
455                        Some(
456                            "safety backup job failed; target restored to original state"
457                                .to_string(),
458                        ),
459                    )
460                    .await?;
461                }
462                JobState::Running => {
463                    patch_status(&restore, &ctx, status).await?;
464                }
465            }
466            Ok(Action::requeue(REQUEUE))
467        }
468        KanidmRestorePhase::PreparingSource => {
469            if is_remote_source(&restore) {
470                let target = get_target(&restore, &ctx).await?;
471                let name = source_prep_job_name(&restore);
472                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473                match job_state(&restore, &ctx, &name).await? {
474                    JobState::Complete => {
475                        let backup_id = restore
476                            .spec
477                            .source
478                            .backup_ref
479                            .as_ref()
480                            .map(|b| b.name.clone())
481                            .unwrap_or_default();
482                        let backup = {
483                            let ns = restore.namespace().unwrap();
484                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485                                .get(&backup_id)
486                                .await
487                                .map_err(|e| {
488                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489                                })?
490                        };
491                        let expected_backup_id = &backup.spec.backup_id;
492                        let result =
493                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494                                .await;
495                        match result {
496                            Ok(_verified) => {
497                                let mut status = restore.status.clone().unwrap_or_default();
498                                status.source_prep_job_name = Some(name);
499                                status.phase = KanidmRestorePhase::RestoringPrimary;
500                                status.message = None;
501                                patch_status(&restore, &ctx, status).await?;
502                            }
503                            Err(error) => {
504                                resume_before_mutation(&restore, &ctx).await?;
505                                set_phase(
506                                    &restore,
507                                    &ctx,
508                                    KanidmRestorePhase::Failed,
509                                    Some(format!(
510                                        "source preparation result verification failed: {error}"
511                                    )),
512                                )
513                                .await?;
514                            }
515                        }
516                    }
517                    JobState::Failed => {
518                        resume_before_mutation(&restore, &ctx).await?;
519                        set_phase(
520                            &restore,
521                            &ctx,
522                            KanidmRestorePhase::Failed,
523                            Some(
524                                "source preparation job failed; target restored to original state"
525                                    .to_string(),
526                            ),
527                        )
528                        .await?;
529                    }
530                    JobState::Running => {}
531                }
532            } else {
533                let target = get_target(&restore, &ctx).await?;
534                let name = source_check_job_name(&restore);
535                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
536                match job_state(&restore, &ctx, &name).await? {
537                    JobState::Complete => {
538                        let mut status = restore.status.clone().unwrap_or_default();
539                        status.source_prep_job_name = Some(name);
540                        status.phase = KanidmRestorePhase::RestoringPrimary;
541                        status.message = None;
542                        patch_status(&restore, &ctx, status).await?;
543                    }
544                    JobState::Failed => {
545                        resume_before_mutation(&restore, &ctx).await?;
546                        set_phase(
547                            &restore,
548                            &ctx,
549                            KanidmRestorePhase::Failed,
550                            Some(
551                                "local backup file check failed; target restored to original state"
552                                    .to_string(),
553                            ),
554                        )
555                        .await?;
556                    }
557                    JobState::Running => {}
558                }
559            }
560            Ok(Action::requeue(REQUEUE))
561        }
562        KanidmRestorePhase::RestoringPrimary => {
563            let target = get_target(&restore, &ctx).await?;
564            let status = restore.status.as_ref().cloned().unwrap_or_default();
565            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
566                return Err(Error::MissingData(
567                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
568                ));
569            }
570            if !status.database_mutation_started {
571                let mut status = status;
572                status.database_mutation_started = true;
573                status.message = Some(
574                    "database mutation boundary persisted; starting primary restore".to_string(),
575                );
576                patch_status(&restore, &ctx, status).await?;
577                return Ok(Action::requeue(REQUEUE));
578            }
579            let name = restore_job_name(&restore);
580            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
581            match job_state(&restore, &ctx, &name).await? {
582                JobState::Complete => {
583                    let mut status = restore.status.clone().unwrap_or_default();
584                    status.restore_job_name = Some(name);
585                    status.phase = KanidmRestorePhase::Verifying;
586                    status.message = None;
587                    patch_status(&restore, &ctx, status).await?;
588                }
589                JobState::Failed => {
590                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
591                }
592                JobState::Running => {}
593            }
594            Ok(Action::requeue(REQUEUE))
595        }
596        KanidmRestorePhase::Verifying => {
597            let target = get_target(&restore, &ctx).await?;
598            let name = verify_job_name(&restore);
599            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
600            match job_state(&restore, &ctx, &name).await? {
601                JobState::Complete => {
602                    let mut status = restore.status.clone().unwrap_or_default();
603                    status.verify_job_name = Some(name);
604                    status.phase = KanidmRestorePhase::RebuildingReplicas;
605                    status.message = None;
606                    patch_status(&restore, &ctx, status).await?;
607                }
608                JobState::Failed => {
609                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
610                }
611                JobState::Running => {}
612            }
613            Ok(Action::requeue(REQUEUE))
614        }
615        KanidmRestorePhase::RebuildingReplicas => {
616            let target = get_target(&restore, &ctx).await?;
617            let mut status = restore.status.clone().unwrap_or_default();
618            if !status.replicas_cleared {
619                if delete_secondary_pvcs(&target, &ctx).await? {
620                    status.replicas_cleared = true;
621                    status.message = Some("secondary database state cleared".to_string());
622                    patch_status(&restore, &ctx, status).await?;
623                }
624                return Ok(Action::requeue(REQUEUE));
625            }
626
627            scale_primary(&target, &ctx, 1).await?;
628            if !primary_ready(&target, &ctx).await? {
629                return Ok(Action::requeue(REQUEUE));
630            }
631            if !status.certificates_cleared {
632                delete_replica_cert_secrets(&target, &ctx).await?;
633                delete_admin_secret(&target, &ctx).await?;
634                status.certificates_cleared = true;
635                status.message = Some(
636                    "replica certificates and admin secret cleared for regeneration".to_string(),
637                );
638                patch_status(&restore, &ctx, status).await?;
639                return Ok(Action::requeue(REQUEUE));
640            }
641            status.phase = KanidmRestorePhase::Resuming;
642            status.message = None;
643            patch_status(&restore, &ctx, status).await?;
644            Ok(Action::requeue(REQUEUE))
645        }
646        KanidmRestorePhase::Resuming => {
647            let target = get_target(&restore, &ctx).await?;
648            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
649            clear_restoring(&restore, &target, &ctx).await?;
650            Ok(Action::requeue(Duration::from_secs(3600)))
651        }
652        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
653            Ok(Action::requeue(Duration::from_secs(3600)))
654        }
655    }
656}
657
658async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
659    let status = restore.status.clone().unwrap_or_default();
660    if status.database_mutation_started
661        && status.phase != KanidmRestorePhase::Completed
662        && status.phase != KanidmRestorePhase::Failed
663    {
664        reconcile_apply(restore.clone(), ctx.clone()).await?;
665        return Ok(Action::requeue(REQUEUE));
666    }
667    if let Ok(target) = get_target(&restore, &ctx).await {
668        let owns_maintenance =
669            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
670        if owns_maintenance && !status.database_mutation_started {
671            scale_desired(&target, &ctx).await?;
672        }
673        clear_restoring(&restore, &target, &ctx).await?;
674    }
675    Ok(Action::await_change())
676}
677
678async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
679    let target = get_target(restore, ctx).await?;
680    let actual_uid = target
681        .uid()
682        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
683    if actual_uid != restore.spec.target_ref.uid {
684        return Err(Error::MissingData(format!(
685            "target UID mismatch: expected {}, got {}",
686            restore.spec.target_ref.uid, actual_uid
687        )));
688    }
689    validate_source(restore)?;
690    if is_remote_source(restore) {
691        validate_backup_ref(restore, &target, ctx).await?;
692    }
693    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
694    {
695        return Err(Error::MissingData(format!(
696            "restoreImage must be the target's pinned Kanidm image (target image is {})",
697            target.spec.image
698        )));
699    }
700    let storage = target
701        .spec
702        .storage
703        .as_ref()
704        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
705    if storage.empty_dir.is_some()
706        || storage.ephemeral.is_some()
707        || storage.volume_claim_template.is_none()
708    {
709        return Err(Error::MissingData(
710            "restore requires PVC-backed Kanidm storage".to_string(),
711        ));
712    }
713    let primaries = target
714        .spec
715        .replica_groups
716        .iter()
717        .filter(|rg| rg.primary_node)
718        .count();
719    if primaries != 1 {
720        return Err(Error::MissingData(
721            "backup/restore requires exactly one primary replica group".to_string(),
722        ));
723    }
724    let ns = restore.namespace().unwrap();
725    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
726        .list(&ListParams::default())
727        .await
728        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
729    if restores.items.iter().any(|other| {
730        other.name_any() != restore.name_any()
731            && other.spec.target_ref.name == restore.spec.target_ref.name
732            && !matches!(
733                other.status.as_ref().map(|s| s.phase),
734                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
735            )
736    }) {
737        return Err(Error::MissingData(
738            "another active restore targets this Kanidm".to_string(),
739        ));
740    }
741    validate_safety_backup_config(restore)?;
742    Ok(target)
743}
744
745fn safe_basename(name: &str) -> bool {
746    !name.is_empty()
747        && name != "."
748        && name != ".."
749        && !name.contains('/')
750        && !name.contains('\\')
751        && !name.contains('\0')
752}
753
754fn mutable_image(image: &str) -> bool {
755    image == "kanidm/server:latest"
756        || image.ends_with(":latest")
757        || (!image.contains('@')
758            && !image
759                .rsplit('/')
760                .next()
761                .is_some_and(|part| part.contains(':')))
762}
763
764async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
765    let ns = restore
766        .namespace()
767        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
768    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
769        .get(&restore.spec.target_ref.name)
770        .await
771        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
772}
773
774async fn mark_restoring(
775    restore: &KanidmRestore,
776    target: &Kanidm,
777    ctx: &RestoreContext,
778) -> Result<()> {
779    let ns = target.namespace().unwrap();
780    let uid = restore
781        .uid()
782        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
783    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784        .patch(
785            &target.name_any(),
786            &PatchParams::default(),
787            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
788        )
789        .await
790        .map(|_| ())
791        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
792}
793
794async fn clear_restoring(
795    restore: &KanidmRestore,
796    target: &Kanidm,
797    ctx: &RestoreContext,
798) -> Result<()> {
799    let ns = target.namespace().unwrap();
800    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
801        return Ok(());
802    }
803    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
804        .patch(
805            &target.name_any(),
806            &PatchParams::default(),
807            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
808        )
809        .await
810        .map(|_| ())
811        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
812}
813
814async fn patch_status(
815    restore: &KanidmRestore,
816    ctx: &RestoreContext,
817    mut status: KanidmRestoreStatus,
818) -> Result<()> {
819    let ns = restore.namespace().unwrap();
820    let previous_phase = restore.status.as_ref().map(|current| current.phase);
821    let next_phase = status.phase;
822    status.observed_generation = restore.metadata.generation;
823    update_restore_conditions(&mut status, restore.metadata.generation);
824
825    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
826        .patch_status(
827            &restore.name_any(),
828            &PatchParams::default(),
829            &Patch::Merge(json!({"status": &status})),
830        )
831        .await
832        .map_err(|e| {
833            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
834        })?;
835
836    if previous_phase != Some(next_phase) {
837        record_restore_transition(
838            restore,
839            ctx,
840            previous_phase,
841            next_phase,
842            status.message.as_deref(),
843        )
844        .await;
845    }
846    Ok(())
847}
848
849fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
850    let previous = status.conditions.clone();
851    let phase = status.phase;
852    let phase_reason = format!("{phase:?}");
853    let message = status
854        .message
855        .clone()
856        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
857    let terminal_success = phase == KanidmRestorePhase::Completed;
858    let terminal_failure = phase == KanidmRestorePhase::Failed;
859    let progressing = !terminal_success && !terminal_failure;
860
861    let break_glass = previous
862        .iter()
863        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
864        .cloned();
865    status.conditions = vec![
866        restore_condition(
867            &previous,
868            CONDITION_PROGRESSING,
869            if progressing {
870                CONDITION_TRUE
871            } else {
872                CONDITION_FALSE
873            },
874            &phase_reason,
875            &message,
876            generation,
877        ),
878        restore_condition(
879            &previous,
880            CONDITION_READY,
881            if terminal_success {
882                CONDITION_TRUE
883            } else {
884                CONDITION_FALSE
885            },
886            if terminal_success {
887                "RestoreCompleted"
888            } else {
889                "RestoreNotCompleted"
890            },
891            if terminal_success {
892                "Kanidm restore completed successfully."
893            } else {
894                "Kanidm restore has not completed successfully."
895            },
896            generation,
897        ),
898        restore_condition(
899            &previous,
900            CONDITION_FAILED,
901            if terminal_failure {
902                CONDITION_TRUE
903            } else {
904                CONDITION_FALSE
905            },
906            if terminal_failure {
907                "RestoreFailed"
908            } else {
909                "NoRestoreFailure"
910            },
911            if terminal_failure {
912                status
913                    .message
914                    .as_deref()
915                    .unwrap_or("Kanidm restore failed.")
916            } else {
917                "No terminal restore failure has been recorded."
918            },
919            generation,
920        ),
921    ];
922    if let Some(condition) = break_glass {
923        status.conditions.push(condition);
924    }
925}
926
927fn restore_condition(
928    previous: &[Condition],
929    condition_type: &str,
930    condition_status: &str,
931    reason: &str,
932    message: &str,
933    generation: Option<i64>,
934) -> Condition {
935    let last_transition_time = previous
936        .iter()
937        .find(|condition| {
938            condition.type_ == condition_type
939                && condition.status == condition_status
940                && condition.reason == reason
941        })
942        .map(|condition| condition.last_transition_time.clone())
943        .unwrap_or_else(|| Time(Timestamp::now()));
944    Condition {
945        type_: condition_type.to_string(),
946        status: condition_status.to_string(),
947        reason: reason.to_string(),
948        message: message.to_string(),
949        last_transition_time,
950        observed_generation: generation,
951    }
952}
953
954async fn record_restore_transition(
955    restore: &KanidmRestore,
956    ctx: &RestoreContext,
957    previous_phase: Option<KanidmRestorePhase>,
958    phase: KanidmRestorePhase,
959    message: Option<&str>,
960) {
961    if phase == KanidmRestorePhase::Validating {
962        ctx.metrics.attempts.add(1, &[]);
963    }
964
965    let result = match phase {
966        KanidmRestorePhase::Completed => Some("success"),
967        KanidmRestorePhase::Failed => Some("failure"),
968        _ => None,
969    };
970    if let Some(result) = result {
971        let attributes = [KeyValue::new("result", result)];
972        ctx.metrics.outcomes.add(1, &attributes);
973        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
974            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
975            ctx.metrics.duration_seconds.record(elapsed, &attributes);
976        }
977    }
978
979    let reason = if phase == KanidmRestorePhase::Failed {
980        "RestoreFailed"
981    } else {
982        "RestorePhaseChanged"
983    };
984    let note = message.map(str::to_string).or_else(|| {
985        Some(format!(
986            "Kanidm restore phase changed from {} to {phase:?}.",
987            previous_phase
988                .map(|previous| format!("{previous:?}"))
989                .unwrap_or_else(|| "None".to_string())
990        ))
991    });
992    if let Err(error) = ctx
993        .recorder
994        .publish(
995            &Event {
996                type_: if phase == KanidmRestorePhase::Failed {
997                    EventType::Warning
998                } else {
999                    EventType::Normal
1000                },
1001                reason: reason.to_string(),
1002                note,
1003                action: "Restore".to_string(),
1004                secondary: None,
1005            },
1006            &restore.object_ref(&()),
1007        )
1008        .await
1009    {
1010        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1011    }
1012}
1013
1014async fn set_phase(
1015    restore: &KanidmRestore,
1016    ctx: &RestoreContext,
1017    phase: KanidmRestorePhase,
1018    message: Option<String>,
1019) -> Result<()> {
1020    let mut status = restore.status.clone().unwrap_or_default();
1021    status.phase = phase;
1022    status.message = message;
1023    patch_status(restore, ctx, status).await
1024}
1025
1026async fn fail_after_mutation(
1027    restore: &KanidmRestore,
1028    ctx: &RestoreContext,
1029    message: &str,
1030) -> Result<()> {
1031    set_phase(
1032        restore,
1033        ctx,
1034        KanidmRestorePhase::Failed,
1035        Some(message.to_string()),
1036    )
1037    .await
1038}
1039
1040fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1041    target
1042        .spec
1043        .replica_groups
1044        .iter()
1045        .find(|rg| rg.primary_node)
1046        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1047}
1048
1049async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1050    let ns = target.namespace().unwrap();
1051    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1052    for rg in &target.spec.replica_groups {
1053        let name = target.statefulset_name(&rg.name);
1054        api.patch(
1055            &name,
1056            &PatchParams::default(),
1057            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1058        )
1059        .await
1060        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1061    }
1062    Ok(())
1063}
1064
1065async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1066    let ns = target.namespace().unwrap();
1067    let rg = primary_group(target)?;
1068    let name = target.statefulset_name(&rg.name);
1069    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1070        .patch(
1071            &name,
1072            &PatchParams::default(),
1073            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1074        )
1075        .await
1076        .map(|_| ())
1077        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1078}
1079
1080async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1081    let ns = target.namespace().unwrap();
1082    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1083    for rg in &target.spec.replica_groups {
1084        let name = target.statefulset_name(&rg.name);
1085        api.patch(
1086            &name,
1087            &PatchParams::default(),
1088            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1089        )
1090        .await
1091        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1092    }
1093    Ok(())
1094}
1095
1096async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1097    let ns = target.namespace().unwrap();
1098    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1099        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1100        .await
1101        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1102    Ok(pods.items.is_empty())
1103}
1104
1105async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1106    let ns = target.namespace().unwrap();
1107    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1108    let mut pv_names = BTreeSet::new();
1109    let primary_name = primary_pvc_name(target)?;
1110    let primary_pvc = pvc_api
1111        .get(&primary_name)
1112        .await
1113        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1114    let primary_pv = primary_pvc
1115        .spec
1116        .as_ref()
1117        .and_then(|spec| spec.volume_name.clone())
1118        .ok_or_else(|| {
1119            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1120        })?;
1121    pv_names.insert(primary_pv);
1122
1123    for rg in &target.spec.replica_groups {
1124        let sts = target.statefulset_name(&rg.name);
1125        for ordinal in 0..rg.replicas {
1126            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1127            if name == primary_name {
1128                continue;
1129            }
1130            if let Some(pvc) = pvc_api
1131                .get_opt(&name)
1132                .await
1133                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1134                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1135            {
1136                pv_names.insert(pv_name);
1137            }
1138        }
1139    }
1140
1141    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1142        .list(&ListParams::default())
1143        .await
1144        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1145    Ok(!attachments.items.iter().any(|attachment| {
1146        attachment
1147            .spec
1148            .source
1149            .persistent_volume_name
1150            .as_ref()
1151            .is_some_and(|pv_name| pv_names.contains(pv_name))
1152    }))
1153}
1154
1155async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1156    let ns = target.namespace().unwrap();
1157    let name = target.statefulset_name(&primary_group(target)?.name);
1158    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1159        .get(&name)
1160        .await
1161        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1162    Ok(sts
1163        .status
1164        .as_ref()
1165        .and_then(|s| s.ready_replicas)
1166        .unwrap_or(0)
1167        >= 1)
1168}
1169
1170fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1171    let rg = primary_group(target)?;
1172    Ok(format!(
1173        "{DATA_VOLUME}-{}-0",
1174        target.statefulset_name(&rg.name)
1175    ))
1176}
1177
1178async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1179    let ns = target.namespace().unwrap();
1180    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1181    let mut all_absent = true;
1182    for rg in &target.spec.replica_groups {
1183        let sts = target.statefulset_name(&rg.name);
1184        for ordinal in 0..rg.replicas {
1185            if rg.primary_node && ordinal == 0 {
1186                continue;
1187            }
1188            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1189            match api
1190                .get_opt(&name)
1191                .await
1192                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1193            {
1194                None => {}
1195                Some(pvc) => {
1196                    all_absent = false;
1197                    if pvc.metadata.deletion_timestamp.is_none() {
1198                        match api.delete(&name, &DeleteParams::default()).await {
1199                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1200                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1201                            Err(error) => {
1202                                return Err(Error::kube_error(
1203                                    "delete",
1204                                    "PersistentVolumeClaim",
1205                                    &ns,
1206                                    &name,
1207                                    error,
1208                                ));
1209                            }
1210                        }
1211                    }
1212                }
1213            }
1214        }
1215    }
1216    Ok(all_absent)
1217}
1218
1219async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1220    let ns = target.namespace().unwrap();
1221    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1222    let label_selector = format!(
1223        "{}={},{}={}",
1224        CLUSTER_LABEL,
1225        target.name_any(),
1226        SECRET_TYPE_LABEL,
1227        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1228    );
1229    let secrets = api
1230        .list(&ListParams::default().labels(&label_selector))
1231        .await
1232        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1233    let mut all_absent = secrets.items.is_empty();
1234    for secret in secrets.items {
1235        let name = secret.name_any();
1236        if secret.metadata.deletion_timestamp.is_none() {
1237            match api.delete(&name, &DeleteParams::default()).await {
1238                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1239                Err(kube::Error::Api(status)) if status.code == 404 => {}
1240                Err(error) => {
1241                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1242                }
1243            }
1244            all_absent = false;
1245        }
1246    }
1247    Ok(all_absent)
1248}
1249
1250async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1251    let ns = target.namespace().unwrap();
1252    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1253    let name = format!("{}-admin-passwords", target.name_any());
1254    match api
1255        .get_opt(&name)
1256        .await
1257        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1258    {
1259        None => Ok(true),
1260        Some(secret) => {
1261            if secret.metadata.deletion_timestamp.is_some() {
1262                return Ok(false);
1263            }
1264            match api.delete(&name, &DeleteParams::default()).await {
1265                Ok(_) => {
1266                    debug!(secret = %name, "deleting stale admin secret");
1267                    Ok(false)
1268                }
1269                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1270                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1271            }
1272        }
1273    }
1274}
1275
1276fn restore_job_name(restore: &KanidmRestore) -> String {
1277    format!("{}-restore", restore.name_any())
1278}
1279fn verify_job_name(restore: &KanidmRestore) -> String {
1280    format!("{}-verify", restore.name_any())
1281}
1282
1283fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1284    let origin = target
1285        .spec
1286        .origin
1287        .clone()
1288        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1289    vec![
1290        EnvVar {
1291            name: "KANIDM_DB_PATH".to_string(),
1292            value: Some("/data/kanidm.db".to_string()),
1293            ..Default::default()
1294        },
1295        EnvVar {
1296            name: "KANIDM_DOMAIN".to_string(),
1297            value: Some(target.spec.domain.clone()),
1298            ..Default::default()
1299        },
1300        EnvVar {
1301            name: "KANIDM_ORIGIN".to_string(),
1302            value: Some(origin),
1303            ..Default::default()
1304        },
1305        EnvVar {
1306            name: "KANIDM_TLS_CHAIN".to_string(),
1307            value: Some(format!("{TLS_PATH}/tls.crt")),
1308            ..Default::default()
1309        },
1310        EnvVar {
1311            name: "KANIDM_TLS_KEY".to_string(),
1312            value: Some(format!("{TLS_PATH}/tls.key")),
1313            ..Default::default()
1314        },
1315    ]
1316}
1317
1318async fn ensure_database_job(
1319    restore: &KanidmRestore,
1320    target: &Kanidm,
1321    ctx: &RestoreContext,
1322    name: &str,
1323    verify: bool,
1324) -> Result<()> {
1325    let ns = restore.namespace().unwrap();
1326    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1327    if api
1328        .get_opt(name)
1329        .await
1330        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1331        .is_some()
1332    {
1333        return Ok(());
1334    }
1335    let mut command = vec![
1336        "kanidmd".to_string(),
1337        "database".to_string(),
1338        if verify { "backup" } else { "restore" }.to_string(),
1339    ];
1340    if verify {
1341        command.push("/verify/verification.json.gz".to_string());
1342    } else {
1343        if is_remote_source(restore) {
1344            command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1345        } else {
1346            let file_name = restore
1347                .spec
1348                .source
1349                .local
1350                .as_ref()
1351                .map(|l| l.file_name.clone())
1352                .unwrap_or_default();
1353            command.push(format!("{BACKUP_PATH}/{file_name}"));
1354        }
1355    }
1356    let job = Job {
1357        metadata: ObjectMeta {
1358            name: Some(name.to_string()),
1359            namespace: Some(ns.clone()),
1360            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1361            ..Default::default()
1362        },
1363        spec: Some(JobSpec {
1364            backoff_limit: Some(0),
1365            template: PodTemplateSpec {
1366                metadata: Some(ObjectMeta::default()),
1367                spec: Some(PodSpec {
1368                    automount_service_account_token: Some(false),
1369                    restart_policy: Some("Never".to_string()),
1370                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1371                        seccomp_profile: Some(SeccompProfile {
1372                            type_: "RuntimeDefault".to_string(),
1373                            ..Default::default()
1374                        }),
1375                        ..Default::default()
1376                    }),
1377                    containers: vec![Container {
1378                        name: if verify { "verify" } else { "restore" }.to_string(),
1379                        image: Some(restore.spec.restore_image.clone()),
1380                        command: Some(command),
1381                        env: Some(build_kanidm_db_env_vars(target)),
1382                        security_context: Some(kanidm_job_security_context()),
1383                        resources: Some(job_resource_requirements()),
1384                        volume_mounts: Some(vec![
1385                            VolumeMount {
1386                                name: DATA_VOLUME.to_string(),
1387                                mount_path: DATA_PATH.to_string(),
1388                                ..Default::default()
1389                            },
1390                            VolumeMount {
1391                                name: TLS_VOLUME.to_string(),
1392                                mount_path: TLS_PATH.to_string(),
1393                                read_only: Some(true),
1394                                ..Default::default()
1395                            },
1396                            VolumeMount {
1397                                name: "verification-output".to_string(),
1398                                mount_path: "/verify".to_string(),
1399                                ..Default::default()
1400                            },
1401                        ]),
1402                        ..Default::default()
1403                    }],
1404                    volumes: Some(vec![
1405                        Volume {
1406                            name: DATA_VOLUME.to_string(),
1407                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1408                                claim_name: primary_pvc_name(target)?,
1409                                read_only: Some(false),
1410                            }),
1411                            ..Default::default()
1412                        },
1413                        Volume {
1414                            name: TLS_VOLUME.to_string(),
1415                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1416                                secret_name: Some(target.effective_tls_secret_name()),
1417                                ..Default::default()
1418                            }),
1419                            ..Default::default()
1420                        },
1421                        Volume {
1422                            name: "verification-output".to_string(),
1423                            empty_dir: Some(Default::default()),
1424                            ..Default::default()
1425                        },
1426                    ]),
1427                    ..Default::default()
1428                }),
1429            },
1430            ..Default::default()
1431        }),
1432        ..Default::default()
1433    };
1434    api.create(&PostParams::default(), &job)
1435        .await
1436        .map(|_| ())
1437        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1438}
1439
1440enum JobState {
1441    Running,
1442    Complete,
1443    Failed,
1444}
1445
1446async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1447    let ns = restore.namespace().unwrap();
1448    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1449        .get(name)
1450        .await
1451        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1452    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1453        return Ok(JobState::Failed);
1454    }
1455    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1456        return Ok(JobState::Complete);
1457    }
1458    Ok(JobState::Running)
1459}
1460
1461fn is_remote_source(restore: &KanidmRestore) -> bool {
1462    restore.spec.source.backup_ref.is_some()
1463}
1464
1465fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1466    let safety = restore.spec.safety_backup.as_ref();
1467    let skip = safety.is_some_and(|s| s.skip);
1468    !skip
1469}
1470
1471fn validate_source(restore: &KanidmRestore) -> Result<()> {
1472    let has_local = restore.spec.source.local.is_some();
1473    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1474    if has_local && has_backup_ref {
1475        return Err(Error::MissingData(
1476            "source.local and source.backupRef are mutually exclusive".to_string(),
1477        ));
1478    }
1479    if !has_local && !has_backup_ref {
1480        return Err(Error::MissingData(
1481            "source must specify either local or backupRef".to_string(),
1482        ));
1483    }
1484    if let Some(local) = &restore.spec.source.local {
1485        if !safe_basename(&local.file_name) {
1486            return Err(Error::MissingData(
1487                "restore source fileName must be a safe basename".to_string(),
1488            ));
1489        }
1490    }
1491    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1492        if backup_ref.name.is_empty() {
1493            return Err(Error::MissingData(
1494                "source.backupRef.name must not be empty".to_string(),
1495            ));
1496        }
1497    }
1498    Ok(())
1499}
1500
1501fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1502    let safety = restore.spec.safety_backup.as_ref();
1503    let skip = safety.is_some_and(|s| s.skip);
1504    if skip {
1505        let annotations = restore.metadata.annotations.as_ref();
1506        let reason = annotations
1507            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1508            .map(|s| s.trim())
1509            .unwrap_or("");
1510        let approver = annotations
1511            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1512            .map(|s| s.trim())
1513            .unwrap_or("");
1514        if reason.is_empty() {
1515            return Err(Error::MissingData(format!(
1516                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1517            )));
1518        }
1519        if approver.is_empty() {
1520            return Err(Error::MissingData(format!(
1521                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1522            )));
1523        }
1524    }
1525    if is_remote_source(restore) && !skip {
1526        let has_repo = safety
1527            .and_then(|s| s.repository_ref.as_ref())
1528            .is_some_and(|r| !r.name.is_empty());
1529        if !has_repo {
1530            return Err(Error::MissingData(
1531                "remote restore requires safetyBackup.repositoryRef".to_string(),
1532            ));
1533        }
1534    }
1535    Ok(())
1536}
1537
1538fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1539    target
1540        .spec
1541        .replica_groups
1542        .iter()
1543        .map(|rg| ReplicaCountEntry {
1544            group: rg.name.clone(),
1545            replicas: rg.replicas,
1546        })
1547        .collect()
1548}
1549
1550async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1551    if let Ok(target) = get_target(restore, ctx).await {
1552        let owns_maintenance =
1553            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1554        if owns_maintenance {
1555            scale_desired(&target, ctx).await?;
1556            clear_restoring(restore, &target, ctx).await?;
1557        }
1558    }
1559    Ok(())
1560}
1561
1562async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1563    ctx.metrics.break_glass_total.add(1, &[]);
1564    let reason = restore
1565        .metadata
1566        .annotations
1567        .as_ref()
1568        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1569        .cloned()
1570        .unwrap_or_default();
1571    let approver = restore
1572        .metadata
1573        .annotations
1574        .as_ref()
1575        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1576        .cloned()
1577        .unwrap_or_default();
1578    let mut status = restore.status.clone().unwrap_or_default();
1579    let break_glass_condition = restore_condition(
1580        &status.conditions,
1581        CONDITION_BREAK_GLASS,
1582        CONDITION_TRUE,
1583        "BreakGlassOverride",
1584        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1585        restore.metadata.generation,
1586    );
1587    status
1588        .conditions
1589        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1590    status.conditions.push(break_glass_condition);
1591    patch_status(restore, ctx, status).await?;
1592    if let Err(error) = ctx
1593        .recorder
1594        .publish(
1595            &Event {
1596                type_: EventType::Warning,
1597                reason: "SafetyBackupSkipped".to_string(),
1598                note: Some(format!(
1599                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1600                )),
1601                action: "Restore".to_string(),
1602                secondary: None,
1603            },
1604            &restore.object_ref(&()),
1605        )
1606        .await
1607    {
1608        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1609    }
1610    warn!(
1611        restore = %restore.name_any(),
1612        reason = %reason,
1613        approver = %approver,
1614        "break-glass safety backup override activated"
1615    );
1616    Ok(())
1617}
1618
1619async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1620    let status = restore.status.as_ref().cloned().unwrap_or_default();
1621    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1622        if let Ok(start_ts) = start.parse::<Timestamp>() {
1623            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1624            ctx.metrics
1625                .safety_backup_duration_seconds
1626                .record(elapsed, &[]);
1627        }
1628    }
1629}
1630
1631fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1632    format!("{}-safety-backup", restore.name_any())
1633}
1634
1635fn source_prep_job_name(restore: &KanidmRestore) -> String {
1636    format!("{}-source-prep", restore.name_any())
1637}
1638
1639fn source_check_job_name(restore: &KanidmRestore) -> String {
1640    format!("{}-source-check", restore.name_any())
1641}
1642
1643async fn ensure_source_check_job(
1644    restore: &KanidmRestore,
1645    target: &Kanidm,
1646    ctx: &RestoreContext,
1647    name: &str,
1648) -> Result<()> {
1649    let ns = restore.namespace().unwrap();
1650    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1651    if api
1652        .get_opt(name)
1653        .await
1654        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1655        .is_some()
1656    {
1657        return Ok(());
1658    }
1659    let file_name = restore
1660        .spec
1661        .source
1662        .local
1663        .as_ref()
1664        .map(|l| l.file_name.clone())
1665        .unwrap_or_default();
1666    let operation_doc = serde_json::json!({
1667        "apiVersion": "backup.kaniop.rs/v1alpha1",
1668        "kind": "OperationDocument",
1669        "operation": "check",
1670        "path": format!("{BACKUP_PATH}/{file_name}"),
1671        "resultPath": "/run/kaniop-result/result.json",
1672    })
1673    .to_string();
1674    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1675    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1676    let data_mover_image = data_mover_image();
1677    let job = Job {
1678        metadata: ObjectMeta {
1679            name: Some(name.to_string()),
1680            namespace: Some(ns.clone()),
1681            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1682            ..Default::default()
1683        },
1684        spec: Some(JobSpec {
1685            backoff_limit: Some(0),
1686            template: PodTemplateSpec {
1687                metadata: Some(ObjectMeta::default()),
1688                spec: Some(PodSpec {
1689                    automount_service_account_token: Some(false),
1690                    restart_policy: Some("Never".to_string()),
1691                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1692                        run_as_non_root: Some(true),
1693                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1694                            type_: "RuntimeDefault".to_string(),
1695                            ..Default::default()
1696                        }),
1697                        ..Default::default()
1698                    }),
1699                    containers: vec![Container {
1700                        name: "source-check".to_string(),
1701                        image: Some(data_mover_image),
1702                        command: Some(vec![
1703                            "/bin/kaniop-data-mover".to_string(),
1704                            "check".to_string(),
1705                            "--operation-doc".to_string(),
1706                            "/run/kaniop/operation.json".to_string(),
1707                        ]),
1708                        security_context: Some(hardened_security_context()),
1709                        resources: Some(job_resource_requirements()),
1710                        env: Some(vec![EnvVar {
1711                            name: "RUST_LOG".to_string(),
1712                            value: Some("info".to_string()),
1713                            ..Default::default()
1714                        }]),
1715                        volume_mounts: Some(vec![
1716                            VolumeMount {
1717                                name: DATA_VOLUME.to_string(),
1718                                mount_path: DATA_PATH.to_string(),
1719                                read_only: Some(true),
1720                                ..Default::default()
1721                            },
1722                            VolumeMount {
1723                                name: "operation".to_string(),
1724                                mount_path: "/run/kaniop".to_string(),
1725                                read_only: Some(true),
1726                                ..Default::default()
1727                            },
1728                            VolumeMount {
1729                                name: "result".to_string(),
1730                                mount_path: "/run/kaniop-result".to_string(),
1731                                ..Default::default()
1732                            },
1733                        ]),
1734                        ..Default::default()
1735                    }],
1736                    volumes: Some(vec![
1737                        Volume {
1738                            name: DATA_VOLUME.to_string(),
1739                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1740                                claim_name: primary_pvc_name(target)?,
1741                                read_only: Some(true),
1742                            }),
1743                            ..Default::default()
1744                        },
1745                        Volume {
1746                            name: "operation".to_string(),
1747                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1748                                name: operation_cm_name,
1749                                ..Default::default()
1750                            }),
1751                            ..Default::default()
1752                        },
1753                        Volume {
1754                            name: "result".to_string(),
1755                            empty_dir: Some(Default::default()),
1756                            ..Default::default()
1757                        },
1758                    ]),
1759                    ..Default::default()
1760                }),
1761            },
1762            ..Default::default()
1763        }),
1764        ..Default::default()
1765    };
1766    api.create(&PostParams::default(), &job)
1767        .await
1768        .map(|_| ())
1769        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1770}
1771
1772const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1773
1774fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1775    let ns = restore.namespace().unwrap_or_default();
1776    let name = restore.name_any();
1777    let namespace_uuid =
1778        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1779    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1780    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1781}
1782
1783fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1784    let ns = restore.namespace().unwrap_or_default();
1785    let kanidm_uid = &restore.spec.target_ref.uid;
1786    let backup_id = compute_safety_backup_id(restore);
1787    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1788}
1789
1790struct VerifiedSafetyBackup {
1791    manifest_key: String,
1792    payload_sha256: String,
1793}
1794
1795async fn read_safety_backup_result(
1796    restore: &KanidmRestore,
1797    ctx: &RestoreContext,
1798    job_name: &str,
1799    expected_backup_id: &str,
1800) -> Result<VerifiedSafetyBackup> {
1801    let ns = restore.namespace().unwrap();
1802    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1803    let pod_list = pods
1804        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1805        .await
1806        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1807    let pod = pod_list.items.first().ok_or_else(|| {
1808        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1809    })?;
1810    let pod_name = pod.name_any();
1811    let container_status = pod
1812        .status
1813        .as_ref()
1814        .and_then(|s| s.container_statuses.as_ref())
1815        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1816        .ok_or_else(|| {
1817            Error::MissingData(format!(
1818                "safety-upload container status not found in pod {pod_name}"
1819            ))
1820        })?;
1821    let termination_message = container_status
1822        .state
1823        .as_ref()
1824        .and_then(|state| state.terminated.as_ref())
1825        .and_then(|t| t.message.as_ref())
1826        .ok_or_else(|| {
1827            Error::MissingData(format!(
1828                "termination message not found in pod {pod_name}; result transport failed"
1829            ))
1830        })?;
1831    if termination_message.is_empty() {
1832        return Err(Error::MissingData(format!(
1833            "termination message is empty in pod {pod_name}; result document was not written"
1834        )));
1835    }
1836    let result_doc = parse_result_document(termination_message).map_err(|e| {
1837        Error::ParseError(format!(
1838            "failed to parse result document from pod {pod_name}: {e}"
1839        ))
1840    })?;
1841    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1842        return Err(Error::ParseError(format!(
1843            "unsupported result document version: {}",
1844            result_doc.api_version
1845        )));
1846    }
1847    if result_doc.kind != "ResultDocument" {
1848        return Err(Error::ParseError(format!(
1849            "invalid result document kind: {}",
1850            result_doc.kind
1851        )));
1852    }
1853    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1854        return Err(Error::ParseError(format!(
1855            "result document operation mismatch: expected '{}', got '{}'",
1856            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1857        )));
1858    }
1859    if !result_doc.success {
1860        return Err(Error::ParseError(format!(
1861            "safety backup reported failure with exit code {:?}",
1862            result_doc.exit_code
1863        )));
1864    }
1865    if result_doc.exit_code != ExitCode::Success {
1866        return Err(Error::ParseError(format!(
1867            "safety backup exit code is not Success: {:?}",
1868            result_doc.exit_code
1869        )));
1870    }
1871    let doc_backup_id = result_doc
1872        .backup_id
1873        .as_deref()
1874        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1875    if doc_backup_id != expected_backup_id {
1876        return Err(Error::ParseError(format!(
1877            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1878        )));
1879    }
1880    let manifest_key = result_doc
1881        .manifest_key
1882        .clone()
1883        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1884    if manifest_key.is_empty() {
1885        return Err(Error::ParseError(
1886            "result document manifestKey is empty".to_string(),
1887        ));
1888    }
1889    let payload_sha256 = result_doc
1890        .payload_sha256
1891        .clone()
1892        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1893    if payload_sha256.is_empty() {
1894        return Err(Error::ParseError(
1895            "result document payloadSha256 is empty".to_string(),
1896        ));
1897    }
1898    info!(
1899        restore = %restore.name_any(),
1900        backup_id = %expected_backup_id,
1901        manifest_key = %manifest_key,
1902        "safety backup result verified"
1903    );
1904    Ok(VerifiedSafetyBackup {
1905        manifest_key,
1906        payload_sha256,
1907    })
1908}
1909
1910struct VerifiedSourcePrepResult {
1911    #[allow(dead_code)]
1912    manifest_key: String,
1913    #[allow(dead_code)]
1914    payload_sha256: String,
1915}
1916
1917async fn read_source_prep_result(
1918    restore: &KanidmRestore,
1919    ctx: &RestoreContext,
1920    job_name: &str,
1921    expected_backup_id: &str,
1922) -> Result<VerifiedSourcePrepResult> {
1923    let ns = restore.namespace().unwrap();
1924    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1925    let pod_list = pods
1926        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1927        .await
1928        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1929    let pod = pod_list.items.first().ok_or_else(|| {
1930        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1931    })?;
1932    let pod_name = pod.name_any();
1933    let container_status = pod
1934        .status
1935        .as_ref()
1936        .and_then(|s| s.container_statuses.as_ref())
1937        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1938        .ok_or_else(|| {
1939            Error::MissingData(format!(
1940                "source-prep container status not found in pod {pod_name}"
1941            ))
1942        })?;
1943    let termination_message = container_status
1944        .state
1945        .as_ref()
1946        .and_then(|state| state.terminated.as_ref())
1947        .and_then(|t| t.message.as_ref())
1948        .ok_or_else(|| {
1949            Error::MissingData(format!(
1950                "termination message not found in pod {pod_name}; result transport failed"
1951            ))
1952        })?;
1953    if termination_message.is_empty() {
1954        return Err(Error::MissingData(format!(
1955            "termination message is empty in pod {pod_name}; result document was not written"
1956        )));
1957    }
1958    let result_doc = parse_result_document(termination_message).map_err(|e| {
1959        Error::ParseError(format!(
1960            "failed to parse result document from pod {pod_name}: {e}"
1961        ))
1962    })?;
1963    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1964        return Err(Error::ParseError(format!(
1965            "unsupported result document version: {}",
1966            result_doc.api_version
1967        )));
1968    }
1969    if result_doc.kind != "ResultDocument" {
1970        return Err(Error::ParseError(format!(
1971            "invalid result document kind: {}",
1972            result_doc.kind
1973        )));
1974    }
1975    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1976        return Err(Error::ParseError(format!(
1977            "result document operation mismatch: expected '{}', got '{}'",
1978            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1979        )));
1980    }
1981    if !result_doc.success {
1982        return Err(Error::ParseError(format!(
1983            "source prep reported failure with exit code {:?}",
1984            result_doc.exit_code
1985        )));
1986    }
1987    if result_doc.exit_code != ExitCode::Success {
1988        return Err(Error::ParseError(format!(
1989            "source prep exit code is not Success: {:?}",
1990            result_doc.exit_code
1991        )));
1992    }
1993    let doc_backup_id = result_doc
1994        .backup_id
1995        .as_deref()
1996        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1997    if doc_backup_id != expected_backup_id {
1998        return Err(Error::ParseError(format!(
1999            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2000        )));
2001    }
2002    let manifest_key = result_doc
2003        .manifest_key
2004        .clone()
2005        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2006    if manifest_key.is_empty() {
2007        return Err(Error::ParseError(
2008            "result document manifestKey is empty".to_string(),
2009        ));
2010    }
2011    let payload_sha256 = result_doc
2012        .payload_sha256
2013        .clone()
2014        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2015    if payload_sha256.is_empty() {
2016        return Err(Error::ParseError(
2017            "result document payloadSha256 is empty".to_string(),
2018        ));
2019    }
2020    info!(
2021        restore = %restore.name_any(),
2022        backup_id = %expected_backup_id,
2023        manifest_key = %manifest_key,
2024        payload_sha256 = %payload_sha256,
2025        "source prep result verified"
2026    );
2027    Ok(VerifiedSourcePrepResult {
2028        manifest_key,
2029        payload_sha256,
2030    })
2031}
2032
2033fn hardened_security_context() -> SecurityContext {
2034    SecurityContext {
2035        allow_privilege_escalation: Some(false),
2036        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2037            drop: Some(vec!["ALL".to_string()]),
2038            ..Default::default()
2039        }),
2040        read_only_root_filesystem: Some(true),
2041        run_as_non_root: Some(true),
2042        seccomp_profile: Some(SeccompProfile {
2043            type_: "RuntimeDefault".to_string(),
2044            ..Default::default()
2045        }),
2046        ..Default::default()
2047    }
2048}
2049
2050fn kanidm_job_security_context() -> SecurityContext {
2051    SecurityContext {
2052        allow_privilege_escalation: Some(false),
2053        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2054            drop: Some(vec!["ALL".to_string()]),
2055            ..Default::default()
2056        }),
2057        seccomp_profile: Some(SeccompProfile {
2058            type_: "RuntimeDefault".to_string(),
2059            ..Default::default()
2060        }),
2061        ..Default::default()
2062    }
2063}
2064
2065fn job_resource_requirements() -> ResourceRequirements {
2066    ResourceRequirements {
2067        requests: Some(std::collections::BTreeMap::from([
2068            ("cpu".to_string(), Quantity("100m".to_string())),
2069            ("memory".to_string(), Quantity("128Mi".to_string())),
2070        ])),
2071        limits: Some(std::collections::BTreeMap::from([
2072            ("cpu".to_string(), Quantity("2".to_string())),
2073            ("memory".to_string(), Quantity("2Gi".to_string())),
2074        ])),
2075        ..Default::default()
2076    }
2077}
2078
2079async fn ensure_safety_backup_job(
2080    restore: &KanidmRestore,
2081    target: &Kanidm,
2082    ctx: &RestoreContext,
2083    name: &str,
2084    backup_id: &str,
2085    manifest_key: &str,
2086) -> Result<()> {
2087    let ns = restore.namespace().unwrap();
2088    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2089    if api
2090        .get_opt(name)
2091        .await
2092        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2093        .is_some()
2094    {
2095        return Ok(());
2096    }
2097    let safety = restore
2098        .spec
2099        .safety_backup
2100        .as_ref()
2101        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2102    let repo_ref = safety
2103        .repository_ref
2104        .as_ref()
2105        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2106    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2107        .get(&repo_ref.name)
2108        .await
2109        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2110    let _namespace_uid = &ns;
2111    let _kanidm_uid = &restore.spec.target_ref.uid;
2112    let endpoint = repo
2113        .spec
2114        .s3
2115        .endpoint
2116        .as_deref()
2117        .unwrap_or("https://s3.amazonaws.com");
2118    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2119    let operation_doc = build_safety_upload_operation_doc(
2120        restore,
2121        target,
2122        backup_id,
2123        manifest_key,
2124        &repo.spec.s3.bucket,
2125        &repo.spec.s3.prefix,
2126        endpoint,
2127        region,
2128        repo.spec.s3.force_path_style,
2129        repo.spec.s3.insecure,
2130        repo.spec
2131            .s3
2132            .ca_bundle_ref
2133            .as_deref()
2134            .map(|_| ca_bundle_path())
2135            .as_deref(),
2136    )?;
2137    let operation_cm_name = format!("{name}-op");
2138    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2139    let auth_method = &repo.spec.authentication.writer;
2140    let data_mover_image = data_mover_image();
2141    let init_command = vec![
2142        "kanidmd".to_string(),
2143        "database".to_string(),
2144        "backup".to_string(),
2145        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2146    ];
2147    let upload_script = format!(
2148        r#"set -eu
2149/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2150RESULT_FILE="/run/kaniop-result/result.json"
2151TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2152if [ ! -f "$RESULT_FILE" ]; then
2153  echo "ERROR: result document not found at $RESULT_FILE"
2154  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2155  exit 1
2156fi
2157if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2158  echo "ERROR: safety backup upload reported failure"
2159  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2160  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2161  exit 1
2162fi
2163dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2164echo "safety backup upload completed successfully"
2165"#
2166    );
2167    let job = Job {
2168        metadata: ObjectMeta {
2169            name: Some(name.to_string()),
2170            namespace: Some(ns.clone()),
2171            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2172            ..Default::default()
2173        },
2174        spec: Some(JobSpec {
2175            backoff_limit: Some(0),
2176            template: PodTemplateSpec {
2177                metadata: Some(ObjectMeta::default()),
2178                spec: Some(PodSpec {
2179                    automount_service_account_token: Some(false),
2180                    restart_policy: Some("Never".to_string()),
2181                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2182                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2183                            type_: "RuntimeDefault".to_string(),
2184                            ..Default::default()
2185                        }),
2186                        ..Default::default()
2187                    }),
2188                    init_containers: Some(vec![Container {
2189                        name: "safety-backup".to_string(),
2190                        image: Some(restore.spec.restore_image.clone()),
2191                        command: Some(init_command),
2192                        security_context: Some(SecurityContext {
2193                            allow_privilege_escalation: Some(false),
2194                            capabilities: Some(Capabilities {
2195                                drop: Some(vec!["ALL".to_string()]),
2196                                ..Default::default()
2197                            }),
2198                            read_only_root_filesystem: Some(true),
2199                            seccomp_profile: Some(SeccompProfile {
2200                                type_: "RuntimeDefault".to_string(),
2201                                ..Default::default()
2202                            }),
2203                            ..Default::default()
2204                        }),
2205                        resources: Some(job_resource_requirements()),
2206                        env: Some(build_kanidm_db_env_vars(target)),
2207                        volume_mounts: Some(vec![
2208                            VolumeMount {
2209                                name: DATA_VOLUME.to_string(),
2210                                mount_path: DATA_PATH.to_string(),
2211                                read_only: Some(false),
2212                                ..Default::default()
2213                            },
2214                            VolumeMount {
2215                                name: TLS_VOLUME.to_string(),
2216                                mount_path: TLS_PATH.to_string(),
2217                                read_only: Some(true),
2218                                ..Default::default()
2219                            },
2220                            VolumeMount {
2221                                name: SHARED_VOLUME.to_string(),
2222                                mount_path: SHARED_VOL_PATH.to_string(),
2223                                ..Default::default()
2224                            },
2225                        ]),
2226                        ..Default::default()
2227                    }]),
2228                    containers: vec![Container {
2229                        name: "safety-upload".to_string(),
2230                        image: Some(data_mover_image),
2231                        command: Some(vec!["/bin/sh".to_string()]),
2232                        args: Some(vec!["-c".to_string(), upload_script]),
2233                        security_context: Some(hardened_security_context()),
2234                        resources: Some(job_resource_requirements()),
2235                        env: {
2236                            let mut env_vars =
2237                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2238                            env_vars.push(EnvVar {
2239                                name: "RUST_LOG".to_string(),
2240                                value: Some("info".to_string()),
2241                                ..Default::default()
2242                            });
2243                            if repo.spec.s3.ca_bundle_ref.is_some() {
2244                                env_vars.push(ca_bundle_env_var());
2245                            }
2246                            Some(env_vars)
2247                        },
2248                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2249                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2250                        volume_mounts: {
2251                            let mut mounts = vec![
2252                                VolumeMount {
2253                                    name: SHARED_VOLUME.to_string(),
2254                                    mount_path: SHARED_VOL_PATH.to_string(),
2255                                    read_only: Some(true),
2256                                    ..Default::default()
2257                                },
2258                                VolumeMount {
2259                                    name: "operation".to_string(),
2260                                    mount_path: "/run/kaniop".to_string(),
2261                                    read_only: Some(true),
2262                                    ..Default::default()
2263                                },
2264                                VolumeMount {
2265                                    name: "result".to_string(),
2266                                    mount_path: "/run/kaniop-result".to_string(),
2267                                    ..Default::default()
2268                                },
2269                            ];
2270                            mounts.extend(build_auth_volume_mounts(auth_method));
2271                            if repo.spec.s3.ca_bundle_ref.is_some() {
2272                                mounts.push(build_ca_bundle_volume_mount());
2273                            }
2274                            Some(mounts)
2275                        },
2276                        ..Default::default()
2277                    }],
2278                    volumes: {
2279                        let mut vols = vec![
2280                            Volume {
2281                                name: DATA_VOLUME.to_string(),
2282                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2283                                    claim_name: primary_pvc_name(target)?,
2284                                    read_only: Some(false),
2285                                }),
2286                                ..Default::default()
2287                            },
2288                            Volume {
2289                                name: TLS_VOLUME.to_string(),
2290                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2291                                    secret_name: Some(target.effective_tls_secret_name()),
2292                                    ..Default::default()
2293                                }),
2294                                ..Default::default()
2295                            },
2296                            Volume {
2297                                name: SHARED_VOLUME.to_string(),
2298                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2299                                    size_limit: Some(Quantity("10Gi".to_string())),
2300                                    ..Default::default()
2301                                }),
2302                                ..Default::default()
2303                            },
2304                            Volume {
2305                                name: "operation".to_string(),
2306                                config_map: Some(ConfigMapVolumeSource {
2307                                    name: operation_cm_name,
2308                                    ..Default::default()
2309                                }),
2310                                ..Default::default()
2311                            },
2312                            Volume {
2313                                name: "result".to_string(),
2314                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2315                                    size_limit: Some(Quantity("16Mi".to_string())),
2316                                    ..Default::default()
2317                                }),
2318                                ..Default::default()
2319                            },
2320                        ];
2321                        vols.extend(build_auth_volumes(auth_method));
2322                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2323                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2324                        }
2325                        Some(vols)
2326                    },
2327                    ..Default::default()
2328                }),
2329            },
2330            ..Default::default()
2331        }),
2332        ..Default::default()
2333    };
2334    api.create(&PostParams::default(), &job)
2335        .await
2336        .map(|_| ())
2337        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2338}
2339
2340async fn ensure_source_prep_job(
2341    restore: &KanidmRestore,
2342    target: &Kanidm,
2343    ctx: &RestoreContext,
2344    name: &str,
2345) -> Result<()> {
2346    let ns = restore.namespace().unwrap();
2347    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2348    if api
2349        .get_opt(name)
2350        .await
2351        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2352        .is_some()
2353    {
2354        return Ok(());
2355    }
2356    let backup_name = restore
2357        .spec
2358        .source
2359        .backup_ref
2360        .as_ref()
2361        .map(|b| b.name.clone())
2362        .unwrap_or_default();
2363    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2364        .get(&backup_name)
2365        .await
2366        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2367    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2368        .get(&backup.spec.repository_ref.name)
2369        .await
2370        .map_err(|e| {
2371            Error::kube_error(
2372                "get",
2373                "KanidmBackupRepository",
2374                &ns,
2375                &backup.spec.repository_ref.name,
2376                e,
2377            )
2378        })?;
2379    let endpoint = repo
2380        .spec
2381        .s3
2382        .endpoint
2383        .as_deref()
2384        .unwrap_or("https://s3.amazonaws.com");
2385    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2386    let operation_doc = build_download_operation_doc(
2387        restore,
2388        target,
2389        &backup.spec.manifest_key,
2390        &backup.spec.backup_id,
2391        &repo.spec.s3.bucket,
2392        &repo.spec.s3.prefix,
2393        endpoint,
2394        region,
2395        repo.spec.s3.force_path_style,
2396        repo.spec.s3.insecure,
2397        repo.spec
2398            .s3
2399            .ca_bundle_ref
2400            .as_deref()
2401            .map(|_| ca_bundle_path())
2402            .as_deref(),
2403    );
2404    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2405    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2406    let auth_method = &repo.spec.authentication.reader;
2407    let data_mover_image = data_mover_image();
2408    let download_script = format!(
2409        r#"set -eu
2410/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2411RESULT_FILE="/run/kaniop-result/result.json"
2412TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2413if [ ! -f "$RESULT_FILE" ]; then
2414  echo "ERROR: result document not found at $RESULT_FILE"
2415  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2416  exit 1
2417fi
2418if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2419  echo "ERROR: source preparation download reported failure"
2420  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2421  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2422  exit 1
2423fi
2424dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2425echo "source preparation download completed successfully"
2426"#
2427    );
2428    let job = Job {
2429        metadata: ObjectMeta {
2430            name: Some(name.to_string()),
2431            namespace: Some(ns.clone()),
2432            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2433            ..Default::default()
2434        },
2435        spec: Some(JobSpec {
2436            backoff_limit: Some(0),
2437            template: PodTemplateSpec {
2438                metadata: Some(ObjectMeta::default()),
2439                spec: Some(PodSpec {
2440                    automount_service_account_token: Some(false),
2441                    restart_policy: Some("Never".to_string()),
2442                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2443                        run_as_non_root: Some(true),
2444                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2445                            type_: "RuntimeDefault".to_string(),
2446                            ..Default::default()
2447                        }),
2448                        ..Default::default()
2449                    }),
2450                    containers: vec![Container {
2451                        name: "source-prep".to_string(),
2452                        image: Some(data_mover_image),
2453                        command: Some(vec!["/bin/sh".to_string()]),
2454                        args: Some(vec!["-c".to_string(), download_script]),
2455                        security_context: Some(hardened_security_context()),
2456                        resources: Some(job_resource_requirements()),
2457                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2458                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2459                        env: {
2460                            let mut env_vars = build_auth_env_vars(
2461                                auth_method,
2462                                &backup.spec.repository_ref.name,
2463                                AuthRole::Reader,
2464                            );
2465                            env_vars.push(EnvVar {
2466                                name: "RUST_LOG".to_string(),
2467                                value: Some("info".to_string()),
2468                                ..Default::default()
2469                            });
2470                            if repo.spec.s3.ca_bundle_ref.is_some() {
2471                                env_vars.push(ca_bundle_env_var());
2472                            }
2473                            Some(env_vars)
2474                        },
2475                        volume_mounts: {
2476                            let mut mounts = vec![
2477                                VolumeMount {
2478                                    name: STAGING_VOLUME.to_string(),
2479                                    mount_path: STAGING_PATH.to_string(),
2480                                    ..Default::default()
2481                                },
2482                                VolumeMount {
2483                                    name: "operation".to_string(),
2484                                    mount_path: "/run/kaniop".to_string(),
2485                                    read_only: Some(true),
2486                                    ..Default::default()
2487                                },
2488                                VolumeMount {
2489                                    name: "result".to_string(),
2490                                    mount_path: "/run/kaniop-result".to_string(),
2491                                    ..Default::default()
2492                                },
2493                            ];
2494                            mounts.extend(build_auth_volume_mounts(auth_method));
2495                            if repo.spec.s3.ca_bundle_ref.is_some() {
2496                                mounts.push(build_ca_bundle_volume_mount());
2497                            }
2498                            Some(mounts)
2499                        },
2500                        ..Default::default()
2501                    }],
2502                    volumes: {
2503                        let mut vols = vec![
2504                            Volume {
2505                                name: STAGING_VOLUME.to_string(),
2506                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2507                                    size_limit: Some(Quantity("10Gi".to_string())),
2508                                    ..Default::default()
2509                                }),
2510                                ..Default::default()
2511                            },
2512                            Volume {
2513                                name: "operation".to_string(),
2514                                config_map: Some(ConfigMapVolumeSource {
2515                                    name: operation_cm_name,
2516                                    ..Default::default()
2517                                }),
2518                                ..Default::default()
2519                            },
2520                            Volume {
2521                                name: "result".to_string(),
2522                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2523                                    size_limit: Some(Quantity("16Mi".to_string())),
2524                                    ..Default::default()
2525                                }),
2526                                ..Default::default()
2527                            },
2528                        ];
2529                        vols.extend(build_auth_volumes(auth_method));
2530                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2531                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2532                        }
2533                        Some(vols)
2534                    },
2535                    ..Default::default()
2536                }),
2537            },
2538            ..Default::default()
2539        }),
2540        ..Default::default()
2541    };
2542    api.create(&PostParams::default(), &job)
2543        .await
2544        .map(|_| ())
2545        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2546}
2547
2548#[allow(clippy::too_many_arguments)]
2549fn build_safety_upload_operation_doc(
2550    restore: &KanidmRestore,
2551    target: &Kanidm,
2552    backup_id: &str,
2553    manifest_key: &str,
2554    bucket: &str,
2555    prefix: &str,
2556    endpoint: &str,
2557    region: &str,
2558    force_path_style: bool,
2559    insecure: bool,
2560    ca_bundle_path: Option<&str>,
2561) -> Result<String> {
2562    let op = UploadOperation {
2563        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2564        bucket: bucket.to_string(),
2565        prefix: prefix.to_string(),
2566        endpoint: endpoint.to_string(),
2567        region: region.to_string(),
2568        force_path_style,
2569        insecure,
2570        ca_bundle_path: ca_bundle_path.map(str::to_string),
2571        backup_id: backup_id.to_string(),
2572        namespace_uid: restore.namespace().unwrap_or_default(),
2573        kanidm_uid: restore.spec.target_ref.uid.clone(),
2574        kanidm_name: restore.spec.target_ref.name.clone(),
2575        domain: target.spec.domain.clone(),
2576        kanidm_version: target
2577            .status
2578            .as_ref()
2579            .and_then(|s| s.version.as_ref())
2580            .map(|v| v.image_tag.clone())
2581            .unwrap_or_default(),
2582        image_digest: None,
2583        consistency: "kanidm-offline".to_string(),
2584        reason: "restore-safety".to_string(),
2585        encryption_mode: None,
2586        encryption_key_id: None,
2587        result_path: "/run/kaniop-result/result.json".to_string(),
2588        max_concurrent_parts: 4,
2589        max_retries: 3,
2590    };
2591    let _ = manifest_key;
2592    let doc = OperationDocument {
2593        api_version: OPERATION_DOC_VERSION.to_string(),
2594        kind: "OperationDocument".to_string(),
2595        spec: OperationSpec::Upload(op),
2596    };
2597    serde_json::to_string(&doc)
2598        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2599}
2600
2601async fn validate_backup_ref(
2602    restore: &KanidmRestore,
2603    target: &Kanidm,
2604    ctx: &RestoreContext,
2605) -> Result<()> {
2606    let backup_ref = restore
2607        .spec
2608        .source
2609        .backup_ref
2610        .as_ref()
2611        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2612    let ns = restore.namespace().unwrap();
2613    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2614        .get(&backup_ref.name)
2615        .await
2616        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2617    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2618        return Err(Error::MissingData(format!(
2619            "KanidmBackup '{}' is not Ready (phase: {:?})",
2620            backup_ref.name,
2621            backup.status.as_ref().map(|s| &s.phase)
2622        )));
2623    }
2624    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2625        return Err(Error::MissingData(format!(
2626            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2627            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2628        )));
2629    }
2630    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2631        return Err(Error::MissingData(format!(
2632            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2633            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2634        )));
2635    }
2636    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2637        .get(&backup.spec.repository_ref.name)
2638        .await
2639        .map_err(|e| {
2640            Error::kube_error(
2641                "get",
2642                "KanidmBackupRepository",
2643                &ns,
2644                &backup.spec.repository_ref.name,
2645                e,
2646            )
2647        })?;
2648    if !has_accepted_condition(
2649        &repo
2650            .status
2651            .as_ref()
2652            .map(|s| &s.conditions)
2653            .cloned()
2654            .unwrap_or_default(),
2655    ) {
2656        return Err(Error::MissingData(format!(
2657            "KanidmBackupRepository '{}' configuration has not been accepted",
2658            backup.spec.repository_ref.name
2659        )));
2660    }
2661    validate_backup_compatibility(restore, target, &backup)?;
2662    Ok(())
2663}
2664
2665fn has_accepted_condition(conditions: &[Condition]) -> bool {
2666    conditions
2667        .iter()
2668        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2669}
2670
2671fn validate_backup_compatibility(
2672    restore: &KanidmRestore,
2673    target: &Kanidm,
2674    backup: &KanidmBackup,
2675) -> Result<()> {
2676    let status = backup.status.as_ref();
2677    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2678        let target_version = target
2679            .status
2680            .as_ref()
2681            .and_then(|s| s.version.as_ref())
2682            .map(|v| v.image_tag.as_str());
2683        if let Some(tv) = target_version {
2684            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2685                return Err(Error::MissingData(format!(
2686                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2687                )));
2688            }
2689        }
2690    }
2691    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2692        if !backup_digest.is_empty()
2693            && restore.spec.restore_image.contains('@')
2694            && !restore.spec.restore_image.contains(backup_digest)
2695        {
2696            return Err(Error::MissingData(format!(
2697                "restore image digest does not match backup image digest '{backup_digest}'"
2698            )));
2699        }
2700    }
2701    Ok(())
2702}
2703
2704#[allow(clippy::too_many_arguments)]
2705fn build_download_operation_doc(
2706    restore: &KanidmRestore,
2707    target: &Kanidm,
2708    manifest_key: &str,
2709    expected_backup_id: &str,
2710    bucket: &str,
2711    prefix: &str,
2712    endpoint: &str,
2713    region: &str,
2714    force_path_style: bool,
2715    insecure: bool,
2716    ca_bundle_path: Option<&str>,
2717) -> String {
2718    serde_json::json!({
2719        "apiVersion": "backup.kaniop.rs/v1alpha1",
2720        "kind": "OperationDocument",
2721        "operation": "download",
2722        "manifestKey": manifest_key,
2723        "bucket": bucket,
2724        "prefix": prefix,
2725        "endpoint": endpoint,
2726        "region": region,
2727        "forcePathStyle": force_path_style,
2728        "insecure": insecure,
2729        "caBundlePath": ca_bundle_path,
2730        "expectedBackupId": expected_backup_id,
2731        "expectedKanidmUid": restore.spec.target_ref.uid,
2732        "expectedDomain": target.spec.domain,
2733        "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2734        "resultPath": "/run/kaniop-result/result.json",
2735        "maxRetries": 3,
2736    })
2737    .to_string()
2738}
2739
2740async fn ensure_operation_configmap(
2741    restore: &KanidmRestore,
2742    name: &str,
2743    data: &str,
2744    ns: &str,
2745    ctx: &RestoreContext,
2746) -> Result<()> {
2747    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2748    if api
2749        .get_opt(name)
2750        .await
2751        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2752        .is_some()
2753    {
2754        return Ok(());
2755    }
2756    let cm = ConfigMap {
2757        metadata: ObjectMeta {
2758            name: Some(name.to_string()),
2759            namespace: Some(ns.to_string()),
2760            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2761            ..Default::default()
2762        },
2763        data: Some(std::collections::BTreeMap::from([(
2764            "operation.json".to_string(),
2765            data.to_string(),
2766        )])),
2767        ..Default::default()
2768    };
2769    api.create(&PostParams::default(), &cm)
2770        .await
2771        .map(|_| ())
2772        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2773}
2774
2775#[cfg(test)]
2776mod tests {
2777    use super::{
2778        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2779        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2780        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2781        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2782        hardened_security_context, has_accepted_condition, is_remote_source,
2783        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2784        validate_safety_backup_config, validate_source,
2785    };
2786    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2787    use kube::api::ObjectMeta as ApiObjectMeta;
2788    use std::collections::BTreeMap;
2789
2790    fn make_restore(
2791        source: KanidmRestoreSource,
2792        safety: Option<SafetyBackupConfig>,
2793    ) -> KanidmRestore {
2794        KanidmRestore {
2795            metadata: ApiObjectMeta {
2796                name: Some("test-restore".to_string()),
2797                namespace: Some("default".to_string()),
2798                ..Default::default()
2799            },
2800            spec: KanidmRestoreSpec {
2801                target_ref: KanidmRestoreTargetRef {
2802                    name: "corp-idm".to_string(),
2803                    uid: "test-uid-123".to_string(),
2804                },
2805                source,
2806                restore_image: "kanidm/server@sha256:abc".to_string(),
2807                safety_backup: safety,
2808            },
2809            status: None,
2810        }
2811    }
2812
2813    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2814        super::super::crd::Kanidm {
2815            status: Some(super::super::crd::KanidmStatus {
2816                available_replicas: 1,
2817                replicas: 1,
2818                unavailable_replicas: 0,
2819                updated_replicas: 1,
2820                replica_statuses: vec![],
2821                replica_column: "1/1".to_string(),
2822                secret_name: None,
2823                version: Some(super::super::crd::KanidmVersionStatus {
2824                    image_tag: image_tag.to_string(),
2825                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2826                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2827                }),
2828                domain_appearance_image: None,
2829                mail_sender: None,
2830                conditions: None,
2831            }),
2832            ..Default::default()
2833        }
2834    }
2835
2836    #[test]
2837    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2838        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2839    }
2840
2841    #[test]
2842    fn completed_restore_sets_ready_condition() {
2843        let mut status = KanidmRestoreStatus {
2844            phase: KanidmRestorePhase::Completed,
2845            ..Default::default()
2846        };
2847        super::update_restore_conditions(&mut status, Some(7));
2848        assert!(status.conditions.iter().any(|condition| {
2849            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2850        }));
2851        assert!(!status.conditions.iter().any(|condition| {
2852            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2853        }));
2854    }
2855
2856    #[test]
2857    fn failed_restore_sets_failed_condition() {
2858        let mut status = KanidmRestoreStatus {
2859            phase: KanidmRestorePhase::Failed,
2860            message: Some("verification failed".to_string()),
2861            ..Default::default()
2862        };
2863        super::update_restore_conditions(&mut status, Some(8));
2864        assert!(status.conditions.iter().any(|condition| {
2865            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2866        }));
2867    }
2868
2869    #[test]
2870    fn restore_filename_is_confined_to_backup_directory() {
2871        assert!(safe_basename("backup.json.gz"));
2872        assert!(!safe_basename("../backup.json.gz"));
2873        assert!(!safe_basename("a/b"));
2874        assert!(!safe_basename(""));
2875    }
2876
2877    #[test]
2878    fn restore_image_rejects_latest_and_unpinned_names() {
2879        assert!(mutable_image("kanidm/server:latest"));
2880        assert!(mutable_image("kanidm/server"));
2881        assert!(!mutable_image("kanidm/server:1.10.0"));
2882        assert!(!mutable_image("kanidm/server@sha256:abc"));
2883    }
2884
2885    #[test]
2886    fn source_validation_rejects_both_local_and_backup_ref() {
2887        let source = KanidmRestoreSource {
2888            local: Some(super::KanidmRestoreLocalSource {
2889                file_name: "backup.json".to_string(),
2890            }),
2891            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2892                name: "some-backup".to_string(),
2893            }),
2894        };
2895        let restore = make_restore(source, None);
2896        assert!(validate_source(&restore).is_err());
2897    }
2898
2899    #[test]
2900    fn source_validation_rejects_empty_source() {
2901        let source = KanidmRestoreSource {
2902            local: None,
2903            backup_ref: None,
2904        };
2905        let restore = make_restore(source, None);
2906        assert!(validate_source(&restore).is_err());
2907    }
2908
2909    #[test]
2910    fn source_validation_accepts_local_source() {
2911        let source = KanidmRestoreSource {
2912            local: Some(super::KanidmRestoreLocalSource {
2913                file_name: "backup.json".to_string(),
2914            }),
2915            backup_ref: None,
2916        };
2917        let restore = make_restore(source, None);
2918        assert!(validate_source(&restore).is_ok());
2919    }
2920
2921    #[test]
2922    fn source_validation_accepts_backup_ref_source() {
2923        let source = KanidmRestoreSource {
2924            local: None,
2925            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2926                name: "corp-idm-019c7c76".to_string(),
2927            }),
2928        };
2929        let restore = make_restore(source, None);
2930        assert!(validate_source(&restore).is_ok());
2931    }
2932
2933    #[test]
2934    fn source_validation_rejects_empty_backup_ref_name() {
2935        let source = KanidmRestoreSource {
2936            local: None,
2937            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2938                name: "".to_string(),
2939            }),
2940        };
2941        let restore = make_restore(source, None);
2942        assert!(validate_source(&restore).is_err());
2943    }
2944
2945    #[test]
2946    fn is_remote_source_true_for_backup_ref() {
2947        let source = KanidmRestoreSource {
2948            local: None,
2949            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2950                name: "backup-1".to_string(),
2951            }),
2952        };
2953        let restore = make_restore(source, None);
2954        assert!(is_remote_source(&restore));
2955    }
2956
2957    #[test]
2958    fn is_remote_source_false_for_local() {
2959        let source = KanidmRestoreSource {
2960            local: Some(super::KanidmRestoreLocalSource {
2961                file_name: "backup.json".to_string(),
2962            }),
2963            backup_ref: None,
2964        };
2965        let restore = make_restore(source, None);
2966        assert!(!is_remote_source(&restore));
2967    }
2968
2969    #[test]
2970    fn requires_safety_backup_default_true() {
2971        let source = KanidmRestoreSource {
2972            local: None,
2973            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2974                name: "backup-1".to_string(),
2975            }),
2976        };
2977        let restore = make_restore(source, None);
2978        assert!(requires_safety_backup(&restore));
2979    }
2980
2981    #[test]
2982    fn requires_safety_backup_false_when_skip() {
2983        let source = KanidmRestoreSource {
2984            local: None,
2985            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2986                name: "backup-1".to_string(),
2987            }),
2988        };
2989        let safety = SafetyBackupConfig {
2990            repository_ref: Some(SafetyBackupRepositoryRef {
2991                name: "offsite".to_string(),
2992            }),
2993            skip: true,
2994        };
2995        let restore = make_restore(source, Some(safety));
2996        assert!(!requires_safety_backup(&restore));
2997    }
2998
2999    #[test]
3000    fn break_glass_validation_rejects_skip_without_reason() {
3001        let source = KanidmRestoreSource {
3002            local: None,
3003            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3004                name: "backup-1".to_string(),
3005            }),
3006        };
3007        let safety = SafetyBackupConfig {
3008            repository_ref: Some(SafetyBackupRepositoryRef {
3009                name: "offsite".to_string(),
3010            }),
3011            skip: true,
3012        };
3013        let mut restore = make_restore(source, Some(safety));
3014        restore.metadata.annotations = Some(BTreeMap::from([(
3015            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3016            "commander@example.com".to_string(),
3017        )]));
3018        assert!(validate_safety_backup_config(&restore).is_err());
3019    }
3020
3021    #[test]
3022    fn break_glass_validation_rejects_skip_without_approver() {
3023        let source = KanidmRestoreSource {
3024            local: None,
3025            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3026                name: "backup-1".to_string(),
3027            }),
3028        };
3029        let safety = SafetyBackupConfig {
3030            repository_ref: Some(SafetyBackupRepositoryRef {
3031                name: "offsite".to_string(),
3032            }),
3033            skip: true,
3034        };
3035        let mut restore = make_restore(source, Some(safety));
3036        restore.metadata.annotations = Some(BTreeMap::from([(
3037            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3038            "PVC is unreadable".to_string(),
3039        )]));
3040        assert!(validate_safety_backup_config(&restore).is_err());
3041    }
3042
3043    #[test]
3044    fn break_glass_validation_accepts_valid_annotations() {
3045        let source = KanidmRestoreSource {
3046            local: None,
3047            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3048                name: "backup-1".to_string(),
3049            }),
3050        };
3051        let safety = SafetyBackupConfig {
3052            repository_ref: Some(SafetyBackupRepositoryRef {
3053                name: "offsite".to_string(),
3054            }),
3055            skip: true,
3056        };
3057        let mut restore = make_restore(source, Some(safety));
3058        restore.metadata.annotations = Some(BTreeMap::from([
3059            (
3060                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3061                "PVC is unreadable".to_string(),
3062            ),
3063            (
3064                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3065                "commander@example.com".to_string(),
3066            ),
3067        ]));
3068        assert!(validate_safety_backup_config(&restore).is_ok());
3069    }
3070
3071    #[test]
3072    fn remote_source_requires_repository_ref_when_not_skipping() {
3073        let source = KanidmRestoreSource {
3074            local: None,
3075            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3076                name: "backup-1".to_string(),
3077            }),
3078        };
3079        let safety = SafetyBackupConfig {
3080            repository_ref: None,
3081            skip: false,
3082        };
3083        let restore = make_restore(source, Some(safety));
3084        assert!(validate_safety_backup_config(&restore).is_err());
3085    }
3086
3087    #[test]
3088    fn remote_source_accepts_repository_ref() {
3089        let source = KanidmRestoreSource {
3090            local: None,
3091            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3092                name: "backup-1".to_string(),
3093            }),
3094        };
3095        let safety = SafetyBackupConfig {
3096            repository_ref: Some(SafetyBackupRepositoryRef {
3097                name: "offsite".to_string(),
3098            }),
3099            skip: false,
3100        };
3101        let restore = make_restore(source, Some(safety));
3102        assert!(validate_safety_backup_config(&restore).is_ok());
3103    }
3104
3105    #[test]
3106    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3107        let phases = vec![
3108            KanidmRestorePhase::Pending,
3109            KanidmRestorePhase::Validating,
3110            KanidmRestorePhase::Quiescing,
3111            KanidmRestorePhase::SafetyBackup,
3112            KanidmRestorePhase::PreparingSource,
3113            KanidmRestorePhase::RestoringPrimary,
3114            KanidmRestorePhase::Verifying,
3115            KanidmRestorePhase::RebuildingReplicas,
3116            KanidmRestorePhase::Resuming,
3117            KanidmRestorePhase::Completed,
3118        ];
3119        assert_eq!(phases.len(), 10);
3120        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3121        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3122    }
3123
3124    #[test]
3125    fn status_default_has_empty_phase_timestamps() {
3126        let status = KanidmRestoreStatus::default();
3127        assert!(status.phase_timestamps.is_empty());
3128    }
3129
3130    #[test]
3131    fn status_default_has_empty_original_replicas() {
3132        let status = KanidmRestoreStatus::default();
3133        assert!(status.original_replicas.is_empty());
3134    }
3135
3136    #[test]
3137    fn hardened_security_context_drops_all_capabilities() {
3138        let ctx = hardened_security_context();
3139        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3140        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3141        assert_eq!(ctx.run_as_non_root, Some(true));
3142        let caps = ctx.capabilities.unwrap();
3143        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3144        assert_eq!(
3145            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3146            Some("RuntimeDefault")
3147        );
3148    }
3149
3150    #[test]
3151    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3152        let ctx = kanidm_job_security_context();
3153        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3154        assert!(ctx.run_as_non_root.is_none());
3155        assert!(ctx.read_only_root_filesystem.is_none());
3156        let caps = ctx.capabilities.unwrap();
3157        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3158        assert_eq!(
3159            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3160            Some("RuntimeDefault")
3161        );
3162    }
3163
3164    #[test]
3165    fn safety_backup_job_name_is_deterministic() {
3166        let source = KanidmRestoreSource {
3167            local: Some(super::KanidmRestoreLocalSource {
3168                file_name: "backup.json".to_string(),
3169            }),
3170            backup_ref: None,
3171        };
3172        let restore = make_restore(source, None);
3173        let name1 = super::safety_backup_job_name(&restore);
3174        let name2 = super::safety_backup_job_name(&restore);
3175        assert_eq!(name1, name2);
3176        assert_eq!(name1, "test-restore-safety-backup");
3177    }
3178
3179    #[test]
3180    fn source_prep_job_name_is_deterministic() {
3181        let source = KanidmRestoreSource {
3182            local: None,
3183            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3184                name: "backup-1".to_string(),
3185            }),
3186        };
3187        let restore = make_restore(source, None);
3188        let name = super::source_prep_job_name(&restore);
3189        assert_eq!(name, "test-restore-source-prep");
3190    }
3191
3192    #[test]
3193    fn break_glass_condition_is_recorded_in_status() {
3194        let mut status = KanidmRestoreStatus::default();
3195        let condition = super::restore_condition(
3196            &status.conditions,
3197            CONDITION_BREAK_GLASS,
3198            CONDITION_TRUE,
3199            "BreakGlassOverride",
3200            "Safety backup skipped",
3201            Some(1),
3202        );
3203        status.conditions.push(condition);
3204        assert!(status.conditions.iter().any(|c| {
3205            c.type_ == CONDITION_BREAK_GLASS
3206                && c.status == CONDITION_TRUE
3207                && c.reason == "BreakGlassOverride"
3208        }));
3209    }
3210
3211    #[test]
3212    fn replica_count_entry_serialization() {
3213        let entry = ReplicaCountEntry {
3214            group: "primary".to_string(),
3215            replicas: 3,
3216        };
3217        let json = serde_json::to_string(&entry).unwrap();
3218        assert!(json.contains("\"group\":\"primary\""));
3219        assert!(json.contains("\"replicas\":3"));
3220    }
3221
3222    #[test]
3223    fn local_source_does_not_require_repository_ref() {
3224        let source = KanidmRestoreSource {
3225            local: Some(super::KanidmRestoreLocalSource {
3226                file_name: "backup.json".to_string(),
3227            }),
3228            backup_ref: None,
3229        };
3230        let safety = SafetyBackupConfig {
3231            repository_ref: None,
3232            skip: false,
3233        };
3234        let restore = make_restore(source, Some(safety));
3235        assert!(validate_safety_backup_config(&restore).is_ok());
3236    }
3237
3238    #[test]
3239    fn skip_without_break_glass_annotations_is_rejected() {
3240        let source = KanidmRestoreSource {
3241            local: Some(super::KanidmRestoreLocalSource {
3242                file_name: "backup.json".to_string(),
3243            }),
3244            backup_ref: None,
3245        };
3246        let safety = SafetyBackupConfig {
3247            repository_ref: None,
3248            skip: true,
3249        };
3250        let restore = make_restore(source, Some(safety));
3251        assert!(validate_safety_backup_config(&restore).is_err());
3252    }
3253
3254    #[test]
3255    fn has_accepted_condition_true() {
3256        let conditions = vec![Condition {
3257            type_: "Ready".to_string(),
3258            status: "True".to_string(),
3259            reason: "Accepted".to_string(),
3260            message: String::new(),
3261            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3262                k8s_openapi::jiff::Timestamp::now(),
3263            ),
3264            observed_generation: None,
3265        }];
3266        assert!(has_accepted_condition(&conditions));
3267    }
3268
3269    #[test]
3270    fn has_accepted_condition_false_when_not_accepted_reason() {
3271        let conditions = vec![Condition {
3272            type_: "Ready".to_string(),
3273            status: "True".to_string(),
3274            reason: "SomeOtherReason".to_string(),
3275            message: String::new(),
3276            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3277                k8s_openapi::jiff::Timestamp::now(),
3278            ),
3279            observed_generation: None,
3280        }];
3281        assert!(!has_accepted_condition(&conditions));
3282    }
3283
3284    #[test]
3285    fn has_accepted_condition_false_when_status_false() {
3286        let conditions = vec![Condition {
3287            type_: "Ready".to_string(),
3288            status: "False".to_string(),
3289            reason: "Accepted".to_string(),
3290            message: String::new(),
3291            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3292                k8s_openapi::jiff::Timestamp::now(),
3293            ),
3294            observed_generation: None,
3295        }];
3296        assert!(!has_accepted_condition(&conditions));
3297    }
3298
3299    #[test]
3300    fn has_accepted_condition_empty() {
3301        assert!(!has_accepted_condition(&[]));
3302    }
3303
3304    #[test]
3305    fn safety_backup_upload_operation_doc_is_valid() {
3306        let source = KanidmRestoreSource {
3307            local: None,
3308            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3309                name: "backup-1".to_string(),
3310            }),
3311        };
3312        let mut restore = make_restore(source, None);
3313        restore.metadata.namespace = Some("test-ns".to_string());
3314        let target = super::super::crd::Kanidm::default();
3315        let doc_str = super::build_safety_upload_operation_doc(
3316            &restore,
3317            &target,
3318            "test-backup-id",
3319            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3320            "my-bucket",
3321            "prod",
3322            "https://s3.example.com",
3323            "us-east-1",
3324            false,
3325            false,
3326            None,
3327        )
3328        .unwrap();
3329        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3330        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3331        assert_eq!(parsed["kind"], "OperationDocument");
3332        assert_eq!(parsed["operation"], "upload");
3333        assert_eq!(parsed["bucket"], "my-bucket");
3334        assert_eq!(parsed["prefix"], "prod");
3335        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3336        assert_eq!(parsed["region"], "us-east-1");
3337        assert_eq!(parsed["backupId"], "test-backup-id");
3338        assert_eq!(parsed["consistency"], "kanidm-offline");
3339        assert_eq!(parsed["reason"], "restore-safety");
3340        assert_eq!(
3341            parsed["payloadPath"],
3342            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3343        );
3344        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3345    }
3346
3347    #[test]
3348    fn download_operation_doc_uses_real_repository_data() {
3349        let source = KanidmRestoreSource {
3350            local: None,
3351            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3352                name: "backup-1".to_string(),
3353            }),
3354        };
3355        let restore = make_restore(source, None);
3356        let target = super::super::crd::Kanidm::default();
3357        let doc_str = super::build_download_operation_doc(
3358            &restore,
3359            &target,
3360            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3361            "019c7c76-f423-7a12-8f41-2bea7588a303",
3362            "real-bucket",
3363            "real-prefix",
3364            "https://real-endpoint.com",
3365            "eu-west-1",
3366            true,
3367            false,
3368            None,
3369        );
3370        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3371        assert_eq!(parsed["bucket"], "real-bucket");
3372        assert_eq!(parsed["prefix"], "real-prefix");
3373        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3374        assert_eq!(parsed["region"], "eu-west-1");
3375        assert_eq!(parsed["forcePathStyle"], true);
3376        assert_eq!(
3377            parsed["manifestKey"],
3378            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3379        );
3380        assert_eq!(
3381            parsed["expectedBackupId"],
3382            "019c7c76-f423-7a12-8f41-2bea7588a303"
3383        );
3384    }
3385
3386    #[test]
3387    fn safety_backup_ref_is_set_after_verification() {
3388        let status = KanidmRestoreStatus {
3389            phase: KanidmRestorePhase::SafetyBackup,
3390            safety_backup_job_name: Some("test-safety-backup".to_string()),
3391            safety_backup_ref: Some("safety-test-restore".to_string()),
3392            safety_backup_expected_backup_id: Some(
3393                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3394            ),
3395            safety_backup_manifest_key: Some(
3396                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3397            ),
3398            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3399            ..Default::default()
3400        };
3401        assert!(status.safety_backup_ref.is_some());
3402        assert!(
3403            status
3404                .safety_backup_ref
3405                .as_ref()
3406                .unwrap()
3407                .starts_with("safety-")
3408        );
3409        assert!(status.safety_backup_expected_backup_id.is_some());
3410        assert!(status.safety_backup_manifest_key.is_some());
3411        assert!(status.safety_backup_payload_sha256.is_some());
3412    }
3413
3414    #[test]
3415    fn database_mutation_not_started_without_safety_backup_ref() {
3416        let status = KanidmRestoreStatus {
3417            phase: KanidmRestorePhase::PreparingSource,
3418            safety_backup_ref: None,
3419            database_mutation_started: false,
3420            ..Default::default()
3421        };
3422        assert!(!status.database_mutation_started);
3423        assert!(status.safety_backup_ref.is_none());
3424    }
3425
3426    #[test]
3427    fn pre_boundary_failure_does_not_set_mutation_started() {
3428        let status = KanidmRestoreStatus {
3429            phase: KanidmRestorePhase::Failed,
3430            message: Some(
3431                "safety backup job failed; target restored to original state".to_string(),
3432            ),
3433            database_mutation_started: false,
3434            ..Default::default()
3435        };
3436        assert!(!status.database_mutation_started);
3437    }
3438
3439    #[test]
3440    fn shared_volume_constants_are_distinct() {
3441        assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3442        assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3443    }
3444
3445    #[test]
3446    fn validate_backup_compatibility_matching_versions_passes() {
3447        let source = KanidmRestoreSource {
3448            local: None,
3449            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3450                name: "backup-1".to_string(),
3451            }),
3452        };
3453        let restore = make_restore(source, None);
3454        let target = kanidm_with_version("1.10.4");
3455        let backup = kaniop_backup_core::crd::KanidmBackup {
3456            metadata: kube::api::ObjectMeta::default(),
3457            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3458                backup_id: "id".to_string(),
3459                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3460                    name: "k".to_string(),
3461                    uid: "uid".to_string(),
3462                },
3463                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3464                    name: "repo".to_string(),
3465                },
3466                manifest_key: "key".to_string(),
3467            },
3468            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3469                kanidm_version: Some("1.10.4".to_string()),
3470                ..Default::default()
3471            }),
3472        };
3473        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3474    }
3475
3476    #[test]
3477    fn validate_backup_compatibility_version_mismatch_fails() {
3478        let source = KanidmRestoreSource {
3479            local: None,
3480            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3481                name: "backup-1".to_string(),
3482            }),
3483        };
3484        let restore = make_restore(source, None);
3485        let target = kanidm_with_version("1.11.0");
3486        let backup = kaniop_backup_core::crd::KanidmBackup {
3487            metadata: kube::api::ObjectMeta::default(),
3488            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3489                backup_id: "id".to_string(),
3490                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3491                    name: "k".to_string(),
3492                    uid: "uid".to_string(),
3493                },
3494                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3495                    name: "repo".to_string(),
3496                },
3497                manifest_key: "key".to_string(),
3498            },
3499            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3500                kanidm_version: Some("1.10.4".to_string()),
3501                ..Default::default()
3502            }),
3503        };
3504        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3505    }
3506
3507    #[test]
3508    fn validate_backup_compatibility_no_backup_version_passes() {
3509        let source = KanidmRestoreSource {
3510            local: None,
3511            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3512                name: "backup-1".to_string(),
3513            }),
3514        };
3515        let restore = make_restore(source, None);
3516        let target = super::super::crd::Kanidm::default();
3517        let backup = kaniop_backup_core::crd::KanidmBackup {
3518            metadata: kube::api::ObjectMeta::default(),
3519            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3520                backup_id: "id".to_string(),
3521                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3522                    name: "k".to_string(),
3523                    uid: "uid".to_string(),
3524                },
3525                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3526                    name: "repo".to_string(),
3527                },
3528                manifest_key: "key".to_string(),
3529            },
3530            status: None,
3531        };
3532        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3533    }
3534
3535    #[test]
3536    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3537        let source = KanidmRestoreSource {
3538            local: None,
3539            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3540                name: "backup-1".to_string(),
3541            }),
3542        };
3543        let mut restore = make_restore(source, None);
3544        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3545        let target = super::super::crd::Kanidm::default();
3546        let backup = kaniop_backup_core::crd::KanidmBackup {
3547            metadata: kube::api::ObjectMeta::default(),
3548            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3549                backup_id: "id".to_string(),
3550                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3551                    name: "k".to_string(),
3552                    uid: "uid".to_string(),
3553                },
3554                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3555                    name: "repo".to_string(),
3556                },
3557                manifest_key: "key".to_string(),
3558            },
3559            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3560                image_digest: Some("sha256:originaldigest".to_string()),
3561                ..Default::default()
3562            }),
3563        };
3564        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3565    }
3566
3567    #[test]
3568    fn validate_backup_compatibility_image_digest_match_passes() {
3569        let source = KanidmRestoreSource {
3570            local: None,
3571            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3572                name: "backup-1".to_string(),
3573            }),
3574        };
3575        let mut restore = make_restore(source, None);
3576        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3577        let target = super::super::crd::Kanidm::default();
3578        let backup = kaniop_backup_core::crd::KanidmBackup {
3579            metadata: kube::api::ObjectMeta::default(),
3580            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3581                backup_id: "id".to_string(),
3582                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3583                    name: "k".to_string(),
3584                    uid: "uid".to_string(),
3585                },
3586                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3587                    name: "repo".to_string(),
3588                },
3589                manifest_key: "key".to_string(),
3590            },
3591            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3592                image_digest: Some("sha256:abc123".to_string()),
3593                ..Default::default()
3594            }),
3595        };
3596        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3597    }
3598
3599    #[test]
3600    fn safety_backup_id_is_deterministic() {
3601        let source = KanidmRestoreSource {
3602            local: None,
3603            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3604                name: "backup-1".to_string(),
3605            }),
3606        };
3607        let restore = make_restore(source, None);
3608        let id1 = super::compute_safety_backup_id(&restore);
3609        let id2 = super::compute_safety_backup_id(&restore);
3610        assert_eq!(id1, id2);
3611        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3612    }
3613
3614    #[test]
3615    fn safety_backup_id_differs_for_different_restores() {
3616        let source1 = KanidmRestoreSource {
3617            local: None,
3618            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3619                name: "backup-1".to_string(),
3620            }),
3621        };
3622        let restore1 = make_restore(source1, None);
3623        let mut restore2 = make_restore(
3624            KanidmRestoreSource {
3625                local: None,
3626                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3627                    name: "backup-2".to_string(),
3628                }),
3629            },
3630            None,
3631        );
3632        restore2.metadata.name = Some("different-restore".to_string());
3633        let id1 = super::compute_safety_backup_id(&restore1);
3634        let id2 = super::compute_safety_backup_id(&restore2);
3635        assert_ne!(id1, id2);
3636    }
3637
3638    #[test]
3639    fn safety_manifest_key_is_deterministic() {
3640        let source = KanidmRestoreSource {
3641            local: None,
3642            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3643                name: "backup-1".to_string(),
3644            }),
3645        };
3646        let restore = make_restore(source, None);
3647        let key1 = super::compute_safety_manifest_key(&restore);
3648        let key2 = super::compute_safety_manifest_key(&restore);
3649        assert_eq!(key1, key2);
3650        assert!(key1.contains("manifest.json"));
3651        assert!(key1.starts_with("v1/tenants/"));
3652    }
3653
3654    #[test]
3655    fn safety_backup_id_survives_restart() {
3656        let source = KanidmRestoreSource {
3657            local: None,
3658            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3659                name: "backup-1".to_string(),
3660            }),
3661        };
3662        let mut restore = make_restore(source, None);
3663        let original_id = super::compute_safety_backup_id(&restore);
3664        restore.status = Some(KanidmRestoreStatus {
3665            phase: KanidmRestorePhase::SafetyBackup,
3666            safety_backup_expected_backup_id: Some(original_id.clone()),
3667            ..Default::default()
3668        });
3669        let recomputed_id = super::compute_safety_backup_id(&restore);
3670        assert_eq!(original_id, recomputed_id);
3671    }
3672
3673    #[test]
3674    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3675        let source = KanidmRestoreSource {
3676            local: None,
3677            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3678                name: "backup-1".to_string(),
3679            }),
3680        };
3681        let restore = make_restore(source, None);
3682        assert!(super::requires_safety_backup(&restore));
3683        let status_without_ref = KanidmRestoreStatus {
3684            phase: KanidmRestorePhase::RestoringPrimary,
3685            safety_backup_ref: None,
3686            database_mutation_started: false,
3687            ..Default::default()
3688        };
3689        assert!(status_without_ref.safety_backup_ref.is_none());
3690        assert!(!status_without_ref.database_mutation_started);
3691    }
3692
3693    #[test]
3694    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3695        let status_with_ref = KanidmRestoreStatus {
3696            phase: KanidmRestorePhase::RestoringPrimary,
3697            safety_backup_ref: Some("safety-test-restore".to_string()),
3698            safety_backup_manifest_key: Some(
3699                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3700            ),
3701            safety_backup_payload_sha256: Some("abc123".to_string()),
3702            database_mutation_started: false,
3703            ..Default::default()
3704        };
3705        assert!(status_with_ref.safety_backup_ref.is_some());
3706        assert!(!status_with_ref.database_mutation_started);
3707    }
3708
3709    #[test]
3710    fn result_document_verification_rejects_missing_backup_id() {
3711        let result_json = r#"{
3712            "apiVersion": "backup.kaniop.rs/v1alpha1",
3713            "kind": "ResultDocument",
3714            "operation": "upload",
3715            "success": true,
3716            "exitCode": "success",
3717            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3718            "payloadSha256": "abc123"
3719        }"#;
3720        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3721        assert!(result.backup_id.is_none());
3722    }
3723
3724    #[test]
3725    fn result_document_verification_rejects_missing_manifest_key() {
3726        let result_json = r#"{
3727            "apiVersion": "backup.kaniop.rs/v1alpha1",
3728            "kind": "ResultDocument",
3729            "operation": "upload",
3730            "success": true,
3731            "exitCode": "success",
3732            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3733            "payloadSha256": "abc123"
3734        }"#;
3735        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3736        assert!(result.manifest_key.is_none());
3737    }
3738
3739    #[test]
3740    fn result_document_verification_rejects_missing_payload_sha256() {
3741        let result_json = r#"{
3742            "apiVersion": "backup.kaniop.rs/v1alpha1",
3743            "kind": "ResultDocument",
3744            "operation": "upload",
3745            "success": true,
3746            "exitCode": "success",
3747            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3748            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3749        }"#;
3750        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3751        assert!(result.payload_sha256.is_none());
3752    }
3753
3754    #[test]
3755    fn result_document_verification_rejects_failure_exit_code() {
3756        let result_json = r#"{
3757            "apiVersion": "backup.kaniop.rs/v1alpha1",
3758            "kind": "ResultDocument",
3759            "operation": "upload",
3760            "success": false,
3761            "exitCode": "integrity",
3762            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3763            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3764            "payloadSha256": "abc123"
3765        }"#;
3766        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3767        assert!(!result.success);
3768        assert_eq!(
3769            result.exit_code,
3770            kaniop_backup_core::result::ExitCode::Integrity
3771        );
3772    }
3773
3774    #[test]
3775    fn result_document_verification_rejects_wrong_operation() {
3776        let result_json = r#"{
3777            "apiVersion": "backup.kaniop.rs/v1alpha1",
3778            "kind": "ResultDocument",
3779            "operation": "download",
3780            "success": true,
3781            "exitCode": "success",
3782            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3783            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3784            "payloadSha256": "abc123"
3785        }"#;
3786        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3787        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3788    }
3789
3790    #[test]
3791    fn result_document_verification_accepts_valid_document() {
3792        let result_json = r#"{
3793            "apiVersion": "backup.kaniop.rs/v1alpha1",
3794            "kind": "ResultDocument",
3795            "operation": "upload",
3796            "success": true,
3797            "exitCode": "success",
3798            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3799            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3800            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3801        }"#;
3802        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3803        assert!(result.success);
3804        assert_eq!(
3805            result.exit_code,
3806            kaniop_backup_core::result::ExitCode::Success
3807        );
3808        assert_eq!(result.operation, "upload");
3809        assert!(result.backup_id.is_some());
3810        assert!(result.manifest_key.is_some());
3811        assert!(result.payload_sha256.is_some());
3812    }
3813
3814    #[test]
3815    fn termination_message_path_constant_is_valid() {
3816        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3817        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3818    }
3819
3820    #[test]
3821    fn status_tracks_safety_backup_evidence() {
3822        let status = KanidmRestoreStatus {
3823            phase: KanidmRestorePhase::PreparingSource,
3824            safety_backup_ref: Some("safety-test-restore".to_string()),
3825            safety_backup_expected_backup_id: Some(
3826                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3827            ),
3828            safety_backup_manifest_key: Some(
3829                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3830            ),
3831            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3832            ..Default::default()
3833        };
3834        assert!(status.safety_backup_ref.is_some());
3835        assert!(status.safety_backup_expected_backup_id.is_some());
3836        assert!(status.safety_backup_manifest_key.is_some());
3837        assert!(status.safety_backup_payload_sha256.is_some());
3838    }
3839
3840    #[test]
3841    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3842        let invalid_json = "not valid json";
3843        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3844        assert!(result.is_err());
3845    }
3846
3847    #[test]
3848    fn fail_closed_semantics_preserved_on_oversized_result() {
3849        let oversized_json = format!(
3850            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3851            kaniop_backup_core::result::RESULT_DOC_VERSION,
3852            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3853        );
3854        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3855        assert!(result.is_err());
3856    }
3857
3858    #[test]
3859    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3860        let source = KanidmRestoreSource {
3861            local: None,
3862            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3863                name: "backup-1".to_string(),
3864            }),
3865        };
3866        let mut restore = make_restore(source, None);
3867        restore.metadata.namespace = Some("test-ns".to_string());
3868        let target = super::super::crd::Kanidm::default();
3869        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3870        let doc_str = super::build_safety_upload_operation_doc(
3871            &restore,
3872            &target,
3873            "test-backup-id",
3874            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3875            "my-bucket",
3876            "prod",
3877            "https://s3.example.com",
3878            "us-east-1",
3879            false,
3880            false,
3881            Some(&ca_path),
3882        )
3883        .unwrap();
3884        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3885        assert_eq!(parsed["caBundlePath"], ca_path);
3886    }
3887
3888    #[test]
3889    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3890        let source = KanidmRestoreSource {
3891            local: None,
3892            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3893                name: "backup-1".to_string(),
3894            }),
3895        };
3896        let mut restore = make_restore(source, None);
3897        restore.metadata.namespace = Some("test-ns".to_string());
3898        let target = super::super::crd::Kanidm::default();
3899        let doc_str = super::build_safety_upload_operation_doc(
3900            &restore,
3901            &target,
3902            "test-backup-id",
3903            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3904            "my-bucket",
3905            "prod",
3906            "https://s3.example.com",
3907            "us-east-1",
3908            false,
3909            false,
3910            None,
3911        )
3912        .unwrap();
3913        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3914        assert!(parsed["caBundlePath"].is_null());
3915    }
3916
3917    #[test]
3918    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3919        let source = KanidmRestoreSource {
3920            local: None,
3921            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3922                name: "backup-1".to_string(),
3923            }),
3924        };
3925        let restore = make_restore(source, None);
3926        let target = super::super::crd::Kanidm::default();
3927        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3928        let doc_str = super::build_download_operation_doc(
3929            &restore,
3930            &target,
3931            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3932            "019c7c76-f423-7a12-8f41-2bea7588a303",
3933            "real-bucket",
3934            "real-prefix",
3935            "https://real-endpoint.com",
3936            "eu-west-1",
3937            true,
3938            false,
3939            Some(&ca_path),
3940        );
3941        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3942        assert_eq!(parsed["caBundlePath"], ca_path);
3943    }
3944
3945    #[test]
3946    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3947        let source = KanidmRestoreSource {
3948            local: None,
3949            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3950                name: "backup-1".to_string(),
3951            }),
3952        };
3953        let restore = make_restore(source, None);
3954        let target = super::super::crd::Kanidm::default();
3955        let doc_str = super::build_download_operation_doc(
3956            &restore,
3957            &target,
3958            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3959            "019c7c76-f423-7a12-8f41-2bea7588a303",
3960            "real-bucket",
3961            "real-prefix",
3962            "https://real-endpoint.com",
3963            "eu-west-1",
3964            true,
3965            false,
3966            None,
3967        );
3968        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3969        assert!(parsed["caBundlePath"].is_null());
3970    }
3971
3972    #[test]
3973    fn source_prep_result_operation_is_download() {
3974        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3975        assert_ne!(
3976            super::SOURCE_PREP_RESULT_OPERATION,
3977            super::SAFETY_BACKUP_RESULT_OPERATION
3978        );
3979    }
3980
3981    #[test]
3982    fn download_result_document_verification_accepts_valid_document() {
3983        let result_json = r#"{
3984            "apiVersion": "backup.kaniop.rs/v1alpha1",
3985            "kind": "ResultDocument",
3986            "operation": "download",
3987            "success": true,
3988            "exitCode": "success",
3989            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3990            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3991            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3992        }"#;
3993        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3994        assert!(result.success);
3995        assert_eq!(
3996            result.exit_code,
3997            kaniop_backup_core::result::ExitCode::Success
3998        );
3999        assert_eq!(result.operation, "download");
4000        assert_eq!(
4001            result.backup_id.as_deref(),
4002            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4003        );
4004        assert!(result.manifest_key.is_some());
4005        assert!(result.payload_sha256.is_some());
4006    }
4007
4008    #[test]
4009    fn download_result_document_rejects_wrong_operation() {
4010        let result_json = r#"{
4011            "apiVersion": "backup.kaniop.rs/v1alpha1",
4012            "kind": "ResultDocument",
4013            "operation": "upload",
4014            "success": true,
4015            "exitCode": "success",
4016            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018            "payloadSha256": "abc123"
4019        }"#;
4020        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4022    }
4023
4024    #[test]
4025    fn download_result_document_rejects_failure_success_field() {
4026        let result_json = r#"{
4027            "apiVersion": "backup.kaniop.rs/v1alpha1",
4028            "kind": "ResultDocument",
4029            "operation": "download",
4030            "success": false,
4031            "exitCode": "retryable",
4032            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4033            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4034            "payloadSha256": "abc123"
4035        }"#;
4036        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4037        assert!(!result.success);
4038        assert_ne!(
4039            result.exit_code,
4040            kaniop_backup_core::result::ExitCode::Success
4041        );
4042    }
4043
4044    #[test]
4045    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4046        let status = KanidmRestoreStatus {
4047            phase: KanidmRestorePhase::PreparingSource,
4048            database_mutation_started: false,
4049            ..Default::default()
4050        };
4051        assert!(!status.database_mutation_started);
4052        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4053    }
4054}