Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78    feature = "schemars",
79    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82    category = "kaniop",
83    group = "kaniop.rs",
84    version = "v1beta1",
85    kind = "KanidmRestore",
86    plural = "kanidmrestores",
87    singular = "kanidmrestore",
88    shortname = "idmrestore",
89    namespaced,
90    status = "KanidmRestoreStatus",
91    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97    pub target_ref: KanidmRestoreTargetRef,
98    pub source: KanidmRestoreSource,
99    pub restore_image: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108    pub name: String,
109    pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115    feature = "schemars",
116    schemars(extend("x-kubernetes-validations" = [
117        {
118            "message": "exactly one of local or backupRef must be set",
119            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120        }
121    ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125    /// Local file source. Mutually exclusive with `backupRef`.
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub local: Option<KanidmRestoreLocalSource>,
128    /// Remote cataloged backup reference. Mutually exclusive with `local`.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137    /// Basename of a backup below /data. Paths and traversal are rejected.
138    pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
146    pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub repository_ref: Option<SafetyBackupRepositoryRef>,
156    /// Skip the mandatory safety backup. Requires break-glass annotations.
157    #[serde(default)]
158    pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165    pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171    #[default]
172    Pending,
173    Validating,
174    Quiescing,
175    SafetyBackup,
176    PreparingSource,
177    RestoringPrimary,
178    Verifying,
179    RebuildingReplicas,
180    Resuming,
181    Completed,
182    Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_generation: Option<i64>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_target_uid: Option<String>,
193    #[serde(default)]
194    pub phase: KanidmRestorePhase,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub restore_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub verify_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub safety_backup_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub source_prep_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_ref: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_expected_backup_id: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_manifest_key: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_payload_sha256: Option<String>,
211    #[serde(default)]
212    pub replicas_cleared: bool,
213    #[serde(default)]
214    pub certificates_cleared: bool,
215    #[serde(default)]
216    pub database_mutation_started: bool,
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub original_replicas: Vec<ReplicaCountEntry>,
219    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220    pub phase_timestamps: std::collections::BTreeMap<String, String>,
221    #[serde(default, skip_serializing_if = "Vec::is_empty")]
222    #[cfg_attr(
223        feature = "schemars",
224        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225    )]
226    pub conditions: Vec<Condition>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235    pub group: String,
236    pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241    attempts: Counter<u64>,
242    outcomes: Counter<u64>,
243    duration_seconds: Histogram<f64>,
244    start_timestamp_seconds: Gauge<i64>,
245    break_glass_total: Counter<u64>,
246    disaster_recovery_total: Counter<u64>,
247    safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251    fn new() -> Self {
252        let meter = global::meter("kaniop");
253        Self {
254            attempts: meter
255                .u64_counter("restore_attempts")
256                .with_description("Number of Kanidm restore attempts started")
257                .build(),
258            outcomes: meter
259                .u64_counter("restore_outcomes")
260                .with_description("Number of terminal Kanidm restore outcomes")
261                .build(),
262            duration_seconds: meter
263                .f64_histogram("restore_duration_seconds")
264                .with_description("Kanidm restore duration from object creation to terminal phase")
265                .with_unit("s")
266                .build(),
267            start_timestamp_seconds: meter
268                .i64_gauge("restore_start_timestamp_seconds")
269                .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270                .with_unit("s")
271                .build(),
272            break_glass_total: meter
273                .u64_counter("restore_break_glass")
274                .with_description("Number of break-glass safety backup overrides")
275                .build(),
276            disaster_recovery_total: meter
277                .u64_counter("restore_disaster_recovery")
278                .with_description("Number of approved disaster-recovery source identity overrides")
279                .build(),
280            safety_backup_duration_seconds: meter
281                .f64_histogram("restore_safety_backup_duration_seconds")
282                .with_description("Duration of pre-restore safety backup creation")
283                .with_unit("s")
284                .build(),
285        }
286    }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290    let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291    let active = restore.metadata.deletion_timestamp.is_none()
292        && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293    let timestamp = if active {
294        restore
295            .metadata
296            .creation_timestamp
297            .as_ref()
298            .map(|created| created.0.as_second())
299            .unwrap_or_else(|| Timestamp::now().as_second())
300    } else {
301        0
302    };
303    let attributes = [
304        KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305        KeyValue::new("restore", restore.name_any()),
306    ];
307    metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312    client: Client,
313    recorder: Recorder,
314    metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319    match phase {
320        KanidmRestorePhase::Pending => {
321            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322            Ok(Action::requeue(REQUEUE))
323        }
324        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325            Ok((target, disaster_recovery_source)) => {
326                mark_restoring(&restore, &target, &ctx).await?;
327                if let Some(source) = disaster_recovery_source.as_ref() {
328                    record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329                }
330                let mut status = restore.status.clone().unwrap_or_default();
331                if let Some(source) = disaster_recovery_source.as_ref() {
332                    let target_uid = target.uid().unwrap_or_default();
333                    let condition = restore_condition(
334                        &status.conditions,
335                        CONDITION_DISASTER_RECOVERY,
336                        CONDITION_TRUE,
337                        "ApprovedSourceIdentityMismatch",
338                        &format!(
339                            "Approved disaster recovery from source {}/{} into target {}/{}",
340                            source.name,
341                            source.uid,
342                            target.name_any(),
343                            target_uid
344                        ),
345                        restore.metadata.generation,
346                    );
347                    status
348                        .conditions
349                        .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350                    status.conditions.push(condition);
351                }
352                status.observed_target_uid = target.uid();
353                status.original_replicas = capture_original_replicas(&target);
354                status.phase = KanidmRestorePhase::Quiescing;
355                status.message = None;
356                patch_status(&restore, &ctx, status).await?;
357                Ok(Action::requeue(REQUEUE))
358            }
359            Err(error) => {
360                set_phase(
361                    &restore,
362                    &ctx,
363                    KanidmRestorePhase::Failed,
364                    Some(error.to_string()),
365                )
366                .await?;
367                Ok(Action::requeue(Duration::from_secs(300)))
368            }
369        },
370        KanidmRestorePhase::Quiescing => {
371            let target = get_target(&restore, &ctx).await?;
372            scale_all(&target, &ctx, 0).await?;
373            if target_pods_stopped(&target, &ctx).await?
374                && target_volumes_detached(&target, &ctx).await?
375            {
376                let next_phase = if requires_safety_backup(&restore) {
377                    KanidmRestorePhase::SafetyBackup
378                } else {
379                    record_break_glass(&restore, &ctx).await?;
380                    KanidmRestorePhase::PreparingSource
381                };
382                if next_phase == KanidmRestorePhase::PreparingSource {
383                    let refreshed = Api::<KanidmRestore>::namespaced(
384                        ctx.client.clone(),
385                        restore.namespace().as_deref().unwrap_or_default(),
386                    )
387                    .get(&restore.name_any())
388                    .await
389                    .map_err(|error| {
390                        Error::kube_error(
391                            "get",
392                            "KanidmRestore",
393                            restore.namespace().as_deref().unwrap_or_default(),
394                            restore.name_any(),
395                            error,
396                        )
397                    })?;
398                    set_phase(&refreshed, &ctx, next_phase, None).await?;
399                } else {
400                    set_phase(&restore, &ctx, next_phase, None).await?;
401                }
402            }
403            Ok(Action::requeue(REQUEUE))
404        }
405        KanidmRestorePhase::SafetyBackup => {
406            let target = get_target(&restore, &ctx).await?;
407            let name = safety_backup_job_name(&restore);
408            let expected_backup_id = compute_safety_backup_id(&restore)?;
409            let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410            ensure_safety_backup_job(
411                &restore,
412                &target,
413                &ctx,
414                &name,
415                &expected_backup_id,
416                &expected_manifest_key,
417            )
418            .await?;
419            let mut status = restore.status.clone().unwrap_or_default();
420            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421            match job_state(&restore, &ctx, &name).await? {
422                JobState::Complete => {
423                    let result =
424                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425                    match result {
426                        Ok(verified) => {
427                            status.safety_backup_job_name = Some(name);
428                            status.safety_backup_ref =
429                                Some(format!("safety-{}", restore.name_any()));
430                            status.safety_backup_manifest_key = Some(verified.manifest_key);
431                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432                            record_safety_backup_duration(&restore, &ctx).await;
433                            status.phase = KanidmRestorePhase::PreparingSource;
434                            status.message = None;
435                            patch_status(&restore, &ctx, status).await?;
436                        }
437                        Err(error) => {
438                            resume_before_mutation(&restore, &ctx).await?;
439                            set_phase(
440                                &restore,
441                                &ctx,
442                                KanidmRestorePhase::Failed,
443                                Some(format!("safety backup result verification failed: {error}")),
444                            )
445                            .await?;
446                        }
447                    }
448                }
449                JobState::Failed => {
450                    resume_before_mutation(&restore, &ctx).await?;
451                    set_phase(
452                        &restore,
453                        &ctx,
454                        KanidmRestorePhase::Failed,
455                        Some(
456                            "safety backup job failed; target restored to original state"
457                                .to_string(),
458                        ),
459                    )
460                    .await?;
461                }
462                JobState::Running => {
463                    patch_status(&restore, &ctx, status).await?;
464                }
465            }
466            Ok(Action::requeue(REQUEUE))
467        }
468        KanidmRestorePhase::PreparingSource => {
469            if is_remote_source(&restore) {
470                let target = get_target(&restore, &ctx).await?;
471                let name = source_prep_job_name(&restore);
472                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473                match job_state(&restore, &ctx, &name).await? {
474                    JobState::Complete => {
475                        let backup_id = restore
476                            .spec
477                            .source
478                            .backup_ref
479                            .as_ref()
480                            .map(|b| b.name.clone())
481                            .unwrap_or_default();
482                        let backup = {
483                            let ns = restore.namespace().unwrap();
484                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485                                .get(&backup_id)
486                                .await
487                                .map_err(|e| {
488                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489                                })?
490                        };
491                        let expected_backup_id = &backup.spec.backup_id;
492                        let result =
493                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494                                .await;
495                        match result {
496                            Ok(verified) => {
497                                if let Some(expected_sha256) = backup
498                                    .status
499                                    .as_ref()
500                                    .and_then(|s| s.payload_sha256.as_deref())
501                                {
502                                    if verified.payload_sha256 != expected_sha256 {
503                                        resume_before_mutation(&restore, &ctx).await?;
504                                        set_phase(
505                                            &restore,
506                                            &ctx,
507                                            KanidmRestorePhase::Failed,
508                                            Some(format!(
509                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510                                                verified.payload_sha256
511                                            )),
512                                        )
513                                        .await?;
514                                        return Ok(Action::requeue(REQUEUE));
515                                    }
516                                }
517                                let mut status = restore.status.clone().unwrap_or_default();
518                                status.source_prep_job_name = Some(name);
519                                status.phase = KanidmRestorePhase::RestoringPrimary;
520                                status.message = None;
521                                patch_status(&restore, &ctx, status).await?;
522                            }
523                            Err(error) => {
524                                resume_before_mutation(&restore, &ctx).await?;
525                                set_phase(
526                                    &restore,
527                                    &ctx,
528                                    KanidmRestorePhase::Failed,
529                                    Some(format!(
530                                        "source preparation result verification failed: {error}"
531                                    )),
532                                )
533                                .await?;
534                            }
535                        }
536                    }
537                    JobState::Failed => {
538                        resume_before_mutation(&restore, &ctx).await?;
539                        set_phase(
540                            &restore,
541                            &ctx,
542                            KanidmRestorePhase::Failed,
543                            Some(
544                                "source preparation job failed; target restored to original state"
545                                    .to_string(),
546                            ),
547                        )
548                        .await?;
549                    }
550                    JobState::Running => {}
551                }
552            } else {
553                let target = get_target(&restore, &ctx).await?;
554                let name = source_check_job_name(&restore);
555                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556                match job_state(&restore, &ctx, &name).await? {
557                    JobState::Complete => {
558                        let mut status = restore.status.clone().unwrap_or_default();
559                        status.source_prep_job_name = Some(name);
560                        status.phase = KanidmRestorePhase::RestoringPrimary;
561                        status.message = None;
562                        patch_status(&restore, &ctx, status).await?;
563                    }
564                    JobState::Failed => {
565                        resume_before_mutation(&restore, &ctx).await?;
566                        set_phase(
567                            &restore,
568                            &ctx,
569                            KanidmRestorePhase::Failed,
570                            Some(
571                                "local backup file check failed; target restored to original state"
572                                    .to_string(),
573                            ),
574                        )
575                        .await?;
576                    }
577                    JobState::Running => {}
578                }
579            }
580            Ok(Action::requeue(REQUEUE))
581        }
582        KanidmRestorePhase::RestoringPrimary => {
583            let target = get_target(&restore, &ctx).await?;
584            let status = restore.status.as_ref().cloned().unwrap_or_default();
585            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586                return Err(Error::MissingData(
587                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588                ));
589            }
590            if !status.database_mutation_started {
591                let mut status = status;
592                status.database_mutation_started = true;
593                status.message = Some(
594                    "database mutation boundary persisted; starting primary restore".to_string(),
595                );
596                patch_status(&restore, &ctx, status).await?;
597                return Ok(Action::requeue(REQUEUE));
598            }
599            let name = restore_job_name(&restore);
600            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601            match job_state(&restore, &ctx, &name).await? {
602                JobState::Complete => {
603                    let mut status = restore.status.clone().unwrap_or_default();
604                    status.restore_job_name = Some(name);
605                    status.phase = KanidmRestorePhase::Verifying;
606                    status.message = None;
607                    patch_status(&restore, &ctx, status).await?;
608                }
609                JobState::Failed => {
610                    let status = restore.status.clone().unwrap_or_default();
611                    let failed_status = restore_primary_failed_status(&status, &name);
612                    patch_status(&restore, &ctx, failed_status).await?;
613                }
614                JobState::Running => {}
615            }
616            Ok(Action::requeue(REQUEUE))
617        }
618        KanidmRestorePhase::Verifying => {
619            let target = get_target(&restore, &ctx).await?;
620            let name = verify_job_name(&restore);
621            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622            match job_state(&restore, &ctx, &name).await? {
623                JobState::Complete => {
624                    let mut status = restore.status.clone().unwrap_or_default();
625                    status.verify_job_name = Some(name);
626                    status.phase = KanidmRestorePhase::RebuildingReplicas;
627                    status.message = None;
628                    patch_status(&restore, &ctx, status).await?;
629                }
630                JobState::Failed => {
631                    let status = restore.status.clone().unwrap_or_default();
632                    let failed_status = verifying_failed_status(&status, &name);
633                    patch_status(&restore, &ctx, failed_status).await?;
634                }
635                JobState::Running => {}
636            }
637            Ok(Action::requeue(REQUEUE))
638        }
639        KanidmRestorePhase::RebuildingReplicas => {
640            let target = get_target(&restore, &ctx).await?;
641            let mut status = restore.status.clone().unwrap_or_default();
642            if !status.replicas_cleared {
643                if delete_secondary_pvcs(&target, &ctx).await? {
644                    status.replicas_cleared = true;
645                    status.message = Some("secondary database state cleared".to_string());
646                    patch_status(&restore, &ctx, status).await?;
647                }
648                return Ok(Action::requeue(REQUEUE));
649            }
650
651            scale_primary(&target, &ctx, 1).await?;
652            if !primary_ready(&target, &ctx).await? {
653                return Ok(Action::requeue(REQUEUE));
654            }
655            if !status.certificates_cleared {
656                delete_replica_cert_secrets(&target, &ctx).await?;
657                delete_admin_secret(&target, &ctx).await?;
658                status.certificates_cleared = true;
659                status.message = Some(
660                    "replica certificates and admin secret cleared for regeneration".to_string(),
661                );
662                patch_status(&restore, &ctx, status).await?;
663                return Ok(Action::requeue(REQUEUE));
664            }
665            status.phase = KanidmRestorePhase::Resuming;
666            status.message = None;
667            patch_status(&restore, &ctx, status).await?;
668            Ok(Action::requeue(REQUEUE))
669        }
670        KanidmRestorePhase::Resuming => {
671            let target = get_target(&restore, &ctx).await?;
672            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673            clear_restoring(&restore, &target, &ctx).await?;
674            Ok(Action::requeue(Duration::from_secs(3600)))
675        }
676        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677            Ok(Action::requeue(Duration::from_secs(3600)))
678        }
679    }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684    name: String,
685    uid: String,
686}
687
688async fn validate(
689    restore: &KanidmRestore,
690    ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692    let target = get_target(restore, ctx).await?;
693    let actual_uid = target
694        .uid()
695        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696    if actual_uid != restore.spec.target_ref.uid {
697        return Err(Error::MissingData(format!(
698            "target UID mismatch: expected {}, got {}",
699            restore.spec.target_ref.uid, actual_uid
700        )));
701    }
702    validate_source(restore)?;
703    let disaster_recovery_source = if is_remote_source(restore) {
704        validate_backup_ref(restore, &target, ctx).await?
705    } else {
706        None
707    };
708    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709    {
710        return Err(Error::MissingData(format!(
711            "restoreImage must be the target's pinned Kanidm image (target image is {})",
712            target.spec.image
713        )));
714    }
715    let storage = target
716        .spec
717        .storage
718        .as_ref()
719        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720    if storage.empty_dir.is_some()
721        || storage.ephemeral.is_some()
722        || storage.volume_claim_template.is_none()
723    {
724        return Err(Error::MissingData(
725            "restore requires PVC-backed Kanidm storage".to_string(),
726        ));
727    }
728    let primaries = target
729        .spec
730        .replica_groups
731        .iter()
732        .filter(|rg| rg.primary_node)
733        .count();
734    if primaries != 1 {
735        return Err(Error::MissingData(
736            "backup/restore requires exactly one primary replica group".to_string(),
737        ));
738    }
739    let ns = restore.namespace().unwrap();
740    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741        .list(&ListParams::default())
742        .await
743        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744    if restores.items.iter().any(|other| {
745        other.name_any() != restore.name_any()
746            && other.spec.target_ref.name == restore.spec.target_ref.name
747            && !matches!(
748                other.status.as_ref().map(|s| s.phase),
749                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750            )
751    }) {
752        return Err(Error::MissingData(
753            "another active restore targets this Kanidm".to_string(),
754        ));
755    }
756    validate_safety_backup_config(restore)?;
757    Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761    !name.is_empty()
762        && name != "."
763        && name != ".."
764        && !name.contains('/')
765        && !name.contains('\\')
766        && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770    image == "kanidm/server:latest"
771        || image.ends_with(":latest")
772        || (!image.contains('@')
773            && !image
774                .rsplit('/')
775                .next()
776                .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780    let ns = restore
781        .namespace()
782        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784        .get(&restore.spec.target_ref.name)
785        .await
786        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790    restore: &KanidmRestore,
791    target: &Kanidm,
792    ctx: &RestoreContext,
793) -> Result<()> {
794    let ns = target.namespace().unwrap();
795    let uid = restore
796        .uid()
797        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799        .patch(
800            &target.name_any(),
801            &PatchParams::default(),
802            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803        )
804        .await
805        .map(|_| ())
806        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810    restore: &KanidmRestore,
811    target: &Kanidm,
812    ctx: &RestoreContext,
813) -> Result<()> {
814    let ns = target.namespace().unwrap();
815    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816        return Ok(());
817    }
818    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819        .patch(
820            &target.name_any(),
821            &PatchParams::default(),
822            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823        )
824        .await
825        .map(|_| ())
826        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830    restore: &KanidmRestore,
831    ctx: &RestoreContext,
832    mut status: KanidmRestoreStatus,
833) -> Result<()> {
834    let ns = restore.namespace().unwrap();
835    let previous_phase = restore.status.as_ref().map(|current| current.phase);
836    let next_phase = status.phase;
837    status.observed_generation = restore.metadata.generation;
838    update_restore_conditions(&mut status, restore.metadata.generation);
839
840    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841        .patch_status(
842            &restore.name_any(),
843            &PatchParams::default(),
844            &Patch::Merge(json!({"status": &status})),
845        )
846        .await
847        .map_err(|e| {
848            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849        })?;
850
851    if previous_phase != Some(next_phase) {
852        record_restore_transition(
853            restore,
854            ctx,
855            previous_phase,
856            next_phase,
857            status.message.as_deref(),
858        )
859        .await;
860    }
861    Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865    let previous = status.conditions.clone();
866    let phase = status.phase;
867    let phase_reason = format!("{phase:?}");
868    let message = status
869        .message
870        .clone()
871        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872    let terminal_success = phase == KanidmRestorePhase::Completed;
873    let terminal_failure = phase == KanidmRestorePhase::Failed;
874    let progressing = !terminal_success && !terminal_failure;
875
876    let sticky_conditions: Vec<Condition> = previous
877        .iter()
878        .filter(|condition| {
879            condition.type_ == CONDITION_BREAK_GLASS
880                || condition.type_ == CONDITION_DISASTER_RECOVERY
881        })
882        .cloned()
883        .collect();
884    status.conditions = vec![
885        restore_condition(
886            &previous,
887            CONDITION_PROGRESSING,
888            if progressing {
889                CONDITION_TRUE
890            } else {
891                CONDITION_FALSE
892            },
893            &phase_reason,
894            &message,
895            generation,
896        ),
897        restore_condition(
898            &previous,
899            CONDITION_READY,
900            if terminal_success {
901                CONDITION_TRUE
902            } else {
903                CONDITION_FALSE
904            },
905            if terminal_success {
906                "RestoreCompleted"
907            } else {
908                "RestoreNotCompleted"
909            },
910            if terminal_success {
911                "Kanidm restore completed successfully."
912            } else {
913                "Kanidm restore has not completed successfully."
914            },
915            generation,
916        ),
917        restore_condition(
918            &previous,
919            CONDITION_FAILED,
920            if terminal_failure {
921                CONDITION_TRUE
922            } else {
923                CONDITION_FALSE
924            },
925            if terminal_failure {
926                "RestoreFailed"
927            } else {
928                "NoRestoreFailure"
929            },
930            if terminal_failure {
931                status
932                    .message
933                    .as_deref()
934                    .unwrap_or("Kanidm restore failed.")
935            } else {
936                "No terminal restore failure has been recorded."
937            },
938            generation,
939        ),
940    ];
941    status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945    previous: &[Condition],
946    condition_type: &str,
947    condition_status: &str,
948    reason: &str,
949    message: &str,
950    generation: Option<i64>,
951) -> Condition {
952    let last_transition_time = previous
953        .iter()
954        .find(|condition| {
955            condition.type_ == condition_type
956                && condition.status == condition_status
957                && condition.reason == reason
958        })
959        .map(|condition| condition.last_transition_time.clone())
960        .unwrap_or_else(|| Time(Timestamp::now()));
961    Condition {
962        type_: condition_type.to_string(),
963        status: condition_status.to_string(),
964        reason: reason.to_string(),
965        message: message.to_string(),
966        last_transition_time,
967        observed_generation: generation,
968    }
969}
970
971async fn record_restore_transition(
972    restore: &KanidmRestore,
973    ctx: &RestoreContext,
974    previous_phase: Option<KanidmRestorePhase>,
975    phase: KanidmRestorePhase,
976    message: Option<&str>,
977) {
978    if phase == KanidmRestorePhase::Validating {
979        ctx.metrics.attempts.add(1, &[]);
980    }
981
982    let result = match phase {
983        KanidmRestorePhase::Completed => Some("success"),
984        KanidmRestorePhase::Failed => Some("failure"),
985        _ => None,
986    };
987    if let Some(result) = result {
988        let attributes = [
989            KeyValue::new("result", result),
990            KeyValue::new("phase", format!("{phase:?}")),
991        ];
992        ctx.metrics.outcomes.add(1, &attributes);
993        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995            ctx.metrics.duration_seconds.record(elapsed, &attributes);
996        }
997    }
998
999    let reason = if phase == KanidmRestorePhase::Failed {
1000        "RestoreFailed"
1001    } else {
1002        "RestorePhaseChanged"
1003    };
1004    let note = message.map(str::to_string).or_else(|| {
1005        Some(format!(
1006            "Kanidm restore phase changed from {} to {phase:?}.",
1007            previous_phase
1008                .map(|previous| format!("{previous:?}"))
1009                .unwrap_or_else(|| "None".to_string())
1010        ))
1011    });
1012    if let Err(error) = ctx
1013        .recorder
1014        .publish(
1015            &Event {
1016                type_: if phase == KanidmRestorePhase::Failed {
1017                    EventType::Warning
1018                } else {
1019                    EventType::Normal
1020                },
1021                reason: reason.to_string(),
1022                note,
1023                action: "Restore".to_string(),
1024                secondary: None,
1025            },
1026            &restore.object_ref(&()),
1027        )
1028        .await
1029    {
1030        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031    }
1032}
1033
1034async fn set_phase(
1035    restore: &KanidmRestore,
1036    ctx: &RestoreContext,
1037    phase: KanidmRestorePhase,
1038    message: Option<String>,
1039) -> Result<()> {
1040    let mut status = restore.status.clone().unwrap_or_default();
1041    status.phase = phase;
1042    status.message = message;
1043    patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047    restore: &KanidmRestore,
1048    ctx: &RestoreContext,
1049    message: &str,
1050) -> Result<()> {
1051    set_phase(
1052        restore,
1053        ctx,
1054        KanidmRestorePhase::Failed,
1055        Some(message.to_string()),
1056    )
1057    .await
1058}
1059
1060fn restore_primary_failed_status(
1061    existing: &KanidmRestoreStatus,
1062    job_name: &str,
1063) -> KanidmRestoreStatus {
1064    let mut status = existing.clone();
1065    status.restore_job_name = Some(job_name.to_string());
1066    status.phase = KanidmRestorePhase::Failed;
1067    status.message = Some("database restore job failed".to_string());
1068    status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072    let mut status = existing.clone();
1073    status.verify_job_name = Some(job_name.to_string());
1074    status.phase = KanidmRestorePhase::Failed;
1075    status.message = Some("database verification failed".to_string());
1076    status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080    target
1081        .spec
1082        .replica_groups
1083        .iter()
1084        .find(|rg| rg.primary_node)
1085        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089    let ns = target.namespace().unwrap();
1090    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091    for rg in &target.spec.replica_groups {
1092        let name = target.statefulset_name(&rg.name);
1093        api.patch(
1094            &name,
1095            &PatchParams::default(),
1096            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097        )
1098        .await
1099        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100    }
1101    Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105    let ns = target.namespace().unwrap();
1106    let rg = primary_group(target)?;
1107    let name = target.statefulset_name(&rg.name);
1108    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109        .patch(
1110            &name,
1111            &PatchParams::default(),
1112            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113        )
1114        .await
1115        .map(|_| ())
1116        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120    let ns = target.namespace().unwrap();
1121    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122    for rg in &target.spec.replica_groups {
1123        let name = target.statefulset_name(&rg.name);
1124        api.patch(
1125            &name,
1126            &PatchParams::default(),
1127            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128        )
1129        .await
1130        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131    }
1132    Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136    let ns = target.namespace().unwrap();
1137    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139        .await
1140        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141    Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145    let ns = target.namespace().unwrap();
1146    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147    let mut pv_names = BTreeSet::new();
1148    let primary_name = primary_pvc_name(target)?;
1149    let primary_pvc = pvc_api
1150        .get(&primary_name)
1151        .await
1152        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153    let primary_pv = primary_pvc
1154        .spec
1155        .as_ref()
1156        .and_then(|spec| spec.volume_name.clone())
1157        .ok_or_else(|| {
1158            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159        })?;
1160    pv_names.insert(primary_pv);
1161
1162    for rg in &target.spec.replica_groups {
1163        let sts = target.statefulset_name(&rg.name);
1164        for ordinal in 0..rg.replicas {
1165            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166            if name == primary_name {
1167                continue;
1168            }
1169            if let Some(pvc) = pvc_api
1170                .get_opt(&name)
1171                .await
1172                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174            {
1175                pv_names.insert(pv_name);
1176            }
1177        }
1178    }
1179
1180    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181        .list(&ListParams::default())
1182        .await
1183        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184    Ok(!attachments.items.iter().any(|attachment| {
1185        attachment
1186            .spec
1187            .source
1188            .persistent_volume_name
1189            .as_ref()
1190            .is_some_and(|pv_name| pv_names.contains(pv_name))
1191    }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195    let ns = target.namespace().unwrap();
1196    let name = target.statefulset_name(&primary_group(target)?.name);
1197    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198        .get(&name)
1199        .await
1200        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201    Ok(sts
1202        .status
1203        .as_ref()
1204        .and_then(|s| s.ready_replicas)
1205        .unwrap_or(0)
1206        >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210    let rg = primary_group(target)?;
1211    Ok(format!(
1212        "{DATA_VOLUME}-{}-0",
1213        target.statefulset_name(&rg.name)
1214    ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218    let ns = target.namespace().unwrap();
1219    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220    let mut all_absent = true;
1221    for rg in &target.spec.replica_groups {
1222        let sts = target.statefulset_name(&rg.name);
1223        for ordinal in 0..rg.replicas {
1224            if rg.primary_node && ordinal == 0 {
1225                continue;
1226            }
1227            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228            match api
1229                .get_opt(&name)
1230                .await
1231                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232            {
1233                None => {}
1234                Some(pvc) => {
1235                    all_absent = false;
1236                    if pvc.metadata.deletion_timestamp.is_none() {
1237                        match api.delete(&name, &DeleteParams::default()).await {
1238                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1240                            Err(error) => {
1241                                return Err(Error::kube_error(
1242                                    "delete",
1243                                    "PersistentVolumeClaim",
1244                                    &ns,
1245                                    &name,
1246                                    error,
1247                                ));
1248                            }
1249                        }
1250                    }
1251                }
1252            }
1253        }
1254    }
1255    Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259    let ns = target.namespace().unwrap();
1260    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261    let label_selector = format!(
1262        "{}={},{}={}",
1263        CLUSTER_LABEL,
1264        target.name_any(),
1265        SECRET_TYPE_LABEL,
1266        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267    );
1268    let secrets = api
1269        .list(&ListParams::default().labels(&label_selector))
1270        .await
1271        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272    let mut all_absent = secrets.items.is_empty();
1273    for secret in secrets.items {
1274        let name = secret.name_any();
1275        if secret.metadata.deletion_timestamp.is_none() {
1276            match api.delete(&name, &DeleteParams::default()).await {
1277                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278                Err(kube::Error::Api(status)) if status.code == 404 => {}
1279                Err(error) => {
1280                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281                }
1282            }
1283            all_absent = false;
1284        }
1285    }
1286    Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290    let ns = target.namespace().unwrap();
1291    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292    let name = format!("{}-admin-passwords", target.name_any());
1293    match api
1294        .get_opt(&name)
1295        .await
1296        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297    {
1298        None => Ok(true),
1299        Some(secret) => {
1300            if secret.metadata.deletion_timestamp.is_some() {
1301                return Ok(false);
1302            }
1303            match api.delete(&name, &DeleteParams::default()).await {
1304                Ok(_) => {
1305                    debug!(secret = %name, "deleting stale admin secret");
1306                    Ok(false)
1307                }
1308                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310            }
1311        }
1312    }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316    format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319    format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323    let origin = target
1324        .spec
1325        .origin
1326        .clone()
1327        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328    vec![
1329        EnvVar {
1330            name: "KANIDM_DB_PATH".to_string(),
1331            value: Some("/data/kanidm.db".to_string()),
1332            ..Default::default()
1333        },
1334        EnvVar {
1335            name: "KANIDM_DOMAIN".to_string(),
1336            value: Some(target.spec.domain.clone()),
1337            ..Default::default()
1338        },
1339        EnvVar {
1340            name: "KANIDM_ORIGIN".to_string(),
1341            value: Some(origin),
1342            ..Default::default()
1343        },
1344        EnvVar {
1345            name: "KANIDM_TLS_CHAIN".to_string(),
1346            value: Some(format!("{TLS_PATH}/tls.crt")),
1347            ..Default::default()
1348        },
1349        EnvVar {
1350            name: "KANIDM_TLS_KEY".to_string(),
1351            value: Some(format!("{TLS_PATH}/tls.key")),
1352            ..Default::default()
1353        },
1354    ]
1355}
1356
1357async fn ensure_database_job(
1358    restore: &KanidmRestore,
1359    target: &Kanidm,
1360    ctx: &RestoreContext,
1361    name: &str,
1362    verify: bool,
1363) -> Result<()> {
1364    let ns = restore.namespace().unwrap();
1365    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366    if api
1367        .get_opt(name)
1368        .await
1369        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370        .is_some()
1371    {
1372        return Ok(());
1373    }
1374    let mut command = vec![
1375        "kanidmd".to_string(),
1376        "database".to_string(),
1377        if verify { "backup" } else { "restore" }.to_string(),
1378    ];
1379    if verify {
1380        command.push("/verify/verification.json.gz".to_string());
1381    } else {
1382        if is_remote_source(restore) {
1383            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384        } else {
1385            let file_name = restore
1386                .spec
1387                .source
1388                .local
1389                .as_ref()
1390                .map(|l| l.file_name.clone())
1391                .unwrap_or_default();
1392            command.push(format!("{BACKUP_PATH}/{file_name}"));
1393        }
1394    }
1395    let job = Job {
1396        metadata: ObjectMeta {
1397            name: Some(name.to_string()),
1398            namespace: Some(ns.clone()),
1399            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400            ..Default::default()
1401        },
1402        spec: Some(JobSpec {
1403            backoff_limit: Some(0),
1404            template: PodTemplateSpec {
1405                metadata: Some(ObjectMeta::default()),
1406                spec: Some(PodSpec {
1407                    automount_service_account_token: Some(false),
1408                    restart_policy: Some("Never".to_string()),
1409                    security_context: Some(source_job_pod_security_context(target)),
1410                    containers: vec![Container {
1411                        name: if verify { "verify" } else { "restore" }.to_string(),
1412                        image: Some(restore.spec.restore_image.clone()),
1413                        command: Some(command),
1414                        env: Some(build_kanidm_db_env_vars(target)),
1415                        security_context: Some(kanidm_job_security_context()),
1416                        resources: Some(job_resource_requirements()),
1417                        volume_mounts: Some(vec![
1418                            VolumeMount {
1419                                name: DATA_VOLUME.to_string(),
1420                                mount_path: DATA_PATH.to_string(),
1421                                ..Default::default()
1422                            },
1423                            VolumeMount {
1424                                name: TLS_VOLUME.to_string(),
1425                                mount_path: TLS_PATH.to_string(),
1426                                read_only: Some(true),
1427                                ..Default::default()
1428                            },
1429                            VolumeMount {
1430                                name: "verification-output".to_string(),
1431                                mount_path: "/verify".to_string(),
1432                                ..Default::default()
1433                            },
1434                        ]),
1435                        ..Default::default()
1436                    }],
1437                    volumes: Some(vec![
1438                        Volume {
1439                            name: DATA_VOLUME.to_string(),
1440                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441                                claim_name: primary_pvc_name(target)?,
1442                                read_only: Some(false),
1443                            }),
1444                            ..Default::default()
1445                        },
1446                        Volume {
1447                            name: TLS_VOLUME.to_string(),
1448                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449                                secret_name: Some(target.effective_tls_secret_name()),
1450                                ..Default::default()
1451                            }),
1452                            ..Default::default()
1453                        },
1454                        Volume {
1455                            name: "verification-output".to_string(),
1456                            empty_dir: Some(Default::default()),
1457                            ..Default::default()
1458                        },
1459                    ]),
1460                    ..Default::default()
1461                }),
1462            },
1463            ..Default::default()
1464        }),
1465        ..Default::default()
1466    };
1467    api.create(&PostParams::default(), &job)
1468        .await
1469        .map(|_| ())
1470        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474    Running,
1475    Complete,
1476    Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480    let ns = restore.namespace().unwrap();
1481    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482        .get(name)
1483        .await
1484        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486        return Ok(JobState::Failed);
1487    }
1488    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489        return Ok(JobState::Complete);
1490    }
1491    Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495    restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499    let safety = restore.spec.safety_backup.as_ref();
1500    let skip = safety.is_some_and(|s| s.skip);
1501    !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505    let has_local = restore.spec.source.local.is_some();
1506    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507    if has_local && has_backup_ref {
1508        return Err(Error::MissingData(
1509            "source.local and source.backupRef are mutually exclusive".to_string(),
1510        ));
1511    }
1512    if !has_local && !has_backup_ref {
1513        return Err(Error::MissingData(
1514            "source must specify either local or backupRef".to_string(),
1515        ));
1516    }
1517    if let Some(local) = &restore.spec.source.local {
1518        if !safe_basename(&local.file_name) {
1519            return Err(Error::MissingData(
1520                "restore source fileName must be a safe basename".to_string(),
1521            ));
1522        }
1523    }
1524    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525        if backup_ref.name.is_empty() {
1526            return Err(Error::MissingData(
1527                "source.backupRef.name must not be empty".to_string(),
1528            ));
1529        }
1530    }
1531    Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535    restore
1536        .metadata
1537        .annotations
1538        .as_ref()
1539        .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540        .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544    let annotations = restore.metadata.annotations.as_ref();
1545    let reason = annotations
1546        .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547        .map(String::as_str)
1548        .filter(|value| !value.trim().is_empty());
1549    let approved_by = annotations
1550        .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551        .map(String::as_str)
1552        .filter(|value| !value.trim().is_empty());
1553
1554    if reason.is_none() || approved_by.is_none() {
1555        return Err(Error::MissingData(format!(
1556            "disaster-recovery restore requires non-empty {} and {} annotations",
1557            BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558        )));
1559    }
1560    Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564    restore: &KanidmRestore,
1565    target: &Kanidm,
1566    ctx: &RestoreContext,
1567    source: &DisasterRecoverySource,
1568) {
1569    ctx.metrics.disaster_recovery_total.add(1, &[]);
1570    let reason = restore
1571        .metadata
1572        .annotations
1573        .as_ref()
1574        .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575        .cloned()
1576        .unwrap_or_default();
1577    let approved_by = restore
1578        .metadata
1579        .annotations
1580        .as_ref()
1581        .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582        .cloned()
1583        .unwrap_or_default();
1584    let target_uid = target.uid().unwrap_or_default();
1585    let note = format!(
1586        "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587        source.name,
1588        source.uid,
1589        target.name_any(),
1590        target_uid,
1591        reason,
1592        approved_by
1593    );
1594    if let Err(error) = ctx
1595        .recorder
1596        .publish(
1597            &Event {
1598                type_: EventType::Warning,
1599                reason: "DisasterRecoveryOverride".to_string(),
1600                note: Some(note.clone()),
1601                action: "Restore".to_string(),
1602                secondary: None,
1603            },
1604            &restore.object_ref(&()),
1605        )
1606        .await
1607    {
1608        warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609    }
1610    warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614    let safety = restore.spec.safety_backup.as_ref();
1615    let skip = safety.is_some_and(|s| s.skip);
1616    if skip {
1617        let annotations = restore.metadata.annotations.as_ref();
1618        let reason = annotations
1619            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620            .map(|s| s.trim())
1621            .unwrap_or("");
1622        let approver = annotations
1623            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624            .map(|s| s.trim())
1625            .unwrap_or("");
1626        if reason.is_empty() {
1627            return Err(Error::MissingData(format!(
1628                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629            )));
1630        }
1631        if approver.is_empty() {
1632            return Err(Error::MissingData(format!(
1633                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634            )));
1635        }
1636    }
1637    if !skip {
1638        let has_repo = safety
1639            .and_then(|s| s.repository_ref.as_ref())
1640            .is_some_and(|r| !r.name.is_empty());
1641        if !has_repo {
1642            return Err(Error::MissingData(
1643                "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644                    .to_string(),
1645            ));
1646        }
1647    }
1648    Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652    target
1653        .spec
1654        .replica_groups
1655        .iter()
1656        .map(|rg| ReplicaCountEntry {
1657            group: rg.name.clone(),
1658            replicas: rg.replicas,
1659        })
1660        .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664    if let Ok(target) = get_target(restore, ctx).await {
1665        let owns_maintenance =
1666            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667        if owns_maintenance {
1668            scale_desired(&target, ctx).await?;
1669            clear_restoring(restore, &target, ctx).await?;
1670        }
1671    }
1672    Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676    ctx.metrics.break_glass_total.add(1, &[]);
1677    let reason = restore
1678        .metadata
1679        .annotations
1680        .as_ref()
1681        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682        .cloned()
1683        .unwrap_or_default();
1684    let approver = restore
1685        .metadata
1686        .annotations
1687        .as_ref()
1688        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689        .cloned()
1690        .unwrap_or_default();
1691    let mut status = restore.status.clone().unwrap_or_default();
1692    let break_glass_condition = restore_condition(
1693        &status.conditions,
1694        CONDITION_BREAK_GLASS,
1695        CONDITION_TRUE,
1696        "BreakGlassOverride",
1697        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698        restore.metadata.generation,
1699    );
1700    status
1701        .conditions
1702        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703    status.conditions.push(break_glass_condition);
1704    patch_status(restore, ctx, status).await?;
1705    if let Err(error) = ctx
1706        .recorder
1707        .publish(
1708            &Event {
1709                type_: EventType::Warning,
1710                reason: "SafetyBackupSkipped".to_string(),
1711                note: Some(format!(
1712                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713                )),
1714                action: "Restore".to_string(),
1715                secondary: None,
1716            },
1717            &restore.object_ref(&()),
1718        )
1719        .await
1720    {
1721        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722    }
1723    warn!(
1724        restore = %restore.name_any(),
1725        reason = %reason,
1726        approver = %approver,
1727        "break-glass safety backup override activated"
1728    );
1729    Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733    let status = restore.status.as_ref().cloned().unwrap_or_default();
1734    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735        if let Ok(start_ts) = start.parse::<Timestamp>() {
1736            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737            ctx.metrics
1738                .safety_backup_duration_seconds
1739                .record(elapsed, &[]);
1740        }
1741    }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745    format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749    format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753    format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757    restore: &KanidmRestore,
1758    target: &Kanidm,
1759    ctx: &RestoreContext,
1760    name: &str,
1761) -> Result<()> {
1762    let ns = restore.namespace().unwrap();
1763    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764    if api
1765        .get_opt(name)
1766        .await
1767        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768        .is_some()
1769    {
1770        return Ok(());
1771    }
1772    let file_name = restore
1773        .spec
1774        .source
1775        .local
1776        .as_ref()
1777        .map(|l| l.file_name.clone())
1778        .unwrap_or_default();
1779    let operation_doc = serde_json::json!({
1780        "apiVersion": "backup.kaniop.rs/v1alpha1",
1781        "kind": "OperationDocument",
1782        "operation": "check",
1783        "path": format!("{BACKUP_PATH}/{file_name}"),
1784        "resultPath": "/run/kaniop-result/result.json",
1785        "format": "kanidmJsonGzip",
1786    })
1787    .to_string();
1788    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790    let data_mover_image = data_mover_image();
1791    let job = Job {
1792        metadata: ObjectMeta {
1793            name: Some(name.to_string()),
1794            namespace: Some(ns.clone()),
1795            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796            ..Default::default()
1797        },
1798        spec: Some(JobSpec {
1799            backoff_limit: Some(0),
1800            template: PodTemplateSpec {
1801                metadata: Some(ObjectMeta::default()),
1802                spec: Some(PodSpec {
1803                    automount_service_account_token: Some(false),
1804                    restart_policy: Some("Never".to_string()),
1805                    security_context: Some(source_job_pod_security_context(target)),
1806                    containers: vec![Container {
1807                        name: "source-check".to_string(),
1808                        image: Some(data_mover_image),
1809                        command: Some(vec![
1810                            "/bin/kaniop-data-mover".to_string(),
1811                            "check".to_string(),
1812                            "--operation-doc".to_string(),
1813                            "/run/kaniop/operation.json".to_string(),
1814                        ]),
1815                        security_context: Some(hardened_security_context()),
1816                        resources: Some(job_resource_requirements()),
1817                        env: Some(vec![EnvVar {
1818                            name: "RUST_LOG".to_string(),
1819                            value: Some("info".to_string()),
1820                            ..Default::default()
1821                        }]),
1822                        volume_mounts: Some(vec![
1823                            VolumeMount {
1824                                name: DATA_VOLUME.to_string(),
1825                                mount_path: DATA_PATH.to_string(),
1826                                read_only: Some(true),
1827                                ..Default::default()
1828                            },
1829                            VolumeMount {
1830                                name: "operation".to_string(),
1831                                mount_path: "/run/kaniop".to_string(),
1832                                read_only: Some(true),
1833                                ..Default::default()
1834                            },
1835                            VolumeMount {
1836                                name: "result".to_string(),
1837                                mount_path: "/run/kaniop-result".to_string(),
1838                                ..Default::default()
1839                            },
1840                        ]),
1841                        ..Default::default()
1842                    }],
1843                    volumes: Some(vec![
1844                        Volume {
1845                            name: DATA_VOLUME.to_string(),
1846                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847                                claim_name: primary_pvc_name(target)?,
1848                                read_only: Some(true),
1849                            }),
1850                            ..Default::default()
1851                        },
1852                        Volume {
1853                            name: "operation".to_string(),
1854                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855                                name: operation_cm_name,
1856                                ..Default::default()
1857                            }),
1858                            ..Default::default()
1859                        },
1860                        Volume {
1861                            name: "result".to_string(),
1862                            empty_dir: Some(Default::default()),
1863                            ..Default::default()
1864                        },
1865                    ]),
1866                    ..Default::default()
1867                }),
1868            },
1869            ..Default::default()
1870        }),
1871        ..Default::default()
1872    };
1873    api.create(&PostParams::default(), &job)
1874        .await
1875        .map(|_| ())
1876        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882    let ns = restore.namespace().unwrap_or_default();
1883    let uid = restore.uid().ok_or_else(|| {
1884        Error::MissingData(
1885            "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886                .to_string(),
1887        )
1888    })?;
1889    let namespace_uuid =
1890        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892    Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1896    let ns = restore.namespace().unwrap_or_default();
1897    let kanidm_uid = &restore.spec.target_ref.uid;
1898    let backup_id = compute_safety_backup_id(restore)?;
1899    Ok(format!(
1900        "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1901    ))
1902}
1903
1904struct VerifiedSafetyBackup {
1905    manifest_key: String,
1906    payload_sha256: String,
1907}
1908
1909async fn read_safety_backup_result(
1910    restore: &KanidmRestore,
1911    ctx: &RestoreContext,
1912    job_name: &str,
1913    expected_backup_id: &str,
1914) -> Result<VerifiedSafetyBackup> {
1915    let ns = restore.namespace().unwrap();
1916    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1917    let pod_list = pods
1918        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1919        .await
1920        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1921    let pod = pod_list.items.first().ok_or_else(|| {
1922        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1923    })?;
1924    let pod_name = pod.name_any();
1925    let container_status = pod
1926        .status
1927        .as_ref()
1928        .and_then(|s| s.container_statuses.as_ref())
1929        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1930        .ok_or_else(|| {
1931            Error::MissingData(format!(
1932                "safety-upload container status not found in pod {pod_name}"
1933            ))
1934        })?;
1935    let termination_message = container_status
1936        .state
1937        .as_ref()
1938        .and_then(|state| state.terminated.as_ref())
1939        .and_then(|t| t.message.as_ref())
1940        .ok_or_else(|| {
1941            Error::MissingData(format!(
1942                "termination message not found in pod {pod_name}; result transport failed"
1943            ))
1944        })?;
1945    if termination_message.is_empty() {
1946        return Err(Error::MissingData(format!(
1947            "termination message is empty in pod {pod_name}; result document was not written"
1948        )));
1949    }
1950    let result_doc = parse_result_document(termination_message).map_err(|e| {
1951        Error::ParseError(format!(
1952            "failed to parse result document from pod {pod_name}: {e}"
1953        ))
1954    })?;
1955    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1956        return Err(Error::ParseError(format!(
1957            "unsupported result document version: {}",
1958            result_doc.api_version
1959        )));
1960    }
1961    if result_doc.kind != "ResultDocument" {
1962        return Err(Error::ParseError(format!(
1963            "invalid result document kind: {}",
1964            result_doc.kind
1965        )));
1966    }
1967    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1968        return Err(Error::ParseError(format!(
1969            "result document operation mismatch: expected '{}', got '{}'",
1970            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1971        )));
1972    }
1973    if !result_doc.success {
1974        return Err(Error::ParseError(format!(
1975            "safety backup reported failure with exit code {:?}",
1976            result_doc.exit_code
1977        )));
1978    }
1979    if result_doc.exit_code != ExitCode::Success {
1980        return Err(Error::ParseError(format!(
1981            "safety backup exit code is not Success: {:?}",
1982            result_doc.exit_code
1983        )));
1984    }
1985    let doc_backup_id = result_doc
1986        .backup_id
1987        .as_deref()
1988        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1989    if doc_backup_id != expected_backup_id {
1990        return Err(Error::ParseError(format!(
1991            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1992        )));
1993    }
1994    let manifest_key = result_doc
1995        .manifest_key
1996        .clone()
1997        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1998    if manifest_key.is_empty() {
1999        return Err(Error::ParseError(
2000            "result document manifestKey is empty".to_string(),
2001        ));
2002    }
2003    let payload_sha256 = result_doc
2004        .payload_sha256
2005        .clone()
2006        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2007    if payload_sha256.is_empty() {
2008        return Err(Error::ParseError(
2009            "result document payloadSha256 is empty".to_string(),
2010        ));
2011    }
2012    info!(
2013        restore = %restore.name_any(),
2014        backup_id = %expected_backup_id,
2015        manifest_key = %manifest_key,
2016        "safety backup result verified"
2017    );
2018    Ok(VerifiedSafetyBackup {
2019        manifest_key,
2020        payload_sha256,
2021    })
2022}
2023
2024struct VerifiedSourcePrepResult {
2025    #[allow(dead_code)]
2026    manifest_key: String,
2027    payload_sha256: String,
2028}
2029
2030async fn read_source_prep_result(
2031    restore: &KanidmRestore,
2032    ctx: &RestoreContext,
2033    job_name: &str,
2034    expected_backup_id: &str,
2035) -> Result<VerifiedSourcePrepResult> {
2036    let ns = restore.namespace().unwrap();
2037    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2038    let pod_list = pods
2039        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2040        .await
2041        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2042    let pod = pod_list.items.first().ok_or_else(|| {
2043        Error::MissingData(format!("no pod found for source prep job {job_name}"))
2044    })?;
2045    let pod_name = pod.name_any();
2046    let container_status = pod
2047        .status
2048        .as_ref()
2049        .and_then(|s| s.container_statuses.as_ref())
2050        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2051        .ok_or_else(|| {
2052            Error::MissingData(format!(
2053                "source-prep container status not found in pod {pod_name}"
2054            ))
2055        })?;
2056    let termination_message = container_status
2057        .state
2058        .as_ref()
2059        .and_then(|state| state.terminated.as_ref())
2060        .and_then(|t| t.message.as_ref())
2061        .ok_or_else(|| {
2062            Error::MissingData(format!(
2063                "termination message not found in pod {pod_name}; result transport failed"
2064            ))
2065        })?;
2066    if termination_message.is_empty() {
2067        return Err(Error::MissingData(format!(
2068            "termination message is empty in pod {pod_name}; result document was not written"
2069        )));
2070    }
2071    let result_doc = parse_result_document(termination_message).map_err(|e| {
2072        Error::ParseError(format!(
2073            "failed to parse result document from pod {pod_name}: {e}"
2074        ))
2075    })?;
2076    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2077        return Err(Error::ParseError(format!(
2078            "unsupported result document version: {}",
2079            result_doc.api_version
2080        )));
2081    }
2082    if result_doc.kind != "ResultDocument" {
2083        return Err(Error::ParseError(format!(
2084            "invalid result document kind: {}",
2085            result_doc.kind
2086        )));
2087    }
2088    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2089        return Err(Error::ParseError(format!(
2090            "result document operation mismatch: expected '{}', got '{}'",
2091            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2092        )));
2093    }
2094    if !result_doc.success {
2095        return Err(Error::ParseError(format!(
2096            "source prep reported failure with exit code {:?}",
2097            result_doc.exit_code
2098        )));
2099    }
2100    if result_doc.exit_code != ExitCode::Success {
2101        return Err(Error::ParseError(format!(
2102            "source prep exit code is not Success: {:?}",
2103            result_doc.exit_code
2104        )));
2105    }
2106    let doc_backup_id = result_doc
2107        .backup_id
2108        .as_deref()
2109        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2110    if doc_backup_id != expected_backup_id {
2111        return Err(Error::ParseError(format!(
2112            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2113        )));
2114    }
2115    let manifest_key = result_doc
2116        .manifest_key
2117        .clone()
2118        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2119    if manifest_key.is_empty() {
2120        return Err(Error::ParseError(
2121            "result document manifestKey is empty".to_string(),
2122        ));
2123    }
2124    let payload_sha256 = result_doc
2125        .payload_sha256
2126        .clone()
2127        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2128    if payload_sha256.is_empty() {
2129        return Err(Error::ParseError(
2130            "result document payloadSha256 is empty".to_string(),
2131        ));
2132    }
2133    info!(
2134        restore = %restore.name_any(),
2135        backup_id = %expected_backup_id,
2136        manifest_key = %manifest_key,
2137        payload_sha256 = %payload_sha256,
2138        "source prep result verified"
2139    );
2140    Ok(VerifiedSourcePrepResult {
2141        manifest_key,
2142        payload_sha256,
2143    })
2144}
2145
2146fn source_job_pod_security_context(
2147    target: &Kanidm,
2148) -> k8s_openapi::api::core::v1::PodSecurityContext {
2149    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2150    security_context.seccomp_profile = Some(SeccompProfile {
2151        type_: "RuntimeDefault".to_string(),
2152        ..Default::default()
2153    });
2154    security_context
2155}
2156
2157fn hardened_security_context() -> SecurityContext {
2158    SecurityContext {
2159        allow_privilege_escalation: Some(false),
2160        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2161            drop: Some(vec!["ALL".to_string()]),
2162            ..Default::default()
2163        }),
2164        read_only_root_filesystem: Some(true),
2165        run_as_non_root: Some(true),
2166        seccomp_profile: Some(SeccompProfile {
2167            type_: "RuntimeDefault".to_string(),
2168            ..Default::default()
2169        }),
2170        ..Default::default()
2171    }
2172}
2173
2174fn kanidm_job_security_context() -> SecurityContext {
2175    SecurityContext {
2176        allow_privilege_escalation: Some(false),
2177        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2178            drop: Some(vec!["ALL".to_string()]),
2179            ..Default::default()
2180        }),
2181        seccomp_profile: Some(SeccompProfile {
2182            type_: "RuntimeDefault".to_string(),
2183            ..Default::default()
2184        }),
2185        ..Default::default()
2186    }
2187}
2188
2189fn job_resource_requirements() -> ResourceRequirements {
2190    ResourceRequirements {
2191        requests: Some(std::collections::BTreeMap::from([
2192            ("cpu".to_string(), Quantity("100m".to_string())),
2193            ("memory".to_string(), Quantity("128Mi".to_string())),
2194        ])),
2195        limits: Some(std::collections::BTreeMap::from([
2196            ("cpu".to_string(), Quantity("2".to_string())),
2197            ("memory".to_string(), Quantity("2Gi".to_string())),
2198        ])),
2199        ..Default::default()
2200    }
2201}
2202
2203async fn ensure_safety_backup_job(
2204    restore: &KanidmRestore,
2205    target: &Kanidm,
2206    ctx: &RestoreContext,
2207    name: &str,
2208    backup_id: &str,
2209    manifest_key: &str,
2210) -> Result<()> {
2211    let ns = restore.namespace().unwrap();
2212    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2213    if api
2214        .get_opt(name)
2215        .await
2216        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2217        .is_some()
2218    {
2219        return Ok(());
2220    }
2221    let safety = restore
2222        .spec
2223        .safety_backup
2224        .as_ref()
2225        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2226    let repo_ref = safety
2227        .repository_ref
2228        .as_ref()
2229        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2230    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2231        .get(&repo_ref.name)
2232        .await
2233        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2234    let _namespace_uid = &ns;
2235    let _kanidm_uid = &restore.spec.target_ref.uid;
2236    let endpoint = &repo.spec.s3.endpoint;
2237    let region = &repo.spec.s3.region;
2238    let operation_doc = build_safety_upload_operation_doc(
2239        restore,
2240        target,
2241        backup_id,
2242        manifest_key,
2243        &repo.spec.s3.bucket,
2244        &repo.spec.s3.prefix,
2245        endpoint,
2246        region,
2247        repo.spec.s3.force_path_style,
2248        repo.spec.s3.insecure,
2249        repo.spec
2250            .s3
2251            .ca_bundle_ref
2252            .as_deref()
2253            .map(|_| ca_bundle_path())
2254            .as_deref(),
2255        repo.spec.encryption.as_ref(),
2256    )?;
2257    let operation_cm_name = format!("{name}-op");
2258    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2259    let auth_method = &repo.spec.authentication.writer;
2260    let data_mover_image = data_mover_image();
2261    let init_command = vec![
2262        "kanidmd".to_string(),
2263        "database".to_string(),
2264        "backup".to_string(),
2265        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2266    ];
2267    let upload_script = format!(
2268        r#"set -eu
2269/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2270RESULT_FILE="/run/kaniop-result/result.json"
2271TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2272if [ ! -f "$RESULT_FILE" ]; then
2273  echo "ERROR: result document not found at $RESULT_FILE"
2274  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2275  exit 1
2276fi
2277if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2278  echo "ERROR: safety backup upload reported failure"
2279  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2280  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2281  exit 1
2282fi
2283dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2284echo "safety backup upload completed successfully"
2285"#
2286    );
2287    let job = Job {
2288        metadata: ObjectMeta {
2289            name: Some(name.to_string()),
2290            namespace: Some(ns.clone()),
2291            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2292            ..Default::default()
2293        },
2294        spec: Some(JobSpec {
2295            backoff_limit: Some(0),
2296            template: PodTemplateSpec {
2297                metadata: Some(ObjectMeta::default()),
2298                spec: Some(PodSpec {
2299                    automount_service_account_token: Some(false),
2300                    restart_policy: Some("Never".to_string()),
2301                    security_context: Some(source_job_pod_security_context(target)),
2302                    init_containers: Some(vec![Container {
2303                        name: "safety-backup".to_string(),
2304                        image: Some(restore.spec.restore_image.clone()),
2305                        command: Some(init_command),
2306                        security_context: Some(SecurityContext {
2307                            allow_privilege_escalation: Some(false),
2308                            capabilities: Some(Capabilities {
2309                                drop: Some(vec!["ALL".to_string()]),
2310                                ..Default::default()
2311                            }),
2312                            read_only_root_filesystem: Some(true),
2313                            seccomp_profile: Some(SeccompProfile {
2314                                type_: "RuntimeDefault".to_string(),
2315                                ..Default::default()
2316                            }),
2317                            ..Default::default()
2318                        }),
2319                        resources: Some(job_resource_requirements()),
2320                        env: Some(build_kanidm_db_env_vars(target)),
2321                        volume_mounts: Some(vec![
2322                            VolumeMount {
2323                                name: DATA_VOLUME.to_string(),
2324                                mount_path: DATA_PATH.to_string(),
2325                                read_only: Some(false),
2326                                ..Default::default()
2327                            },
2328                            VolumeMount {
2329                                name: TLS_VOLUME.to_string(),
2330                                mount_path: TLS_PATH.to_string(),
2331                                read_only: Some(true),
2332                                ..Default::default()
2333                            },
2334                            VolumeMount {
2335                                name: SHARED_VOLUME.to_string(),
2336                                mount_path: SHARED_VOL_PATH.to_string(),
2337                                ..Default::default()
2338                            },
2339                        ]),
2340                        ..Default::default()
2341                    }]),
2342                    containers: vec![Container {
2343                        name: "safety-upload".to_string(),
2344                        image: Some(data_mover_image),
2345                        command: Some(vec!["/bin/sh".to_string()]),
2346                        args: Some(vec!["-c".to_string(), upload_script]),
2347                        security_context: Some(hardened_security_context()),
2348                        resources: Some(job_resource_requirements()),
2349                        env: {
2350                            let mut env_vars =
2351                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2352                            env_vars.push(EnvVar {
2353                                name: "RUST_LOG".to_string(),
2354                                value: Some("info".to_string()),
2355                                ..Default::default()
2356                            });
2357                            if repo.spec.s3.ca_bundle_ref.is_some() {
2358                                env_vars.push(ca_bundle_env_var());
2359                            }
2360                            env_vars.extend(build_encryption_env_vars(
2361                                repo.spec
2362                                    .encryption
2363                                    .as_ref()
2364                                    .and_then(|e| e.key_ref.as_ref()),
2365                            ));
2366                            Some(env_vars)
2367                        },
2368                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2369                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2370                        volume_mounts: {
2371                            let mut mounts = vec![
2372                                VolumeMount {
2373                                    name: SHARED_VOLUME.to_string(),
2374                                    mount_path: SHARED_VOL_PATH.to_string(),
2375                                    read_only: Some(true),
2376                                    ..Default::default()
2377                                },
2378                                VolumeMount {
2379                                    name: "operation".to_string(),
2380                                    mount_path: "/run/kaniop".to_string(),
2381                                    read_only: Some(true),
2382                                    ..Default::default()
2383                                },
2384                                VolumeMount {
2385                                    name: "result".to_string(),
2386                                    mount_path: "/run/kaniop-result".to_string(),
2387                                    ..Default::default()
2388                                },
2389                            ];
2390                            mounts.extend(build_auth_volume_mounts(auth_method));
2391                            if repo.spec.s3.ca_bundle_ref.is_some() {
2392                                mounts.push(build_ca_bundle_volume_mount());
2393                            }
2394                            Some(mounts)
2395                        },
2396                        ..Default::default()
2397                    }],
2398                    volumes: {
2399                        let mut vols = vec![
2400                            Volume {
2401                                name: DATA_VOLUME.to_string(),
2402                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2403                                    claim_name: primary_pvc_name(target)?,
2404                                    read_only: Some(false),
2405                                }),
2406                                ..Default::default()
2407                            },
2408                            Volume {
2409                                name: TLS_VOLUME.to_string(),
2410                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2411                                    secret_name: Some(target.effective_tls_secret_name()),
2412                                    ..Default::default()
2413                                }),
2414                                ..Default::default()
2415                            },
2416                            Volume {
2417                                name: SHARED_VOLUME.to_string(),
2418                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2419                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2420                                    ..Default::default()
2421                                }),
2422                                ..Default::default()
2423                            },
2424                            Volume {
2425                                name: "operation".to_string(),
2426                                config_map: Some(ConfigMapVolumeSource {
2427                                    name: operation_cm_name,
2428                                    ..Default::default()
2429                                }),
2430                                ..Default::default()
2431                            },
2432                            Volume {
2433                                name: "result".to_string(),
2434                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2435                                    size_limit: Some(Quantity("16Mi".to_string())),
2436                                    ..Default::default()
2437                                }),
2438                                ..Default::default()
2439                            },
2440                        ];
2441                        vols.extend(build_auth_volumes(auth_method));
2442                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2443                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2444                        }
2445                        Some(vols)
2446                    },
2447                    ..Default::default()
2448                }),
2449            },
2450            ..Default::default()
2451        }),
2452        ..Default::default()
2453    };
2454    api.create(&PostParams::default(), &job)
2455        .await
2456        .map(|_| ())
2457        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2458}
2459
2460async fn ensure_source_prep_job(
2461    restore: &KanidmRestore,
2462    target: &Kanidm,
2463    ctx: &RestoreContext,
2464    name: &str,
2465) -> Result<()> {
2466    let ns = restore.namespace().unwrap();
2467    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2468    if api
2469        .get_opt(name)
2470        .await
2471        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2472        .is_some()
2473    {
2474        return Ok(());
2475    }
2476    let backup_name = restore
2477        .spec
2478        .source
2479        .backup_ref
2480        .as_ref()
2481        .map(|b| b.name.clone())
2482        .unwrap_or_default();
2483    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2484        .get(&backup_name)
2485        .await
2486        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2487    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2488        .get(&backup.spec.repository_ref.name)
2489        .await
2490        .map_err(|e| {
2491            Error::kube_error(
2492                "get",
2493                "KanidmBackupRepository",
2494                &ns,
2495                &backup.spec.repository_ref.name,
2496                e,
2497            )
2498        })?;
2499    let endpoint = &repo.spec.s3.endpoint;
2500    let region = &repo.spec.s3.region;
2501    let operation_doc = build_download_operation_doc(
2502        target,
2503        &backup.spec.manifest_key,
2504        &backup.spec.backup_id,
2505        &backup.spec.kanidm_ref.uid,
2506        &repo.spec.s3.bucket,
2507        &repo.spec.s3.prefix,
2508        endpoint,
2509        region,
2510        repo.spec.s3.force_path_style,
2511        repo.spec.s3.insecure,
2512        repo.spec
2513            .s3
2514            .ca_bundle_ref
2515            .as_deref()
2516            .map(|_| ca_bundle_path())
2517            .as_deref(),
2518        repo.spec.encryption.as_ref(),
2519    );
2520    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2521    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2522    let auth_method = &repo.spec.authentication.reader;
2523    let data_mover_image = data_mover_image();
2524    let download_script = format!(
2525        r#"set -eu
2526/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2527RESULT_FILE="/run/kaniop-result/result.json"
2528TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2529if [ ! -f "$RESULT_FILE" ]; then
2530  echo "ERROR: result document not found at $RESULT_FILE"
2531  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2532  exit 1
2533fi
2534if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2535  echo "ERROR: source preparation download reported failure"
2536  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2537  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2538  exit 1
2539fi
2540dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2541echo "source preparation download completed successfully"
2542"#
2543    );
2544    let job = Job {
2545        metadata: ObjectMeta {
2546            name: Some(name.to_string()),
2547            namespace: Some(ns.clone()),
2548            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2549            ..Default::default()
2550        },
2551        spec: Some(JobSpec {
2552            backoff_limit: Some(0),
2553            template: PodTemplateSpec {
2554                metadata: Some(ObjectMeta::default()),
2555                spec: Some(PodSpec {
2556                    automount_service_account_token: Some(false),
2557                    restart_policy: Some("Never".to_string()),
2558                    security_context: Some(source_job_pod_security_context(target)),
2559                    containers: vec![Container {
2560                        name: "source-prep".to_string(),
2561                        image: Some(data_mover_image),
2562                        command: Some(vec!["/bin/sh".to_string()]),
2563                        args: Some(vec!["-c".to_string(), download_script]),
2564                        security_context: Some(hardened_security_context()),
2565                        resources: Some(job_resource_requirements()),
2566                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2567                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2568                        env: {
2569                            let mut env_vars = build_auth_env_vars(
2570                                auth_method,
2571                                &backup.spec.repository_ref.name,
2572                                AuthRole::Reader,
2573                            );
2574                            env_vars.push(EnvVar {
2575                                name: "RUST_LOG".to_string(),
2576                                value: Some("info".to_string()),
2577                                ..Default::default()
2578                            });
2579                            if repo.spec.s3.ca_bundle_ref.is_some() {
2580                                env_vars.push(ca_bundle_env_var());
2581                            }
2582                            env_vars.extend(build_encryption_env_vars(
2583                                repo.spec
2584                                    .encryption
2585                                    .as_ref()
2586                                    .and_then(|e| e.key_ref.as_ref()),
2587                            ));
2588                            Some(env_vars)
2589                        },
2590                        volume_mounts: {
2591                            let mut mounts = vec![
2592                                VolumeMount {
2593                                    name: DATA_VOLUME.to_string(),
2594                                    mount_path: DATA_PATH.to_string(),
2595                                    ..Default::default()
2596                                },
2597                                VolumeMount {
2598                                    name: "operation".to_string(),
2599                                    mount_path: "/run/kaniop".to_string(),
2600                                    read_only: Some(true),
2601                                    ..Default::default()
2602                                },
2603                                VolumeMount {
2604                                    name: "result".to_string(),
2605                                    mount_path: "/run/kaniop-result".to_string(),
2606                                    ..Default::default()
2607                                },
2608                            ];
2609                            mounts.extend(build_auth_volume_mounts(auth_method));
2610                            if repo.spec.s3.ca_bundle_ref.is_some() {
2611                                mounts.push(build_ca_bundle_volume_mount());
2612                            }
2613                            Some(mounts)
2614                        },
2615                        ..Default::default()
2616                    }],
2617                    volumes: {
2618                        let mut vols = vec![
2619                            Volume {
2620                                name: DATA_VOLUME.to_string(),
2621                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2622                                    claim_name: primary_pvc_name(target)?,
2623                                    read_only: Some(false),
2624                                }),
2625                                ..Default::default()
2626                            },
2627                            Volume {
2628                                name: "operation".to_string(),
2629                                config_map: Some(ConfigMapVolumeSource {
2630                                    name: operation_cm_name,
2631                                    ..Default::default()
2632                                }),
2633                                ..Default::default()
2634                            },
2635                            Volume {
2636                                name: "result".to_string(),
2637                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2638                                    size_limit: Some(Quantity("16Mi".to_string())),
2639                                    ..Default::default()
2640                                }),
2641                                ..Default::default()
2642                            },
2643                        ];
2644                        vols.extend(build_auth_volumes(auth_method));
2645                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2646                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2647                        }
2648                        Some(vols)
2649                    },
2650                    ..Default::default()
2651                }),
2652            },
2653            ..Default::default()
2654        }),
2655        ..Default::default()
2656    };
2657    api.create(&PostParams::default(), &job)
2658        .await
2659        .map(|_| ())
2660        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2661}
2662
2663#[allow(clippy::too_many_arguments)]
2664fn build_safety_upload_operation_doc(
2665    restore: &KanidmRestore,
2666    target: &Kanidm,
2667    backup_id: &str,
2668    manifest_key: &str,
2669    bucket: &str,
2670    prefix: &str,
2671    endpoint: &str,
2672    region: &str,
2673    force_path_style: bool,
2674    insecure: bool,
2675    ca_bundle_path: Option<&str>,
2676    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2677) -> Result<String> {
2678    let enc_mode =
2679        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2680    let enc_key_id = encryption
2681        .and_then(|e| e.key_id.as_ref())
2682        .map(|s| serde_json::Value::String(s.clone()));
2683    let op = UploadOperation {
2684        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2685        bucket: bucket.to_string(),
2686        prefix: prefix.to_string(),
2687        endpoint: endpoint.to_string(),
2688        region: region.to_string(),
2689        force_path_style,
2690        insecure,
2691        ca_bundle_path: ca_bundle_path.map(str::to_string),
2692        backup_id: backup_id.to_string(),
2693        namespace_uid: restore.namespace().unwrap_or_default(),
2694        kanidm_uid: restore.spec.target_ref.uid.clone(),
2695        kanidm_name: restore.spec.target_ref.name.clone(),
2696        domain: target.spec.domain.clone(),
2697        kanidm_version: target
2698            .status
2699            .as_ref()
2700            .and_then(|s| s.version.as_ref())
2701            .map(|v| v.image_tag.clone())
2702            .unwrap_or_default(),
2703        image_digest: None,
2704        consistency: "kanidm-offline".to_string(),
2705        reason: "restore-safety".to_string(),
2706        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2707        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2708        result_path: "/run/kaniop-result/result.json".to_string(),
2709        max_concurrent_parts: 4,
2710        max_retries: 3,
2711    };
2712    let _ = manifest_key;
2713    let doc = OperationDocument {
2714        api_version: OPERATION_DOC_VERSION.to_string(),
2715        kind: "OperationDocument".to_string(),
2716        spec: OperationSpec::Upload(op),
2717    };
2718    serde_json::to_string(&doc)
2719        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2720}
2721
2722async fn validate_backup_ref(
2723    restore: &KanidmRestore,
2724    target: &Kanidm,
2725    ctx: &RestoreContext,
2726) -> Result<Option<DisasterRecoverySource>> {
2727    let backup_ref = restore
2728        .spec
2729        .source
2730        .backup_ref
2731        .as_ref()
2732        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2733    let ns = restore.namespace().unwrap();
2734    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2735        .get(&backup_ref.name)
2736        .await
2737        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2738    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2739        return Err(Error::MissingData(format!(
2740            "KanidmBackup '{}' is not Ready (phase: {:?})",
2741            backup_ref.name,
2742            backup.status.as_ref().map(|s| &s.phase)
2743        )));
2744    }
2745    let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2746        || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2747    let disaster_recovery_source = if source_identity_mismatch {
2748        if !disaster_recovery_requested(restore) {
2749            return Err(Error::MissingData(format!(
2750                "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2751                backup.spec.kanidm_ref.name,
2752                backup.spec.kanidm_ref.uid,
2753                restore.spec.target_ref.name,
2754                restore.spec.target_ref.uid,
2755                DISASTER_RECOVERY_ANNOTATION,
2756                BREAK_GLASS_REASON_ANNOTATION,
2757                BREAK_GLASS_APPROVED_BY_ANNOTATION,
2758            )));
2759        }
2760        validate_disaster_recovery_approval(restore)?;
2761        Some(DisasterRecoverySource {
2762            name: backup.spec.kanidm_ref.name.clone(),
2763            uid: backup.spec.kanidm_ref.uid.clone(),
2764        })
2765    } else {
2766        None
2767    };
2768    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2769        .get(&backup.spec.repository_ref.name)
2770        .await
2771        .map_err(|e| {
2772            Error::kube_error(
2773                "get",
2774                "KanidmBackupRepository",
2775                &ns,
2776                &backup.spec.repository_ref.name,
2777                e,
2778            )
2779        })?;
2780    if !has_accepted_condition(
2781        &repo
2782            .status
2783            .as_ref()
2784            .map(|s| &s.conditions)
2785            .cloned()
2786            .unwrap_or_default(),
2787    ) {
2788        return Err(Error::MissingData(format!(
2789            "KanidmBackupRepository '{}' configuration has not been accepted",
2790            backup.spec.repository_ref.name
2791        )));
2792    }
2793    validate_backup_compatibility(restore, target, &backup)?;
2794    Ok(disaster_recovery_source)
2795}
2796
2797fn has_accepted_condition(conditions: &[Condition]) -> bool {
2798    conditions
2799        .iter()
2800        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2801}
2802
2803fn validate_backup_compatibility(
2804    restore: &KanidmRestore,
2805    target: &Kanidm,
2806    backup: &KanidmBackup,
2807) -> Result<()> {
2808    let status = backup.status.as_ref();
2809    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2810        let target_version = target
2811            .status
2812            .as_ref()
2813            .and_then(|s| s.version.as_ref())
2814            .map(|v| v.image_tag.as_str());
2815        if let Some(tv) = target_version {
2816            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2817                return Err(Error::MissingData(format!(
2818                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2819                )));
2820            }
2821        }
2822    }
2823    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2824        if !backup_digest.is_empty()
2825            && restore.spec.restore_image.contains('@')
2826            && !restore.spec.restore_image.contains(backup_digest)
2827        {
2828            return Err(Error::MissingData(format!(
2829                "restore image digest does not match backup image digest '{backup_digest}'"
2830            )));
2831        }
2832    }
2833    Ok(())
2834}
2835
2836#[allow(clippy::too_many_arguments)]
2837fn build_download_operation_doc(
2838    target: &Kanidm,
2839    manifest_key: &str,
2840    expected_backup_id: &str,
2841    expected_source_uid: &str,
2842    bucket: &str,
2843    prefix: &str,
2844    endpoint: &str,
2845    region: &str,
2846    force_path_style: bool,
2847    insecure: bool,
2848    ca_bundle_path: Option<&str>,
2849    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2850) -> String {
2851    let enc_mode =
2852        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2853    let enc_key_id = encryption
2854        .and_then(|e| e.key_id.as_ref())
2855        .map(|s| serde_json::Value::String(s.clone()));
2856    serde_json::json!({
2857        "apiVersion": "backup.kaniop.rs/v1alpha1",
2858        "kind": "OperationDocument",
2859        "operation": "download",
2860        "manifestKey": manifest_key,
2861        "bucket": bucket,
2862        "prefix": prefix,
2863        "endpoint": endpoint,
2864        "region": region,
2865        "forcePathStyle": force_path_style,
2866        "insecure": insecure,
2867        "caBundlePath": ca_bundle_path,
2868        "expectedBackupId": expected_backup_id,
2869        "expectedKanidmUid": expected_source_uid,
2870        "expectedDomain": target.spec.domain,
2871        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2872        "resultPath": "/run/kaniop-result/result.json",
2873        "maxRetries": 3,
2874        "encryptionMode": enc_mode,
2875        "encryptionKeyId": enc_key_id,
2876    })
2877    .to_string()
2878}
2879
2880async fn ensure_operation_configmap(
2881    restore: &KanidmRestore,
2882    name: &str,
2883    data: &str,
2884    ns: &str,
2885    ctx: &RestoreContext,
2886) -> Result<()> {
2887    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2888    if api
2889        .get_opt(name)
2890        .await
2891        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2892        .is_some()
2893    {
2894        return Ok(());
2895    }
2896    let cm = ConfigMap {
2897        metadata: ObjectMeta {
2898            name: Some(name.to_string()),
2899            namespace: Some(ns.to_string()),
2900            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2901            ..Default::default()
2902        },
2903        data: Some(std::collections::BTreeMap::from([(
2904            "operation.json".to_string(),
2905            data.to_string(),
2906        )])),
2907        ..Default::default()
2908    };
2909    api.create(&PostParams::default(), &cm)
2910        .await
2911        .map(|_| ())
2912        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2913}
2914
2915#[cfg(test)]
2916mod tests {
2917    use super::{
2918        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2919        CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2920        DISASTER_RECOVERY_ANNOTATION,
2921        KanidmRestore, KanidmRestorePhase,
2922        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2923        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2924        disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2925        is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2926        restore_job_name, safe_basename, validate_disaster_recovery_approval,
2927        validate_safety_backup_config, validate_source, verify_job_name,
2928    };
2929    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2930    use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2931    use k8s_openapi::jiff::Timestamp;
2932    use kube::api::ObjectMeta as ApiObjectMeta;
2933    use std::collections::BTreeMap;
2934
2935    fn make_restore(
2936        source: KanidmRestoreSource,
2937        safety: Option<SafetyBackupConfig>,
2938    ) -> KanidmRestore {
2939        KanidmRestore {
2940            metadata: ApiObjectMeta {
2941                name: Some("test-restore".to_string()),
2942                namespace: Some("default".to_string()),
2943                uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2944                ..Default::default()
2945            },
2946            spec: KanidmRestoreSpec {
2947                target_ref: KanidmRestoreTargetRef {
2948                    name: "corp-idm".to_string(),
2949                    uid: "test-uid-123".to_string(),
2950                },
2951                source,
2952                restore_image: "kanidm/server@sha256:abc".to_string(),
2953                safety_backup: safety,
2954            },
2955            status: None,
2956        }
2957    }
2958
2959    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2960        super::super::crd::Kanidm {
2961            status: Some(super::super::crd::KanidmStatus {
2962                available_replicas: 1,
2963                replicas: 1,
2964                unavailable_replicas: 0,
2965                updated_replicas: 1,
2966                replica_statuses: vec![],
2967                replica_column: "1/1".to_string(),
2968                secret_name: None,
2969                version: Some(super::super::crd::KanidmVersionStatus {
2970                    image_tag: image_tag.to_string(),
2971                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2972                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2973                }),
2974                domain_appearance_image: None,
2975                mail_sender: None,
2976                conditions: None,
2977            }),
2978            ..Default::default()
2979        }
2980    }
2981
2982    #[test]
2983    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2984        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2985    }
2986
2987    #[test]
2988    fn completed_restore_sets_ready_condition() {
2989        let mut status = KanidmRestoreStatus {
2990            phase: KanidmRestorePhase::Completed,
2991            ..Default::default()
2992        };
2993        super::update_restore_conditions(&mut status, Some(7));
2994        assert!(status.conditions.iter().any(|condition| {
2995            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2996        }));
2997        assert!(!status.conditions.iter().any(|condition| {
2998            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2999        }));
3000    }
3001
3002    #[test]
3003    fn disaster_recovery_condition_survives_phase_updates() {
3004        let dr_condition = Condition {
3005            type_: CONDITION_DISASTER_RECOVERY.to_string(),
3006            status: CONDITION_TRUE.to_string(),
3007            reason: "ApprovedSourceIdentityMismatch".to_string(),
3008            message: "approved DR".to_string(),
3009            last_transition_time: Time(Timestamp::now()),
3010            observed_generation: Some(7),
3011        };
3012        let mut status = KanidmRestoreStatus {
3013            phase: KanidmRestorePhase::Quiescing,
3014            conditions: vec![dr_condition.clone()],
3015            ..Default::default()
3016        };
3017
3018        super::update_restore_conditions(&mut status, Some(8));
3019
3020        assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3021    }
3022
3023    #[test]
3024    fn failed_restore_sets_failed_condition() {
3025        let mut status = KanidmRestoreStatus {
3026            phase: KanidmRestorePhase::Failed,
3027            message: Some("verification failed".to_string()),
3028            ..Default::default()
3029        };
3030        super::update_restore_conditions(&mut status, Some(8));
3031        assert!(status.conditions.iter().any(|condition| {
3032            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3033        }));
3034    }
3035
3036    #[test]
3037    fn restore_filename_is_confined_to_backup_directory() {
3038        assert!(safe_basename("backup.json.gz"));
3039        assert!(!safe_basename("../backup.json.gz"));
3040        assert!(!safe_basename("a/b"));
3041        assert!(!safe_basename(""));
3042    }
3043
3044    #[test]
3045    fn restore_image_rejects_latest_and_unpinned_names() {
3046        assert!(mutable_image("kanidm/server:latest"));
3047        assert!(mutable_image("kanidm/server"));
3048        assert!(!mutable_image("kanidm/server:1.10.0"));
3049        assert!(!mutable_image("kanidm/server@sha256:abc"));
3050    }
3051
3052    #[test]
3053    fn source_validation_rejects_both_local_and_backup_ref() {
3054        let source = KanidmRestoreSource {
3055            local: Some(super::KanidmRestoreLocalSource {
3056                file_name: "backup.json".to_string(),
3057            }),
3058            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3059                name: "some-backup".to_string(),
3060            }),
3061        };
3062        let restore = make_restore(source, None);
3063        assert!(validate_source(&restore).is_err());
3064    }
3065
3066    #[test]
3067    fn source_validation_rejects_empty_source() {
3068        let source = KanidmRestoreSource {
3069            local: None,
3070            backup_ref: None,
3071        };
3072        let restore = make_restore(source, None);
3073        assert!(validate_source(&restore).is_err());
3074    }
3075
3076    #[test]
3077    fn source_validation_accepts_local_source() {
3078        let source = KanidmRestoreSource {
3079            local: Some(super::KanidmRestoreLocalSource {
3080                file_name: "backup.json".to_string(),
3081            }),
3082            backup_ref: None,
3083        };
3084        let restore = make_restore(source, None);
3085        assert!(validate_source(&restore).is_ok());
3086    }
3087
3088    #[test]
3089    fn source_validation_accepts_backup_ref_source() {
3090        let source = KanidmRestoreSource {
3091            local: None,
3092            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3093                name: "corp-idm-019c7c76".to_string(),
3094            }),
3095        };
3096        let restore = make_restore(source, None);
3097        assert!(validate_source(&restore).is_ok());
3098    }
3099
3100    #[test]
3101    fn source_validation_rejects_empty_backup_ref_name() {
3102        let source = KanidmRestoreSource {
3103            local: None,
3104            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3105                name: "".to_string(),
3106            }),
3107        };
3108        let restore = make_restore(source, None);
3109        assert!(validate_source(&restore).is_err());
3110    }
3111
3112    #[test]
3113    fn is_remote_source_true_for_backup_ref() {
3114        let source = KanidmRestoreSource {
3115            local: None,
3116            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3117                name: "backup-1".to_string(),
3118            }),
3119        };
3120        let restore = make_restore(source, None);
3121        assert!(is_remote_source(&restore));
3122    }
3123
3124    #[test]
3125    fn is_remote_source_false_for_local() {
3126        let source = KanidmRestoreSource {
3127            local: Some(super::KanidmRestoreLocalSource {
3128                file_name: "backup.json".to_string(),
3129            }),
3130            backup_ref: None,
3131        };
3132        let restore = make_restore(source, None);
3133        assert!(!is_remote_source(&restore));
3134    }
3135
3136    #[test]
3137    fn requires_safety_backup_default_true() {
3138        let source = KanidmRestoreSource {
3139            local: None,
3140            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3141                name: "backup-1".to_string(),
3142            }),
3143        };
3144        let restore = make_restore(source, None);
3145        assert!(requires_safety_backup(&restore));
3146    }
3147
3148    #[test]
3149    fn requires_safety_backup_false_when_skip() {
3150        let source = KanidmRestoreSource {
3151            local: None,
3152            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3153                name: "backup-1".to_string(),
3154            }),
3155        };
3156        let safety = SafetyBackupConfig {
3157            repository_ref: Some(SafetyBackupRepositoryRef {
3158                name: "offsite".to_string(),
3159            }),
3160            skip: true,
3161        };
3162        let restore = make_restore(source, Some(safety));
3163        assert!(!requires_safety_backup(&restore));
3164    }
3165
3166    #[test]
3167    fn break_glass_validation_rejects_skip_without_reason() {
3168        let source = KanidmRestoreSource {
3169            local: None,
3170            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3171                name: "backup-1".to_string(),
3172            }),
3173        };
3174        let safety = SafetyBackupConfig {
3175            repository_ref: Some(SafetyBackupRepositoryRef {
3176                name: "offsite".to_string(),
3177            }),
3178            skip: true,
3179        };
3180        let mut restore = make_restore(source, Some(safety));
3181        restore.metadata.annotations = Some(BTreeMap::from([(
3182            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3183            "commander@example.com".to_string(),
3184        )]));
3185        assert!(validate_safety_backup_config(&restore).is_err());
3186    }
3187
3188    #[test]
3189    fn break_glass_validation_rejects_skip_without_approver() {
3190        let source = KanidmRestoreSource {
3191            local: None,
3192            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3193                name: "backup-1".to_string(),
3194            }),
3195        };
3196        let safety = SafetyBackupConfig {
3197            repository_ref: Some(SafetyBackupRepositoryRef {
3198                name: "offsite".to_string(),
3199            }),
3200            skip: true,
3201        };
3202        let mut restore = make_restore(source, Some(safety));
3203        restore.metadata.annotations = Some(BTreeMap::from([(
3204            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3205            "PVC is unreadable".to_string(),
3206        )]));
3207        assert!(validate_safety_backup_config(&restore).is_err());
3208    }
3209
3210    #[test]
3211    fn break_glass_validation_accepts_valid_annotations() {
3212        let source = KanidmRestoreSource {
3213            local: None,
3214            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3215                name: "backup-1".to_string(),
3216            }),
3217        };
3218        let safety = SafetyBackupConfig {
3219            repository_ref: Some(SafetyBackupRepositoryRef {
3220                name: "offsite".to_string(),
3221            }),
3222            skip: true,
3223        };
3224        let mut restore = make_restore(source, Some(safety));
3225        restore.metadata.annotations = Some(BTreeMap::from([
3226            (
3227                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3228                "PVC is unreadable".to_string(),
3229            ),
3230            (
3231                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3232                "commander@example.com".to_string(),
3233            ),
3234        ]));
3235        assert!(validate_safety_backup_config(&restore).is_ok());
3236    }
3237
3238    #[test]
3239    fn disaster_recovery_requires_explicit_approval_annotations() {
3240        let source = KanidmRestoreSource {
3241            local: None,
3242            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3243                name: "backup-1".to_string(),
3244            }),
3245        };
3246        let mut restore = make_restore(source, None);
3247        restore.metadata.annotations = Some(BTreeMap::from([(
3248            DISASTER_RECOVERY_ANNOTATION.to_string(),
3249            "true".to_string(),
3250        )]));
3251        assert!(disaster_recovery_requested(&restore));
3252        assert!(validate_disaster_recovery_approval(&restore).is_err());
3253    }
3254
3255    #[test]
3256    fn disaster_recovery_accepts_reason_and_approver() {
3257        let source = KanidmRestoreSource {
3258            local: None,
3259            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3260                name: "backup-1".to_string(),
3261            }),
3262        };
3263        let mut restore = make_restore(source, None);
3264        restore.metadata.annotations = Some(BTreeMap::from([
3265            (
3266                DISASTER_RECOVERY_ANNOTATION.to_string(),
3267                "true".to_string(),
3268            ),
3269            (
3270                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3271                "original cluster lost".to_string(),
3272            ),
3273            (
3274                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3275                "incident-commander".to_string(),
3276            ),
3277        ]));
3278        assert!(disaster_recovery_requested(&restore));
3279        assert!(validate_disaster_recovery_approval(&restore).is_ok());
3280    }
3281
3282    #[test]
3283    fn remote_source_requires_repository_ref_when_not_skipping() {
3284        let source = KanidmRestoreSource {
3285            local: None,
3286            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3287                name: "backup-1".to_string(),
3288            }),
3289        };
3290        let safety = SafetyBackupConfig {
3291            repository_ref: None,
3292            skip: false,
3293        };
3294        let restore = make_restore(source, Some(safety));
3295        assert!(validate_safety_backup_config(&restore).is_err());
3296    }
3297
3298    #[test]
3299    fn remote_source_accepts_repository_ref() {
3300        let source = KanidmRestoreSource {
3301            local: None,
3302            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3303                name: "backup-1".to_string(),
3304            }),
3305        };
3306        let safety = SafetyBackupConfig {
3307            repository_ref: Some(SafetyBackupRepositoryRef {
3308                name: "offsite".to_string(),
3309            }),
3310            skip: false,
3311        };
3312        let restore = make_restore(source, Some(safety));
3313        assert!(validate_safety_backup_config(&restore).is_ok());
3314    }
3315
3316    #[test]
3317    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3318        let phases = vec![
3319            KanidmRestorePhase::Pending,
3320            KanidmRestorePhase::Validating,
3321            KanidmRestorePhase::Quiescing,
3322            KanidmRestorePhase::SafetyBackup,
3323            KanidmRestorePhase::PreparingSource,
3324            KanidmRestorePhase::RestoringPrimary,
3325            KanidmRestorePhase::Verifying,
3326            KanidmRestorePhase::RebuildingReplicas,
3327            KanidmRestorePhase::Resuming,
3328            KanidmRestorePhase::Completed,
3329        ];
3330        assert_eq!(phases.len(), 10);
3331        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3332        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3333    }
3334
3335    #[test]
3336    fn status_default_has_empty_phase_timestamps() {
3337        let status = KanidmRestoreStatus::default();
3338        assert!(status.phase_timestamps.is_empty());
3339    }
3340
3341    #[test]
3342    fn status_default_has_empty_original_replicas() {
3343        let status = KanidmRestoreStatus::default();
3344        assert!(status.original_replicas.is_empty());
3345    }
3346
3347    #[test]
3348    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3349        let mut target = super::super::crd::Kanidm::default();
3350        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3351            run_as_non_root: Some(false),
3352            run_as_user: Some(389),
3353            run_as_group: Some(389),
3354            fs_group: Some(389),
3355            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3356            supplemental_groups: Some(vec![390, 391]),
3357            ..Default::default()
3358        });
3359
3360        let ctx = super::source_job_pod_security_context(&target);
3361
3362        assert_eq!(ctx.run_as_non_root, Some(false));
3363        assert_eq!(ctx.run_as_user, Some(389));
3364        assert_eq!(ctx.run_as_group, Some(389));
3365        assert_eq!(ctx.fs_group, Some(389));
3366        assert_eq!(
3367            ctx.fs_group_change_policy.as_deref(),
3368            Some("OnRootMismatch")
3369        );
3370        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3371        assert_eq!(
3372            ctx.seccomp_profile
3373                .as_ref()
3374                .map(|profile| profile.type_.as_str()),
3375            Some("RuntimeDefault")
3376        );
3377    }
3378
3379    #[test]
3380    fn hardened_security_context_drops_all_capabilities() {
3381        let ctx = hardened_security_context();
3382        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3383        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3384        assert_eq!(ctx.run_as_non_root, Some(true));
3385        let caps = ctx.capabilities.unwrap();
3386        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3387        assert_eq!(
3388            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3389            Some("RuntimeDefault")
3390        );
3391    }
3392
3393    #[test]
3394    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3395        let ctx = kanidm_job_security_context();
3396        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3397        assert!(ctx.run_as_non_root.is_none());
3398        assert!(ctx.read_only_root_filesystem.is_none());
3399        let caps = ctx.capabilities.unwrap();
3400        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3401        assert_eq!(
3402            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3403            Some("RuntimeDefault")
3404        );
3405    }
3406
3407    #[test]
3408    fn safety_backup_job_name_is_deterministic() {
3409        let source = KanidmRestoreSource {
3410            local: Some(super::KanidmRestoreLocalSource {
3411                file_name: "backup.json".to_string(),
3412            }),
3413            backup_ref: None,
3414        };
3415        let restore = make_restore(source, None);
3416        let name1 = super::safety_backup_job_name(&restore);
3417        let name2 = super::safety_backup_job_name(&restore);
3418        assert_eq!(name1, name2);
3419        assert_eq!(name1, "test-restore-safety-backup");
3420    }
3421
3422    #[test]
3423    fn source_prep_job_name_is_deterministic() {
3424        let source = KanidmRestoreSource {
3425            local: None,
3426            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3427                name: "backup-1".to_string(),
3428            }),
3429        };
3430        let restore = make_restore(source, None);
3431        let name = super::source_prep_job_name(&restore);
3432        assert_eq!(name, "test-restore-source-prep");
3433    }
3434
3435    #[test]
3436    fn break_glass_condition_is_recorded_in_status() {
3437        let mut status = KanidmRestoreStatus::default();
3438        let condition = super::restore_condition(
3439            &status.conditions,
3440            CONDITION_BREAK_GLASS,
3441            CONDITION_TRUE,
3442            "BreakGlassOverride",
3443            "Safety backup skipped",
3444            Some(1),
3445        );
3446        status.conditions.push(condition);
3447        assert!(status.conditions.iter().any(|c| {
3448            c.type_ == CONDITION_BREAK_GLASS
3449                && c.status == CONDITION_TRUE
3450                && c.reason == "BreakGlassOverride"
3451        }));
3452    }
3453
3454    #[test]
3455    fn replica_count_entry_serialization() {
3456        let entry = ReplicaCountEntry {
3457            group: "primary".to_string(),
3458            replicas: 3,
3459        };
3460        let json = serde_json::to_string(&entry).unwrap();
3461        assert!(json.contains("\"group\":\"primary\""));
3462        assert!(json.contains("\"replicas\":3"));
3463    }
3464
3465    #[test]
3466    fn local_source_requires_repository_ref_when_not_skipping() {
3467        let source = KanidmRestoreSource {
3468            local: Some(super::KanidmRestoreLocalSource {
3469                file_name: "backup.json".to_string(),
3470            }),
3471            backup_ref: None,
3472        };
3473        let safety = SafetyBackupConfig {
3474            repository_ref: None,
3475            skip: false,
3476        };
3477        let restore = make_restore(source, Some(safety));
3478        assert!(validate_safety_backup_config(&restore).is_err());
3479    }
3480
3481    #[test]
3482    fn local_source_accepts_repository_ref_when_not_skipping() {
3483        let source = KanidmRestoreSource {
3484            local: Some(super::KanidmRestoreLocalSource {
3485                file_name: "backup.json".to_string(),
3486            }),
3487            backup_ref: None,
3488        };
3489        let safety = SafetyBackupConfig {
3490            repository_ref: Some(SafetyBackupRepositoryRef {
3491                name: "offsite".to_string(),
3492            }),
3493            skip: false,
3494        };
3495        let restore = make_restore(source, Some(safety));
3496        assert!(validate_safety_backup_config(&restore).is_ok());
3497    }
3498
3499    #[test]
3500    fn skip_without_break_glass_annotations_is_rejected() {
3501        let source = KanidmRestoreSource {
3502            local: Some(super::KanidmRestoreLocalSource {
3503                file_name: "backup.json".to_string(),
3504            }),
3505            backup_ref: None,
3506        };
3507        let safety = SafetyBackupConfig {
3508            repository_ref: None,
3509            skip: true,
3510        };
3511        let restore = make_restore(source, Some(safety));
3512        assert!(validate_safety_backup_config(&restore).is_err());
3513    }
3514
3515    #[test]
3516    fn has_accepted_condition_true() {
3517        let conditions = vec![Condition {
3518            type_: "Ready".to_string(),
3519            status: "True".to_string(),
3520            reason: "Accepted".to_string(),
3521            message: String::new(),
3522            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3523                k8s_openapi::jiff::Timestamp::now(),
3524            ),
3525            observed_generation: None,
3526        }];
3527        assert!(has_accepted_condition(&conditions));
3528    }
3529
3530    #[test]
3531    fn has_accepted_condition_false_when_not_accepted_reason() {
3532        let conditions = vec![Condition {
3533            type_: "Ready".to_string(),
3534            status: "True".to_string(),
3535            reason: "SomeOtherReason".to_string(),
3536            message: String::new(),
3537            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3538                k8s_openapi::jiff::Timestamp::now(),
3539            ),
3540            observed_generation: None,
3541        }];
3542        assert!(!has_accepted_condition(&conditions));
3543    }
3544
3545    #[test]
3546    fn has_accepted_condition_false_when_status_false() {
3547        let conditions = vec![Condition {
3548            type_: "Ready".to_string(),
3549            status: "False".to_string(),
3550            reason: "Accepted".to_string(),
3551            message: String::new(),
3552            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3553                k8s_openapi::jiff::Timestamp::now(),
3554            ),
3555            observed_generation: None,
3556        }];
3557        assert!(!has_accepted_condition(&conditions));
3558    }
3559
3560    #[test]
3561    fn has_accepted_condition_empty() {
3562        assert!(!has_accepted_condition(&[]));
3563    }
3564
3565    #[test]
3566    fn safety_backup_upload_operation_doc_is_valid() {
3567        let source = KanidmRestoreSource {
3568            local: None,
3569            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3570                name: "backup-1".to_string(),
3571            }),
3572        };
3573        let mut restore = make_restore(source, None);
3574        restore.metadata.namespace = Some("test-ns".to_string());
3575        let target = super::super::crd::Kanidm::default();
3576        let doc_str = super::build_safety_upload_operation_doc(
3577            &restore,
3578            &target,
3579            "test-backup-id",
3580            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3581            "my-bucket",
3582            "prod",
3583            "https://s3.example.com",
3584            "us-east-1",
3585            false,
3586            false,
3587            None,
3588            None,
3589        )
3590        .unwrap();
3591        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3592        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3593        assert_eq!(parsed["kind"], "OperationDocument");
3594        assert_eq!(parsed["operation"], "upload");
3595        assert_eq!(parsed["bucket"], "my-bucket");
3596        assert_eq!(parsed["prefix"], "prod");
3597        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3598        assert_eq!(parsed["region"], "us-east-1");
3599        assert_eq!(parsed["backupId"], "test-backup-id");
3600        assert_eq!(parsed["consistency"], "kanidm-offline");
3601        assert_eq!(parsed["reason"], "restore-safety");
3602        assert_eq!(
3603            parsed["payloadPath"],
3604            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3605        );
3606        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3607    }
3608
3609    #[test]
3610    fn download_operation_doc_uses_real_repository_data() {
3611        let source = KanidmRestoreSource {
3612            local: None,
3613            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3614                name: "backup-1".to_string(),
3615            }),
3616        };
3617        let _restore = make_restore(source, None);
3618        let target = super::super::crd::Kanidm::default();
3619        let doc_str = super::build_download_operation_doc(
3620            &target,
3621            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3622            "019c7c76-f423-7a12-8f41-2bea7588a303",
3623            "source-uid-456",
3624            "real-bucket",
3625            "real-prefix",
3626            "https://real-endpoint.com",
3627            "eu-west-1",
3628            true,
3629            false,
3630            None,
3631            None,
3632        );
3633        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3634        assert_eq!(parsed["bucket"], "real-bucket");
3635        assert_eq!(parsed["prefix"], "real-prefix");
3636        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3637        assert_eq!(parsed["region"], "eu-west-1");
3638        assert_eq!(parsed["forcePathStyle"], true);
3639        assert_eq!(
3640            parsed["manifestKey"],
3641            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3642        );
3643        assert_eq!(
3644            parsed["expectedBackupId"],
3645            "019c7c76-f423-7a12-8f41-2bea7588a303"
3646        );
3647        assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3648    }
3649
3650    #[test]
3651    fn safety_backup_ref_is_set_after_verification() {
3652        let status = KanidmRestoreStatus {
3653            phase: KanidmRestorePhase::SafetyBackup,
3654            safety_backup_job_name: Some("test-safety-backup".to_string()),
3655            safety_backup_ref: Some("safety-test-restore".to_string()),
3656            safety_backup_expected_backup_id: Some(
3657                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3658            ),
3659            safety_backup_manifest_key: Some(
3660                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3661            ),
3662            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3663            ..Default::default()
3664        };
3665        assert!(status.safety_backup_ref.is_some());
3666        assert!(
3667            status
3668                .safety_backup_ref
3669                .as_ref()
3670                .unwrap()
3671                .starts_with("safety-")
3672        );
3673        assert!(status.safety_backup_expected_backup_id.is_some());
3674        assert!(status.safety_backup_manifest_key.is_some());
3675        assert!(status.safety_backup_payload_sha256.is_some());
3676    }
3677
3678    #[test]
3679    fn database_mutation_not_started_without_safety_backup_ref() {
3680        let status = KanidmRestoreStatus {
3681            phase: KanidmRestorePhase::PreparingSource,
3682            safety_backup_ref: None,
3683            database_mutation_started: false,
3684            ..Default::default()
3685        };
3686        assert!(!status.database_mutation_started);
3687        assert!(status.safety_backup_ref.is_none());
3688    }
3689
3690    #[test]
3691    fn pre_boundary_failure_does_not_set_mutation_started() {
3692        let status = KanidmRestoreStatus {
3693            phase: KanidmRestorePhase::Failed,
3694            message: Some(
3695                "safety backup job failed; target restored to original state".to_string(),
3696            ),
3697            database_mutation_started: false,
3698            ..Default::default()
3699        };
3700        assert!(!status.database_mutation_started);
3701    }
3702
3703    #[test]
3704    fn shared_volume_constants_are_distinct() {
3705        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3706        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3707    }
3708
3709    #[test]
3710    fn validate_backup_compatibility_matching_versions_passes() {
3711        let source = KanidmRestoreSource {
3712            local: None,
3713            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3714                name: "backup-1".to_string(),
3715            }),
3716        };
3717        let restore = make_restore(source, None);
3718        let target = kanidm_with_version("1.10.4");
3719        let backup = kaniop_backup_core::crd::KanidmBackup {
3720            metadata: kube::api::ObjectMeta::default(),
3721            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3722                backup_id: "id".to_string(),
3723                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3724                    name: "k".to_string(),
3725                    uid: "uid".to_string(),
3726                },
3727                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3728                    name: "repo".to_string(),
3729                },
3730                manifest_key: "key".to_string(),
3731            },
3732            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3733                kanidm_version: Some("1.10.4".to_string()),
3734                ..Default::default()
3735            }),
3736        };
3737        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3738    }
3739
3740    #[test]
3741    fn validate_backup_compatibility_version_mismatch_fails() {
3742        let source = KanidmRestoreSource {
3743            local: None,
3744            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3745                name: "backup-1".to_string(),
3746            }),
3747        };
3748        let restore = make_restore(source, None);
3749        let target = kanidm_with_version("1.11.0");
3750        let backup = kaniop_backup_core::crd::KanidmBackup {
3751            metadata: kube::api::ObjectMeta::default(),
3752            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3753                backup_id: "id".to_string(),
3754                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3755                    name: "k".to_string(),
3756                    uid: "uid".to_string(),
3757                },
3758                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3759                    name: "repo".to_string(),
3760                },
3761                manifest_key: "key".to_string(),
3762            },
3763            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3764                kanidm_version: Some("1.10.4".to_string()),
3765                ..Default::default()
3766            }),
3767        };
3768        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3769    }
3770
3771    #[test]
3772    fn validate_backup_compatibility_no_backup_version_passes() {
3773        let source = KanidmRestoreSource {
3774            local: None,
3775            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3776                name: "backup-1".to_string(),
3777            }),
3778        };
3779        let restore = make_restore(source, None);
3780        let target = super::super::crd::Kanidm::default();
3781        let backup = kaniop_backup_core::crd::KanidmBackup {
3782            metadata: kube::api::ObjectMeta::default(),
3783            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3784                backup_id: "id".to_string(),
3785                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3786                    name: "k".to_string(),
3787                    uid: "uid".to_string(),
3788                },
3789                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3790                    name: "repo".to_string(),
3791                },
3792                manifest_key: "key".to_string(),
3793            },
3794            status: None,
3795        };
3796        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3797    }
3798
3799    #[test]
3800    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3801        let source = KanidmRestoreSource {
3802            local: None,
3803            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3804                name: "backup-1".to_string(),
3805            }),
3806        };
3807        let mut restore = make_restore(source, None);
3808        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3809        let target = super::super::crd::Kanidm::default();
3810        let backup = kaniop_backup_core::crd::KanidmBackup {
3811            metadata: kube::api::ObjectMeta::default(),
3812            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3813                backup_id: "id".to_string(),
3814                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3815                    name: "k".to_string(),
3816                    uid: "uid".to_string(),
3817                },
3818                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3819                    name: "repo".to_string(),
3820                },
3821                manifest_key: "key".to_string(),
3822            },
3823            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3824                image_digest: Some("sha256:originaldigest".to_string()),
3825                ..Default::default()
3826            }),
3827        };
3828        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3829    }
3830
3831    #[test]
3832    fn validate_backup_compatibility_image_digest_match_passes() {
3833        let source = KanidmRestoreSource {
3834            local: None,
3835            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3836                name: "backup-1".to_string(),
3837            }),
3838        };
3839        let mut restore = make_restore(source, None);
3840        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3841        let target = super::super::crd::Kanidm::default();
3842        let backup = kaniop_backup_core::crd::KanidmBackup {
3843            metadata: kube::api::ObjectMeta::default(),
3844            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3845                backup_id: "id".to_string(),
3846                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3847                    name: "k".to_string(),
3848                    uid: "uid".to_string(),
3849                },
3850                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3851                    name: "repo".to_string(),
3852                },
3853                manifest_key: "key".to_string(),
3854            },
3855            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3856                image_digest: Some("sha256:abc123".to_string()),
3857                ..Default::default()
3858            }),
3859        };
3860        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3861    }
3862
3863    #[test]
3864    fn safety_backup_id_is_deterministic() {
3865        let source = KanidmRestoreSource {
3866            local: None,
3867            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3868                name: "backup-1".to_string(),
3869            }),
3870        };
3871        let restore = make_restore(source, None);
3872        let id1 = super::compute_safety_backup_id(&restore).unwrap();
3873        let id2 = super::compute_safety_backup_id(&restore).unwrap();
3874        assert_eq!(id1, id2);
3875        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3876    }
3877
3878    #[test]
3879    fn safety_manifest_key_is_deterministic() {
3880        let source = KanidmRestoreSource {
3881            local: None,
3882            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3883                name: "backup-1".to_string(),
3884            }),
3885        };
3886        let restore = make_restore(source, None);
3887        let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3888        let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3889        assert_eq!(key1, key2);
3890        assert!(key1.contains("manifest.json"));
3891        assert!(key1.starts_with("v1/tenants/"));
3892    }
3893
3894    #[test]
3895    fn safety_backup_id_survives_restart() {
3896        let source = KanidmRestoreSource {
3897            local: None,
3898            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3899                name: "backup-1".to_string(),
3900            }),
3901        };
3902        let mut restore = make_restore(source, None);
3903        let original_id = super::compute_safety_backup_id(&restore).unwrap();
3904        restore.status = Some(KanidmRestoreStatus {
3905            phase: KanidmRestorePhase::SafetyBackup,
3906            safety_backup_expected_backup_id: Some(original_id.clone()),
3907            ..Default::default()
3908        });
3909        let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3910        assert_eq!(original_id, recomputed_id);
3911    }
3912
3913    #[test]
3914    fn safety_backup_id_derived_from_uid_not_name() {
3915        let source = KanidmRestoreSource {
3916            local: None,
3917            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3918                name: "backup-1".to_string(),
3919            }),
3920        };
3921        let restore1 = make_restore(source.clone(), None);
3922        let mut restore2 = make_restore(source, None);
3923        restore2.metadata.name = Some("different-name-same-uid".to_string());
3924        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3925        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3926        assert_eq!(id1, id2);
3927    }
3928
3929    #[test]
3930    fn safety_backup_id_differs_for_different_uids() {
3931        let source = KanidmRestoreSource {
3932            local: None,
3933            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3934                name: "backup-1".to_string(),
3935            }),
3936        };
3937        let restore1 = make_restore(source.clone(), None);
3938        let mut restore2 = make_restore(source, None);
3939        restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3940        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3941        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3942        assert_ne!(id1, id2);
3943    }
3944
3945    #[test]
3946    fn safety_backup_id_errors_without_uid() {
3947        let source = KanidmRestoreSource {
3948            local: None,
3949            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3950                name: "backup-1".to_string(),
3951            }),
3952        };
3953        let mut restore = make_restore(source, None);
3954        restore.metadata.uid = None;
3955        assert!(super::compute_safety_backup_id(&restore).is_err());
3956    }
3957
3958    #[test]
3959    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3960        let source = KanidmRestoreSource {
3961            local: None,
3962            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3963                name: "backup-1".to_string(),
3964            }),
3965        };
3966        let restore = make_restore(source, None);
3967        assert!(super::requires_safety_backup(&restore));
3968        let status_without_ref = KanidmRestoreStatus {
3969            phase: KanidmRestorePhase::RestoringPrimary,
3970            safety_backup_ref: None,
3971            database_mutation_started: false,
3972            ..Default::default()
3973        };
3974        assert!(status_without_ref.safety_backup_ref.is_none());
3975        assert!(!status_without_ref.database_mutation_started);
3976    }
3977
3978    #[test]
3979    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3980        let status_with_ref = KanidmRestoreStatus {
3981            phase: KanidmRestorePhase::RestoringPrimary,
3982            safety_backup_ref: Some("safety-test-restore".to_string()),
3983            safety_backup_manifest_key: Some(
3984                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3985            ),
3986            safety_backup_payload_sha256: Some("abc123".to_string()),
3987            database_mutation_started: false,
3988            ..Default::default()
3989        };
3990        assert!(status_with_ref.safety_backup_ref.is_some());
3991        assert!(!status_with_ref.database_mutation_started);
3992    }
3993
3994    #[test]
3995    fn result_document_verification_rejects_missing_backup_id() {
3996        let result_json = r#"{
3997            "apiVersion": "backup.kaniop.rs/v1alpha1",
3998            "kind": "ResultDocument",
3999            "operation": "upload",
4000            "success": true,
4001            "exitCode": "success",
4002            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4003            "payloadSha256": "abc123"
4004        }"#;
4005        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4006        assert!(result.backup_id.is_none());
4007    }
4008
4009    #[test]
4010    fn result_document_verification_rejects_missing_manifest_key() {
4011        let result_json = r#"{
4012            "apiVersion": "backup.kaniop.rs/v1alpha1",
4013            "kind": "ResultDocument",
4014            "operation": "upload",
4015            "success": true,
4016            "exitCode": "success",
4017            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4018            "payloadSha256": "abc123"
4019        }"#;
4020        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021        assert!(result.manifest_key.is_none());
4022    }
4023
4024    #[test]
4025    fn result_document_verification_rejects_missing_payload_sha256() {
4026        let result_json = r#"{
4027            "apiVersion": "backup.kaniop.rs/v1alpha1",
4028            "kind": "ResultDocument",
4029            "operation": "upload",
4030            "success": true,
4031            "exitCode": "success",
4032            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4033            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4034        }"#;
4035        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4036        assert!(result.payload_sha256.is_none());
4037    }
4038
4039    #[test]
4040    fn result_document_verification_rejects_failure_exit_code() {
4041        let result_json = r#"{
4042            "apiVersion": "backup.kaniop.rs/v1alpha1",
4043            "kind": "ResultDocument",
4044            "operation": "upload",
4045            "success": false,
4046            "exitCode": "integrity",
4047            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4048            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4049            "payloadSha256": "abc123"
4050        }"#;
4051        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4052        assert!(!result.success);
4053        assert_eq!(
4054            result.exit_code,
4055            kaniop_backup_core::result::ExitCode::Integrity
4056        );
4057    }
4058
4059    #[test]
4060    fn result_document_verification_rejects_wrong_operation() {
4061        let result_json = r#"{
4062            "apiVersion": "backup.kaniop.rs/v1alpha1",
4063            "kind": "ResultDocument",
4064            "operation": "download",
4065            "success": true,
4066            "exitCode": "success",
4067            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4068            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4069            "payloadSha256": "abc123"
4070        }"#;
4071        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4072        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4073    }
4074
4075    #[test]
4076    fn result_document_verification_accepts_valid_document() {
4077        let result_json = r#"{
4078            "apiVersion": "backup.kaniop.rs/v1alpha1",
4079            "kind": "ResultDocument",
4080            "operation": "upload",
4081            "success": true,
4082            "exitCode": "success",
4083            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4084            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4085            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4086        }"#;
4087        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4088        assert!(result.success);
4089        assert_eq!(
4090            result.exit_code,
4091            kaniop_backup_core::result::ExitCode::Success
4092        );
4093        assert_eq!(result.operation, "upload");
4094        assert!(result.backup_id.is_some());
4095        assert!(result.manifest_key.is_some());
4096        assert!(result.payload_sha256.is_some());
4097    }
4098
4099    #[test]
4100    fn termination_message_path_constant_is_valid() {
4101        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4102        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4103    }
4104
4105    #[test]
4106    fn status_tracks_safety_backup_evidence() {
4107        let status = KanidmRestoreStatus {
4108            phase: KanidmRestorePhase::PreparingSource,
4109            safety_backup_ref: Some("safety-test-restore".to_string()),
4110            safety_backup_expected_backup_id: Some(
4111                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4112            ),
4113            safety_backup_manifest_key: Some(
4114                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4115            ),
4116            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4117            ..Default::default()
4118        };
4119        assert!(status.safety_backup_ref.is_some());
4120        assert!(status.safety_backup_expected_backup_id.is_some());
4121        assert!(status.safety_backup_manifest_key.is_some());
4122        assert!(status.safety_backup_payload_sha256.is_some());
4123    }
4124
4125    #[test]
4126    fn fail_closed_semantics_preserved_on_result_parse_failure() {
4127        let invalid_json = "not valid json";
4128        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4129        assert!(result.is_err());
4130    }
4131
4132    #[test]
4133    fn fail_closed_semantics_preserved_on_oversized_result() {
4134        let oversized_json = format!(
4135            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4136            kaniop_backup_core::result::RESULT_DOC_VERSION,
4137            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4138        );
4139        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4140        assert!(result.is_err());
4141    }
4142
4143    #[test]
4144    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4145        let source = KanidmRestoreSource {
4146            local: None,
4147            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4148                name: "backup-1".to_string(),
4149            }),
4150        };
4151        let mut restore = make_restore(source, None);
4152        restore.metadata.namespace = Some("test-ns".to_string());
4153        let target = super::super::crd::Kanidm::default();
4154        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4155        let doc_str = super::build_safety_upload_operation_doc(
4156            &restore,
4157            &target,
4158            "test-backup-id",
4159            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4160            "my-bucket",
4161            "prod",
4162            "https://s3.example.com",
4163            "us-east-1",
4164            false,
4165            false,
4166            Some(&ca_path),
4167            None,
4168        )
4169        .unwrap();
4170        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4171        assert_eq!(parsed["caBundlePath"], ca_path);
4172    }
4173
4174    #[test]
4175    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4176        let source = KanidmRestoreSource {
4177            local: None,
4178            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4179                name: "backup-1".to_string(),
4180            }),
4181        };
4182        let mut restore = make_restore(source, None);
4183        restore.metadata.namespace = Some("test-ns".to_string());
4184        let target = super::super::crd::Kanidm::default();
4185        let doc_str = super::build_safety_upload_operation_doc(
4186            &restore,
4187            &target,
4188            "test-backup-id",
4189            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4190            "my-bucket",
4191            "prod",
4192            "https://s3.example.com",
4193            "us-east-1",
4194            false,
4195            false,
4196            None,
4197            None,
4198        )
4199        .unwrap();
4200        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4201        assert!(parsed["caBundlePath"].is_null());
4202    }
4203
4204    #[test]
4205    fn download_operation_doc_includes_ca_bundle_path_when_set() {
4206        let source = KanidmRestoreSource {
4207            local: None,
4208            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4209                name: "backup-1".to_string(),
4210            }),
4211        };
4212        let _restore = make_restore(source, None);
4213        let target = super::super::crd::Kanidm::default();
4214        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4215        let doc_str = super::build_download_operation_doc(
4216            &target,
4217            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4218            "019c7c76-f423-7a12-8f41-2bea7588a303",
4219            "source-uid-456",
4220            "real-bucket",
4221            "real-prefix",
4222            "https://real-endpoint.com",
4223            "eu-west-1",
4224            true,
4225            false,
4226            Some(&ca_path),
4227            None,
4228        );
4229        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4230        assert_eq!(parsed["caBundlePath"], ca_path);
4231    }
4232
4233    #[test]
4234    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4235        let source = KanidmRestoreSource {
4236            local: None,
4237            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4238                name: "backup-1".to_string(),
4239            }),
4240        };
4241        let _restore = make_restore(source, None);
4242        let target = super::super::crd::Kanidm::default();
4243        let doc_str = super::build_download_operation_doc(
4244            &target,
4245            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4246            "019c7c76-f423-7a12-8f41-2bea7588a303",
4247            "source-uid-456",
4248            "real-bucket",
4249            "real-prefix",
4250            "https://real-endpoint.com",
4251            "eu-west-1",
4252            true,
4253            false,
4254            None,
4255            None,
4256        );
4257        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4258        assert!(parsed["caBundlePath"].is_null());
4259    }
4260
4261    #[test]
4262    fn source_prep_result_operation_is_download() {
4263        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4264        assert_ne!(
4265            super::SOURCE_PREP_RESULT_OPERATION,
4266            super::SAFETY_BACKUP_RESULT_OPERATION
4267        );
4268    }
4269
4270    #[test]
4271    fn download_result_document_verification_accepts_valid_document() {
4272        let result_json = r#"{
4273            "apiVersion": "backup.kaniop.rs/v1alpha1",
4274            "kind": "ResultDocument",
4275            "operation": "download",
4276            "success": true,
4277            "exitCode": "success",
4278            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4279            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4280            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4281        }"#;
4282        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4283        assert!(result.success);
4284        assert_eq!(
4285            result.exit_code,
4286            kaniop_backup_core::result::ExitCode::Success
4287        );
4288        assert_eq!(result.operation, "download");
4289        assert_eq!(
4290            result.backup_id.as_deref(),
4291            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4292        );
4293        assert!(result.manifest_key.is_some());
4294        assert!(result.payload_sha256.is_some());
4295    }
4296
4297    #[test]
4298    fn download_result_document_rejects_wrong_operation() {
4299        let result_json = r#"{
4300            "apiVersion": "backup.kaniop.rs/v1alpha1",
4301            "kind": "ResultDocument",
4302            "operation": "upload",
4303            "success": true,
4304            "exitCode": "success",
4305            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4306            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4307            "payloadSha256": "abc123"
4308        }"#;
4309        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4310        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4311    }
4312
4313    #[test]
4314    fn download_result_document_rejects_failure_success_field() {
4315        let result_json = r#"{
4316            "apiVersion": "backup.kaniop.rs/v1alpha1",
4317            "kind": "ResultDocument",
4318            "operation": "download",
4319            "success": false,
4320            "exitCode": "retryable",
4321            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4322            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4323            "payloadSha256": "abc123"
4324        }"#;
4325        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4326        assert!(!result.success);
4327        assert_ne!(
4328            result.exit_code,
4329            kaniop_backup_core::result::ExitCode::Success
4330        );
4331    }
4332
4333    #[test]
4334    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4335        let status = KanidmRestoreStatus {
4336            phase: KanidmRestorePhase::PreparingSource,
4337            database_mutation_started: false,
4338            ..Default::default()
4339        };
4340        assert!(!status.database_mutation_started);
4341        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4342    }
4343
4344    #[test]
4345    fn backup_job_volume_size_default_is_10gi() {
4346        let qty = crate::controller::backup_job_volume_size();
4347        assert_eq!(qty, Quantity("10Gi".to_string()));
4348    }
4349
4350    #[test]
4351    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4352        use kaniop_backup_core::auth::build_encryption_env_vars;
4353        use kaniop_backup_core::crd::SecretRef;
4354
4355        let key_ref = SecretRef {
4356            name: "kek-secret".to_string(),
4357        };
4358        let env_vars = build_encryption_env_vars(Some(&key_ref));
4359        assert_eq!(env_vars.len(), 1);
4360        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4361        let secret_ref = env_vars[0]
4362            .value_from
4363            .as_ref()
4364            .unwrap()
4365            .secret_key_ref
4366            .as_ref()
4367            .unwrap();
4368        assert_eq!(secret_ref.name, "kek-secret");
4369        assert_eq!(secret_ref.key, "encryption-key");
4370    }
4371
4372    #[test]
4373    fn encryption_env_vars_absent_when_no_key_ref() {
4374        use kaniop_backup_core::auth::build_encryption_env_vars;
4375
4376        let env_vars = build_encryption_env_vars(None);
4377        assert!(
4378            env_vars.is_empty(),
4379            "encryption env vars must be empty when key_ref is None"
4380        );
4381    }
4382
4383    #[test]
4384    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4385        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4386
4387        let source = KanidmRestoreSource {
4388            local: None,
4389            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4390                name: "backup-1".to_string(),
4391            }),
4392        };
4393        let mut restore = make_restore(source, None);
4394        restore.metadata.namespace = Some("test-ns".to_string());
4395        let target = super::super::crd::Kanidm::default();
4396        let encryption = RepositoryEncryption {
4397            mode: EncryptionMode::ClientSide,
4398            key_id: None,
4399            key_ref: Some(SecretRef {
4400                name: "kek-secret".to_string(),
4401            }),
4402        };
4403        let doc_str = super::build_safety_upload_operation_doc(
4404            &restore,
4405            &target,
4406            "test-backup-id",
4407            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4408            "my-bucket",
4409            "prod",
4410            "https://s3.example.com",
4411            "us-east-1",
4412            false,
4413            false,
4414            None,
4415            Some(&encryption),
4416        )
4417        .unwrap();
4418        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4419        assert_eq!(parsed["encryptionMode"], "clientSide");
4420        assert!(parsed["encryptionKeyId"].is_null());
4421    }
4422
4423    #[test]
4424    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4425        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4426
4427        let source = KanidmRestoreSource {
4428            local: None,
4429            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4430                name: "backup-1".to_string(),
4431            }),
4432        };
4433        let _restore = make_restore(source, None);
4434        let target = super::super::crd::Kanidm::default();
4435        let encryption = RepositoryEncryption {
4436            mode: EncryptionMode::ProviderKms,
4437            key_id: Some("alias/kaniop-backups".to_string()),
4438            key_ref: None,
4439        };
4440        let doc_str = super::build_download_operation_doc(
4441            &target,
4442            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4443            "019c7c76-f423-7a12-8f41-2bea7588a303",
4444            "source-uid-456",
4445            "real-bucket",
4446            "real-prefix",
4447            "https://real-endpoint.com",
4448            "eu-west-1",
4449            true,
4450            false,
4451            None,
4452            Some(&encryption),
4453        );
4454        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4455        assert_eq!(parsed["encryptionMode"], "providerKms");
4456        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4457    }
4458
4459    #[test]
4460    fn post_mutation_failure_records_job_names() {
4461        let existing = KanidmRestoreStatus {
4462            phase: KanidmRestorePhase::RestoringPrimary,
4463            database_mutation_started: true,
4464            ..Default::default()
4465        };
4466
4467        let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4468        assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4469        assert_eq!(
4470            restore_failed.restore_job_name.as_deref(),
4471            Some("my-restore-job")
4472        );
4473        assert!(restore_failed.verify_job_name.is_none());
4474        assert!(restore_failed.database_mutation_started);
4475        assert_eq!(
4476            restore_failed.message.as_deref(),
4477            Some("database restore job failed")
4478        );
4479
4480        let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4481        assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4482        assert_eq!(
4483            verify_failed.verify_job_name.as_deref(),
4484            Some("my-verify-job")
4485        );
4486        assert!(verify_failed.restore_job_name.is_none());
4487        assert!(verify_failed.database_mutation_started);
4488        assert_eq!(
4489            verify_failed.message.as_deref(),
4490            Some("database verification failed")
4491        );
4492    }
4493
4494    #[test]
4495    fn restore_and_verify_job_names_are_deterministic() {
4496        let source = KanidmRestoreSource {
4497            local: Some(super::KanidmRestoreLocalSource {
4498                file_name: "backup.json".to_string(),
4499            }),
4500            backup_ref: None,
4501        };
4502        let restore = make_restore(source, None);
4503        assert_eq!(restore_job_name(&restore), "test-restore-restore");
4504        assert_eq!(verify_job_name(&restore), "test-restore-verify");
4505    }
4506}