Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{
12    KanidmBackup, KanidmBackupPhase, KanidmBackupRepository, KanidmBackupRepositoryStatus,
13};
14use kaniop_backup_core::image::data_mover_image;
15use kaniop_backup_core::operation::{
16    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
17};
18use kaniop_backup_core::result::{ExitCode, parse_result_document};
19use kaniop_k8s_util::error::{Error, Result};
20
21use std::collections::BTreeSet;
22use std::sync::Arc;
23use std::time::Duration;
24
25use futures::StreamExt;
26use k8s_openapi::api::apps::v1::StatefulSet;
27use k8s_openapi::api::batch::v1::{Job, JobSpec};
28use k8s_openapi::api::core::v1::{
29    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
30    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
31    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
32};
33use k8s_openapi::api::storage::v1::VolumeAttachment;
34use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
35use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
36use k8s_openapi::jiff::Timestamp;
37use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
38use kube::runtime::controller::{Action, Controller};
39use kube::runtime::events::{Event, EventType, Recorder, Reporter};
40use kube::runtime::finalizer::{Event as Finalizer, finalizer};
41use kube::runtime::watcher;
42use kube::{Api, Client, CustomResource, Resource, ResourceExt};
43use opentelemetry::metrics::{Counter, Gauge, Histogram};
44use opentelemetry::{KeyValue, global};
45#[cfg(feature = "schemars")]
46use schemars::JsonSchema;
47use serde::{Deserialize, Serialize};
48use serde_json::json;
49use tracing::{debug, error, info, warn};
50
51pub const CONTROLLER_ID: &str = "kanidm-restore";
52pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
53const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
54const DATA_VOLUME: &str = "kanidm-data";
55const SHARED_VOLUME: &str = "safety-backup-shared";
56const DATA_PATH: &str = "/data";
57const TLS_VOLUME: &str = "kanidm-certs";
58const TLS_PATH: &str = "/etc/kanidm/tls";
59const BACKUP_PATH: &str = "/data";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
71const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
72const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
73const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
74const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
75const SOURCE_PREP_RESULT_OPERATION: &str = "download";
76
77#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
78#[cfg_attr(feature = "schemars", derive(JsonSchema))]
79#[cfg_attr(
80    feature = "schemars",
81    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
82)]
83#[kube(
84    category = "kaniop",
85    group = "kaniop.rs",
86    version = "v1beta1",
87    kind = "KanidmRestore",
88    plural = "kanidmrestores",
89    singular = "kanidmrestore",
90    shortname = "idmrestore",
91    namespaced,
92    status = "KanidmRestoreStatus",
93    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
94    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
95    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
96)]
97#[serde(rename_all = "camelCase")]
98pub struct KanidmRestoreSpec {
99    pub target_ref: KanidmRestoreTargetRef,
100    pub source: KanidmRestoreSource,
101    pub restore_image: String,
102    #[serde(skip_serializing_if = "Option::is_none")]
103    pub safety_backup: Option<SafetyBackupConfig>,
104}
105
106#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
107#[cfg_attr(feature = "schemars", derive(JsonSchema))]
108#[serde(rename_all = "camelCase")]
109pub struct KanidmRestoreTargetRef {
110    pub name: String,
111    pub uid: String,
112}
113
114#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
115#[cfg_attr(feature = "schemars", derive(JsonSchema))]
116#[cfg_attr(
117    feature = "schemars",
118    schemars(extend("x-kubernetes-validations" = [
119        {
120            "message": "exactly one of local or backupRef must be set",
121            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
122        }
123    ]))
124)]
125#[serde(rename_all = "camelCase")]
126pub struct KanidmRestoreSource {
127    /// Local file source. Mutually exclusive with `backupRef`.
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub local: Option<KanidmRestoreLocalSource>,
130    /// Remote cataloged backup reference. Mutually exclusive with `local`.
131    #[serde(skip_serializing_if = "Option::is_none")]
132    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
133}
134
135#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
136#[cfg_attr(feature = "schemars", derive(JsonSchema))]
137#[serde(rename_all = "camelCase")]
138pub struct KanidmRestoreLocalSource {
139    /// Basename of a backup below /data. Paths and traversal are rejected.
140    pub file_name: String,
141}
142
143#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
144#[cfg_attr(feature = "schemars", derive(JsonSchema))]
145#[serde(rename_all = "camelCase")]
146pub struct KanidmRestoreBackupRefSource {
147    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
148    pub name: String,
149}
150
151#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
152#[cfg_attr(feature = "schemars", derive(JsonSchema))]
153#[serde(rename_all = "camelCase")]
154pub struct SafetyBackupConfig {
155    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub repository_ref: Option<SafetyBackupRepositoryRef>,
158    /// Skip the mandatory safety backup. Requires break-glass annotations.
159    #[serde(default)]
160    pub skip: bool,
161}
162
163#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
164#[cfg_attr(feature = "schemars", derive(JsonSchema))]
165#[serde(rename_all = "camelCase")]
166pub struct SafetyBackupRepositoryRef {
167    pub name: String,
168}
169
170#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
171#[cfg_attr(feature = "schemars", derive(JsonSchema))]
172pub enum KanidmRestorePhase {
173    #[default]
174    Pending,
175    Validating,
176    Quiescing,
177    SafetyBackup,
178    PreparingSource,
179    RestoringPrimary,
180    Verifying,
181    RebuildingReplicas,
182    Resuming,
183    Completed,
184    Failed,
185}
186
187#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
188#[cfg_attr(feature = "schemars", derive(JsonSchema))]
189#[serde(rename_all = "camelCase")]
190pub struct KanidmRestoreStatus {
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_generation: Option<i64>,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub observed_target_uid: Option<String>,
195    #[serde(default)]
196    pub phase: KanidmRestorePhase,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub restore_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub verify_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub safety_backup_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub source_prep_job_name: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_ref: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_expected_backup_id: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_manifest_key: Option<String>,
211    #[serde(skip_serializing_if = "Option::is_none")]
212    pub safety_backup_payload_sha256: Option<String>,
213    #[serde(default)]
214    pub replicas_cleared: bool,
215    #[serde(default)]
216    pub certificates_cleared: bool,
217    #[serde(default)]
218    pub database_mutation_started: bool,
219    #[serde(default, skip_serializing_if = "Vec::is_empty")]
220    pub original_replicas: Vec<ReplicaCountEntry>,
221    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
222    pub phase_timestamps: std::collections::BTreeMap<String, String>,
223    #[serde(default, skip_serializing_if = "Vec::is_empty")]
224    #[cfg_attr(
225        feature = "schemars",
226        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
227    )]
228    pub conditions: Vec<Condition>,
229    #[serde(skip_serializing_if = "Option::is_none")]
230    pub message: Option<String>,
231}
232
233#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
234#[cfg_attr(feature = "schemars", derive(JsonSchema))]
235#[serde(rename_all = "camelCase")]
236pub struct ReplicaCountEntry {
237    pub group: String,
238    pub replicas: i32,
239}
240
241#[derive(Clone)]
242struct RestoreMetrics {
243    attempts: Counter<u64>,
244    outcomes: Counter<u64>,
245    duration_seconds: Histogram<f64>,
246    start_timestamp_seconds: Gauge<i64>,
247    break_glass_total: Counter<u64>,
248    disaster_recovery_total: Counter<u64>,
249    safety_backup_duration_seconds: Histogram<f64>,
250}
251
252impl RestoreMetrics {
253    fn new() -> Self {
254        let meter = global::meter("kaniop");
255        Self {
256            attempts: meter
257                .u64_counter("restore_attempts")
258                .with_description("Number of Kanidm restore attempts started")
259                .build(),
260            outcomes: meter
261                .u64_counter("restore_outcomes")
262                .with_description("Number of terminal Kanidm restore outcomes")
263                .build(),
264            duration_seconds: meter
265                .f64_histogram("restore_duration_seconds")
266                .with_description("Kanidm restore duration from object creation to terminal phase")
267                .with_unit("s")
268                .build(),
269            start_timestamp_seconds: meter
270                .i64_gauge("restore_start_timestamp_seconds")
271                .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
272                .with_unit("s")
273                .build(),
274            break_glass_total: meter
275                .u64_counter("restore_break_glass")
276                .with_description("Number of break-glass safety backup overrides")
277                .build(),
278            disaster_recovery_total: meter
279                .u64_counter("restore_disaster_recovery")
280                .with_description("Number of approved disaster-recovery source identity overrides")
281                .build(),
282            safety_backup_duration_seconds: meter
283                .f64_histogram("restore_safety_backup_duration_seconds")
284                .with_description("Duration of pre-restore safety backup creation")
285                .with_unit("s")
286                .build(),
287        }
288    }
289}
290
291fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
292    let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
293    let active = restore.metadata.deletion_timestamp.is_none()
294        && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
295    let timestamp = if active {
296        restore
297            .metadata
298            .creation_timestamp
299            .as_ref()
300            .map(|created| created.0.as_second())
301            .unwrap_or_else(|| Timestamp::now().as_second())
302    } else {
303        0
304    };
305    let attributes = [
306        KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
307        KeyValue::new("restore", restore.name_any()),
308    ];
309    metrics.start_timestamp_seconds.record(timestamp, &attributes);
310}
311
312#[derive(Clone)]
313struct RestoreContext {
314    client: Client,
315    recorder: Recorder,
316    metrics: RestoreMetrics,
317}
318
319async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
320    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
321    match phase {
322        KanidmRestorePhase::Pending => {
323            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
324            Ok(Action::requeue(REQUEUE))
325        }
326        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
327            Ok((target, disaster_recovery_source)) => {
328                // Do not quiesce the target until the repository's encryption identity
329                // has been durably locked against a concurrent spec edit.
330                if requires_safety_backup(&restore)
331                    && !safety_repository_usage_locked(&restore, &ctx).await?
332                {
333                    return Ok(Action::requeue(REQUEUE));
334                }
335                mark_restoring(&restore, &target, &ctx).await?;
336                if let Some(source) = disaster_recovery_source.as_ref() {
337                    record_disaster_recovery_override(&restore, &target, &ctx, source).await;
338                }
339                let mut status = restore.status.clone().unwrap_or_default();
340                if let Some(source) = disaster_recovery_source.as_ref() {
341                    let target_uid = target.uid().unwrap_or_default();
342                    let condition = restore_condition(
343                        &status.conditions,
344                        CONDITION_DISASTER_RECOVERY,
345                        CONDITION_TRUE,
346                        "ApprovedSourceIdentityMismatch",
347                        &format!(
348                            "Approved disaster recovery from source {}/{} into target {}/{}",
349                            source.name,
350                            source.uid,
351                            target.name_any(),
352                            target_uid
353                        ),
354                        restore.metadata.generation,
355                    );
356                    status
357                        .conditions
358                        .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
359                    status.conditions.push(condition);
360                }
361                status.observed_target_uid = target.uid();
362                status.original_replicas = capture_original_replicas(&target);
363                status.phase = KanidmRestorePhase::Quiescing;
364                status.message = None;
365                patch_status(&restore, &ctx, status).await?;
366                Ok(Action::requeue(REQUEUE))
367            }
368            Err(error) => {
369                set_phase(
370                    &restore,
371                    &ctx,
372                    KanidmRestorePhase::Failed,
373                    Some(error.to_string()),
374                )
375                .await?;
376                Ok(Action::requeue(Duration::from_secs(300)))
377            }
378        },
379        KanidmRestorePhase::Quiescing => {
380            let target = get_target(&restore, &ctx).await?;
381            scale_all(&target, &ctx, 0).await?;
382            if target_pods_stopped(&target, &ctx).await?
383                && target_volumes_detached(&target, &ctx).await?
384            {
385                let next_phase = if requires_safety_backup(&restore) {
386                    KanidmRestorePhase::SafetyBackup
387                } else {
388                    record_break_glass(&restore, &ctx).await?;
389                    KanidmRestorePhase::PreparingSource
390                };
391                if next_phase == KanidmRestorePhase::PreparingSource {
392                    let refreshed = Api::<KanidmRestore>::namespaced(
393                        ctx.client.clone(),
394                        restore.namespace().as_deref().unwrap_or_default(),
395                    )
396                    .get(&restore.name_any())
397                    .await
398                    .map_err(|error| {
399                        Error::kube_error(
400                            "get",
401                            "KanidmRestore",
402                            restore.namespace().as_deref().unwrap_or_default(),
403                            restore.name_any(),
404                            error,
405                        )
406                    })?;
407                    set_phase(&refreshed, &ctx, next_phase, None).await?;
408                } else {
409                    set_phase(&restore, &ctx, next_phase, None).await?;
410                }
411            }
412            Ok(Action::requeue(REQUEUE))
413        }
414        KanidmRestorePhase::SafetyBackup => {
415            // Recheck after restarts or manual phase changes, before creating
416            // any Job capable of uploading to the remote repository.
417            if !safety_repository_usage_locked(&restore, &ctx).await? {
418                return Ok(Action::requeue(REQUEUE));
419            }
420            let target = get_target(&restore, &ctx).await?;
421            let name = safety_backup_job_name(&restore);
422            let expected_backup_id = compute_safety_backup_id(&restore)?;
423            let expected_manifest_key = compute_safety_manifest_key(&restore)?;
424            ensure_safety_backup_job(
425                &restore,
426                &target,
427                &ctx,
428                &name,
429                &expected_backup_id,
430                &expected_manifest_key,
431            )
432            .await?;
433            let mut status = restore.status.clone().unwrap_or_default();
434            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
435            match job_state(&restore, &ctx, &name).await? {
436                JobState::Complete => {
437                    let result =
438                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
439                    match result {
440                        Ok(verified) => {
441                            status.safety_backup_job_name = Some(name);
442                            status.safety_backup_ref =
443                                Some(format!("safety-{}", restore.name_any()));
444                            status.safety_backup_manifest_key = Some(verified.manifest_key);
445                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
446                            record_safety_backup_duration(&restore, &ctx).await;
447                            status.phase = KanidmRestorePhase::PreparingSource;
448                            status.message = None;
449                            patch_status(&restore, &ctx, status).await?;
450                        }
451                        Err(error) => {
452                            resume_before_mutation(&restore, &ctx).await?;
453                            set_phase(
454                                &restore,
455                                &ctx,
456                                KanidmRestorePhase::Failed,
457                                Some(format!("safety backup result verification failed: {error}")),
458                            )
459                            .await?;
460                        }
461                    }
462                }
463                JobState::Failed => {
464                    resume_before_mutation(&restore, &ctx).await?;
465                    set_phase(
466                        &restore,
467                        &ctx,
468                        KanidmRestorePhase::Failed,
469                        Some(
470                            "safety backup job failed; target restored to original state"
471                                .to_string(),
472                        ),
473                    )
474                    .await?;
475                }
476                JobState::Running => {
477                    patch_status(&restore, &ctx, status).await?;
478                }
479            }
480            Ok(Action::requeue(REQUEUE))
481        }
482        KanidmRestorePhase::PreparingSource => {
483            if is_remote_source(&restore) {
484                let target = get_target(&restore, &ctx).await?;
485                let name = source_prep_job_name(&restore);
486                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
487                match job_state(&restore, &ctx, &name).await? {
488                    JobState::Complete => {
489                        let backup_id = restore
490                            .spec
491                            .source
492                            .backup_ref
493                            .as_ref()
494                            .map(|b| b.name.clone())
495                            .unwrap_or_default();
496                        let backup = {
497                            let ns = restore.namespace().unwrap();
498                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
499                                .get(&backup_id)
500                                .await
501                                .map_err(|e| {
502                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
503                                })?
504                        };
505                        let expected_backup_id = &backup.spec.backup_id;
506                        let result =
507                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
508                                .await;
509                        match result {
510                            Ok(verified) => {
511                                if let Some(expected_sha256) = backup
512                                    .status
513                                    .as_ref()
514                                    .and_then(|s| s.payload_sha256.as_deref())
515                                {
516                                    if verified.payload_sha256 != expected_sha256 {
517                                        resume_before_mutation(&restore, &ctx).await?;
518                                        set_phase(
519                                            &restore,
520                                            &ctx,
521                                            KanidmRestorePhase::Failed,
522                                            Some(format!(
523                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
524                                                verified.payload_sha256
525                                            )),
526                                        )
527                                        .await?;
528                                        return Ok(Action::requeue(REQUEUE));
529                                    }
530                                }
531                                let mut status = restore.status.clone().unwrap_or_default();
532                                status.source_prep_job_name = Some(name);
533                                status.phase = KanidmRestorePhase::RestoringPrimary;
534                                status.message = None;
535                                patch_status(&restore, &ctx, status).await?;
536                            }
537                            Err(error) => {
538                                resume_before_mutation(&restore, &ctx).await?;
539                                set_phase(
540                                    &restore,
541                                    &ctx,
542                                    KanidmRestorePhase::Failed,
543                                    Some(format!(
544                                        "source preparation result verification failed: {error}"
545                                    )),
546                                )
547                                .await?;
548                            }
549                        }
550                    }
551                    JobState::Failed => {
552                        resume_before_mutation(&restore, &ctx).await?;
553                        set_phase(
554                            &restore,
555                            &ctx,
556                            KanidmRestorePhase::Failed,
557                            Some(
558                                "source preparation job failed; target restored to original state"
559                                    .to_string(),
560                            ),
561                        )
562                        .await?;
563                    }
564                    JobState::Running => {}
565                }
566            } else {
567                let target = get_target(&restore, &ctx).await?;
568                let name = source_check_job_name(&restore);
569                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
570                match job_state(&restore, &ctx, &name).await? {
571                    JobState::Complete => {
572                        let mut status = restore.status.clone().unwrap_or_default();
573                        status.source_prep_job_name = Some(name);
574                        status.phase = KanidmRestorePhase::RestoringPrimary;
575                        status.message = None;
576                        patch_status(&restore, &ctx, status).await?;
577                    }
578                    JobState::Failed => {
579                        resume_before_mutation(&restore, &ctx).await?;
580                        set_phase(
581                            &restore,
582                            &ctx,
583                            KanidmRestorePhase::Failed,
584                            Some(
585                                "local backup file check failed; target restored to original state"
586                                    .to_string(),
587                            ),
588                        )
589                        .await?;
590                    }
591                    JobState::Running => {}
592                }
593            }
594            Ok(Action::requeue(REQUEUE))
595        }
596        KanidmRestorePhase::RestoringPrimary => {
597            let target = get_target(&restore, &ctx).await?;
598            let status = restore.status.as_ref().cloned().unwrap_or_default();
599            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
600                return Err(Error::MissingData(
601                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
602                ));
603            }
604            if !status.database_mutation_started {
605                let mut status = status;
606                status.database_mutation_started = true;
607                status.message = Some(
608                    "database mutation boundary persisted; starting primary restore".to_string(),
609                );
610                patch_status(&restore, &ctx, status).await?;
611                return Ok(Action::requeue(REQUEUE));
612            }
613            let name = restore_job_name(&restore);
614            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
615            match job_state(&restore, &ctx, &name).await? {
616                JobState::Complete => {
617                    let mut status = restore.status.clone().unwrap_or_default();
618                    status.restore_job_name = Some(name);
619                    status.phase = KanidmRestorePhase::Verifying;
620                    status.message = None;
621                    patch_status(&restore, &ctx, status).await?;
622                }
623                JobState::Failed => {
624                    let status = restore.status.clone().unwrap_or_default();
625                    let failed_status = restore_primary_failed_status(&status, &name);
626                    patch_status(&restore, &ctx, failed_status).await?;
627                }
628                JobState::Running => {}
629            }
630            Ok(Action::requeue(REQUEUE))
631        }
632        KanidmRestorePhase::Verifying => {
633            let target = get_target(&restore, &ctx).await?;
634            let name = verify_job_name(&restore);
635            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
636            match job_state(&restore, &ctx, &name).await? {
637                JobState::Complete => {
638                    let mut status = restore.status.clone().unwrap_or_default();
639                    status.verify_job_name = Some(name);
640                    status.phase = KanidmRestorePhase::RebuildingReplicas;
641                    status.message = None;
642                    patch_status(&restore, &ctx, status).await?;
643                }
644                JobState::Failed => {
645                    let status = restore.status.clone().unwrap_or_default();
646                    let failed_status = verifying_failed_status(&status, &name);
647                    patch_status(&restore, &ctx, failed_status).await?;
648                }
649                JobState::Running => {}
650            }
651            Ok(Action::requeue(REQUEUE))
652        }
653        KanidmRestorePhase::RebuildingReplicas => {
654            let target = get_target(&restore, &ctx).await?;
655            let mut status = restore.status.clone().unwrap_or_default();
656            if !status.replicas_cleared {
657                if delete_secondary_pvcs(&target, &ctx).await? {
658                    status.replicas_cleared = true;
659                    status.message = Some("secondary database state cleared".to_string());
660                    patch_status(&restore, &ctx, status).await?;
661                }
662                return Ok(Action::requeue(REQUEUE));
663            }
664
665            scale_primary(&target, &ctx, 1).await?;
666            if !primary_ready(&target, &ctx).await? {
667                return Ok(Action::requeue(REQUEUE));
668            }
669            if !status.certificates_cleared {
670                delete_replica_cert_secrets(&target, &ctx).await?;
671                delete_admin_secret(&target, &ctx).await?;
672                status.certificates_cleared = true;
673                status.message = Some(
674                    "replica certificates and admin secret cleared for regeneration".to_string(),
675                );
676                patch_status(&restore, &ctx, status).await?;
677                return Ok(Action::requeue(REQUEUE));
678            }
679            status.phase = KanidmRestorePhase::Resuming;
680            status.message = None;
681            patch_status(&restore, &ctx, status).await?;
682            Ok(Action::requeue(REQUEUE))
683        }
684        KanidmRestorePhase::Resuming => {
685            let target = get_target(&restore, &ctx).await?;
686            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
687            clear_restoring(&restore, &target, &ctx).await?;
688            Ok(Action::requeue(Duration::from_secs(3600)))
689        }
690        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
691            Ok(Action::requeue(Duration::from_secs(3600)))
692        }
693    }
694}
695
696#[derive(Clone, Debug, PartialEq, Eq)]
697struct DisasterRecoverySource {
698    name: String,
699    uid: String,
700}
701
702async fn validate(
703    restore: &KanidmRestore,
704    ctx: &RestoreContext,
705) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
706    let target = get_target(restore, ctx).await?;
707    let actual_uid = target
708        .uid()
709        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
710    if actual_uid != restore.spec.target_ref.uid {
711        return Err(Error::MissingData(format!(
712            "target UID mismatch: expected {}, got {}",
713            restore.spec.target_ref.uid, actual_uid
714        )));
715    }
716    validate_source(restore)?;
717    let disaster_recovery_source = if is_remote_source(restore) {
718        validate_backup_ref(restore, &target, ctx).await?
719    } else {
720        None
721    };
722    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
723    {
724        return Err(Error::MissingData(format!(
725            "restoreImage must be the target's pinned Kanidm image (target image is {})",
726            target.spec.image
727        )));
728    }
729    let storage = target
730        .spec
731        .storage
732        .as_ref()
733        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
734    if storage.empty_dir.is_some()
735        || storage.ephemeral.is_some()
736        || storage.volume_claim_template.is_none()
737    {
738        return Err(Error::MissingData(
739            "restore requires PVC-backed Kanidm storage".to_string(),
740        ));
741    }
742    let primaries = target
743        .spec
744        .replica_groups
745        .iter()
746        .filter(|rg| rg.primary_node)
747        .count();
748    if primaries != 1 {
749        return Err(Error::MissingData(
750            "backup/restore requires exactly one primary replica group".to_string(),
751        ));
752    }
753    let ns = restore.namespace().unwrap();
754    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
755        .list(&ListParams::default())
756        .await
757        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
758    if restores.items.iter().any(|other| {
759        other.name_any() != restore.name_any()
760            && other.spec.target_ref.name == restore.spec.target_ref.name
761            && !matches!(
762                other.status.as_ref().map(|s| s.phase),
763                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
764            )
765    }) {
766        return Err(Error::MissingData(
767            "another active restore targets this Kanidm".to_string(),
768        ));
769    }
770    validate_safety_backup_config(restore)?;
771    Ok((target, disaster_recovery_source))
772}
773
774fn safe_basename(name: &str) -> bool {
775    !name.is_empty()
776        && name != "."
777        && name != ".."
778        && !name.contains('/')
779        && !name.contains('\\')
780        && !name.contains('\0')
781}
782
783fn mutable_image(image: &str) -> bool {
784    image == "kanidm/server:latest"
785        || image.ends_with(":latest")
786        || (!image.contains('@')
787            && !image
788                .rsplit('/')
789                .next()
790                .is_some_and(|part| part.contains(':')))
791}
792
793async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
794    let ns = restore
795        .namespace()
796        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
797    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
798        .get(&restore.spec.target_ref.name)
799        .await
800        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
801}
802
803async fn mark_restoring(
804    restore: &KanidmRestore,
805    target: &Kanidm,
806    ctx: &RestoreContext,
807) -> Result<()> {
808    let ns = target.namespace().unwrap();
809    let uid = restore
810        .uid()
811        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
812    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
813        .patch(
814            &target.name_any(),
815            &PatchParams::default(),
816            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
817        )
818        .await
819        .map(|_| ())
820        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
821}
822
823async fn clear_restoring(
824    restore: &KanidmRestore,
825    target: &Kanidm,
826    ctx: &RestoreContext,
827) -> Result<()> {
828    let ns = target.namespace().unwrap();
829    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
830        return Ok(());
831    }
832    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
833        .patch(
834            &target.name_any(),
835            &PatchParams::default(),
836            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
837        )
838        .await
839        .map(|_| ())
840        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
841}
842
843async fn patch_status(
844    restore: &KanidmRestore,
845    ctx: &RestoreContext,
846    mut status: KanidmRestoreStatus,
847) -> Result<()> {
848    let ns = restore.namespace().unwrap();
849    let previous_phase = restore.status.as_ref().map(|current| current.phase);
850    let next_phase = status.phase;
851    status.observed_generation = restore.metadata.generation;
852    update_restore_conditions(&mut status, restore.metadata.generation);
853
854    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
855        .patch_status(
856            &restore.name_any(),
857            &PatchParams::default(),
858            &Patch::Merge(json!({"status": &status})),
859        )
860        .await
861        .map_err(|e| {
862            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
863        })?;
864
865    if previous_phase != Some(next_phase) {
866        record_restore_transition(
867            restore,
868            ctx,
869            previous_phase,
870            next_phase,
871            status.message.as_deref(),
872        )
873        .await;
874    }
875    Ok(())
876}
877
878fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
879    let previous = status.conditions.clone();
880    let phase = status.phase;
881    let phase_reason = format!("{phase:?}");
882    let message = status
883        .message
884        .clone()
885        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
886    let terminal_success = phase == KanidmRestorePhase::Completed;
887    let terminal_failure = phase == KanidmRestorePhase::Failed;
888    let progressing = !terminal_success && !terminal_failure;
889
890    let sticky_conditions: Vec<Condition> = previous
891        .iter()
892        .filter(|condition| {
893            condition.type_ == CONDITION_BREAK_GLASS
894                || condition.type_ == CONDITION_DISASTER_RECOVERY
895        })
896        .cloned()
897        .collect();
898    status.conditions = vec![
899        restore_condition(
900            &previous,
901            CONDITION_PROGRESSING,
902            if progressing {
903                CONDITION_TRUE
904            } else {
905                CONDITION_FALSE
906            },
907            &phase_reason,
908            &message,
909            generation,
910        ),
911        restore_condition(
912            &previous,
913            CONDITION_READY,
914            if terminal_success {
915                CONDITION_TRUE
916            } else {
917                CONDITION_FALSE
918            },
919            if terminal_success {
920                "RestoreCompleted"
921            } else {
922                "RestoreNotCompleted"
923            },
924            if terminal_success {
925                "Kanidm restore completed successfully."
926            } else {
927                "Kanidm restore has not completed successfully."
928            },
929            generation,
930        ),
931        restore_condition(
932            &previous,
933            CONDITION_FAILED,
934            if terminal_failure {
935                CONDITION_TRUE
936            } else {
937                CONDITION_FALSE
938            },
939            if terminal_failure {
940                "RestoreFailed"
941            } else {
942                "NoRestoreFailure"
943            },
944            if terminal_failure {
945                status
946                    .message
947                    .as_deref()
948                    .unwrap_or("Kanidm restore failed.")
949            } else {
950                "No terminal restore failure has been recorded."
951            },
952            generation,
953        ),
954    ];
955    status.conditions.extend(sticky_conditions);
956}
957
958fn restore_condition(
959    previous: &[Condition],
960    condition_type: &str,
961    condition_status: &str,
962    reason: &str,
963    message: &str,
964    generation: Option<i64>,
965) -> Condition {
966    let last_transition_time = previous
967        .iter()
968        .find(|condition| {
969            condition.type_ == condition_type
970                && condition.status == condition_status
971                && condition.reason == reason
972        })
973        .map(|condition| condition.last_transition_time.clone())
974        .unwrap_or_else(|| Time(Timestamp::now()));
975    Condition {
976        type_: condition_type.to_string(),
977        status: condition_status.to_string(),
978        reason: reason.to_string(),
979        message: message.to_string(),
980        last_transition_time,
981        observed_generation: generation,
982    }
983}
984
985async fn record_restore_transition(
986    restore: &KanidmRestore,
987    ctx: &RestoreContext,
988    previous_phase: Option<KanidmRestorePhase>,
989    phase: KanidmRestorePhase,
990    message: Option<&str>,
991) {
992    if phase == KanidmRestorePhase::Validating {
993        ctx.metrics.attempts.add(1, &[]);
994    }
995
996    let result = match phase {
997        KanidmRestorePhase::Completed => Some("success"),
998        KanidmRestorePhase::Failed => Some("failure"),
999        _ => None,
1000    };
1001    if let Some(result) = result {
1002        let attributes = [
1003            KeyValue::new("result", result),
1004            KeyValue::new("phase", format!("{phase:?}")),
1005        ];
1006        ctx.metrics.outcomes.add(1, &attributes);
1007        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
1008            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
1009            ctx.metrics.duration_seconds.record(elapsed, &attributes);
1010        }
1011    }
1012
1013    let reason = if phase == KanidmRestorePhase::Failed {
1014        "RestoreFailed"
1015    } else {
1016        "RestorePhaseChanged"
1017    };
1018    let note = message.map(str::to_string).or_else(|| {
1019        Some(format!(
1020            "Kanidm restore phase changed from {} to {phase:?}.",
1021            previous_phase
1022                .map(|previous| format!("{previous:?}"))
1023                .unwrap_or_else(|| "None".to_string())
1024        ))
1025    });
1026    if let Err(error) = ctx
1027        .recorder
1028        .publish(
1029            &Event {
1030                type_: if phase == KanidmRestorePhase::Failed {
1031                    EventType::Warning
1032                } else {
1033                    EventType::Normal
1034                },
1035                reason: reason.to_string(),
1036                note,
1037                action: "Restore".to_string(),
1038                secondary: None,
1039            },
1040            &restore.object_ref(&()),
1041        )
1042        .await
1043    {
1044        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1045    }
1046}
1047
1048async fn set_phase(
1049    restore: &KanidmRestore,
1050    ctx: &RestoreContext,
1051    phase: KanidmRestorePhase,
1052    message: Option<String>,
1053) -> Result<()> {
1054    let mut status = restore.status.clone().unwrap_or_default();
1055    status.phase = phase;
1056    status.message = message;
1057    patch_status(restore, ctx, status).await
1058}
1059
1060async fn fail_after_mutation(
1061    restore: &KanidmRestore,
1062    ctx: &RestoreContext,
1063    message: &str,
1064) -> Result<()> {
1065    set_phase(
1066        restore,
1067        ctx,
1068        KanidmRestorePhase::Failed,
1069        Some(message.to_string()),
1070    )
1071    .await
1072}
1073
1074fn restore_primary_failed_status(
1075    existing: &KanidmRestoreStatus,
1076    job_name: &str,
1077) -> KanidmRestoreStatus {
1078    let mut status = existing.clone();
1079    status.restore_job_name = Some(job_name.to_string());
1080    status.phase = KanidmRestorePhase::Failed;
1081    status.message = Some("database restore job failed".to_string());
1082    status
1083}
1084
1085fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1086    let mut status = existing.clone();
1087    status.verify_job_name = Some(job_name.to_string());
1088    status.phase = KanidmRestorePhase::Failed;
1089    status.message = Some("database verification failed".to_string());
1090    status
1091}
1092
1093fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1094    target
1095        .spec
1096        .replica_groups
1097        .iter()
1098        .find(|rg| rg.primary_node)
1099        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1100}
1101
1102async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1103    let ns = target.namespace().unwrap();
1104    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1105    for rg in &target.spec.replica_groups {
1106        let name = target.statefulset_name(&rg.name);
1107        api.patch(
1108            &name,
1109            &PatchParams::default(),
1110            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1111        )
1112        .await
1113        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1114    }
1115    Ok(())
1116}
1117
1118async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1119    let ns = target.namespace().unwrap();
1120    let rg = primary_group(target)?;
1121    let name = target.statefulset_name(&rg.name);
1122    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1123        .patch(
1124            &name,
1125            &PatchParams::default(),
1126            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1127        )
1128        .await
1129        .map(|_| ())
1130        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1131}
1132
1133async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1134    let ns = target.namespace().unwrap();
1135    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1136    for rg in &target.spec.replica_groups {
1137        let name = target.statefulset_name(&rg.name);
1138        api.patch(
1139            &name,
1140            &PatchParams::default(),
1141            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1142        )
1143        .await
1144        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1145    }
1146    Ok(())
1147}
1148
1149async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1150    let ns = target.namespace().unwrap();
1151    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1152        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1153        .await
1154        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1155    Ok(pods.items.is_empty())
1156}
1157
1158async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1159    let ns = target.namespace().unwrap();
1160    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1161    let mut pv_names = BTreeSet::new();
1162    let primary_name = primary_pvc_name(target)?;
1163    let primary_pvc = pvc_api
1164        .get(&primary_name)
1165        .await
1166        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1167    let primary_pv = primary_pvc
1168        .spec
1169        .as_ref()
1170        .and_then(|spec| spec.volume_name.clone())
1171        .ok_or_else(|| {
1172            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1173        })?;
1174    pv_names.insert(primary_pv);
1175
1176    for rg in &target.spec.replica_groups {
1177        let sts = target.statefulset_name(&rg.name);
1178        for ordinal in 0..rg.replicas {
1179            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1180            if name == primary_name {
1181                continue;
1182            }
1183            if let Some(pvc) = pvc_api
1184                .get_opt(&name)
1185                .await
1186                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1187                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1188            {
1189                pv_names.insert(pv_name);
1190            }
1191        }
1192    }
1193
1194    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1195        .list(&ListParams::default())
1196        .await
1197        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1198    Ok(!attachments.items.iter().any(|attachment| {
1199        attachment
1200            .spec
1201            .source
1202            .persistent_volume_name
1203            .as_ref()
1204            .is_some_and(|pv_name| pv_names.contains(pv_name))
1205    }))
1206}
1207
1208async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1209    let ns = target.namespace().unwrap();
1210    let name = target.statefulset_name(&primary_group(target)?.name);
1211    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1212        .get(&name)
1213        .await
1214        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1215    Ok(sts
1216        .status
1217        .as_ref()
1218        .and_then(|s| s.ready_replicas)
1219        .unwrap_or(0)
1220        >= 1)
1221}
1222
1223fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1224    let rg = primary_group(target)?;
1225    Ok(format!(
1226        "{DATA_VOLUME}-{}-0",
1227        target.statefulset_name(&rg.name)
1228    ))
1229}
1230
1231async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1232    let ns = target.namespace().unwrap();
1233    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1234    let mut all_absent = true;
1235    for rg in &target.spec.replica_groups {
1236        let sts = target.statefulset_name(&rg.name);
1237        for ordinal in 0..rg.replicas {
1238            if rg.primary_node && ordinal == 0 {
1239                continue;
1240            }
1241            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1242            match api
1243                .get_opt(&name)
1244                .await
1245                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1246            {
1247                None => {}
1248                Some(pvc) => {
1249                    all_absent = false;
1250                    if pvc.metadata.deletion_timestamp.is_none() {
1251                        match api.delete(&name, &DeleteParams::default()).await {
1252                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1253                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1254                            Err(error) => {
1255                                return Err(Error::kube_error(
1256                                    "delete",
1257                                    "PersistentVolumeClaim",
1258                                    &ns,
1259                                    &name,
1260                                    error,
1261                                ));
1262                            }
1263                        }
1264                    }
1265                }
1266            }
1267        }
1268    }
1269    Ok(all_absent)
1270}
1271
1272async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1273    let ns = target.namespace().unwrap();
1274    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1275    let label_selector = format!(
1276        "{}={},{}={}",
1277        CLUSTER_LABEL,
1278        target.name_any(),
1279        SECRET_TYPE_LABEL,
1280        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1281    );
1282    let secrets = api
1283        .list(&ListParams::default().labels(&label_selector))
1284        .await
1285        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1286    let mut all_absent = secrets.items.is_empty();
1287    for secret in secrets.items {
1288        let name = secret.name_any();
1289        if secret.metadata.deletion_timestamp.is_none() {
1290            match api.delete(&name, &DeleteParams::default()).await {
1291                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1292                Err(kube::Error::Api(status)) if status.code == 404 => {}
1293                Err(error) => {
1294                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1295                }
1296            }
1297            all_absent = false;
1298        }
1299    }
1300    Ok(all_absent)
1301}
1302
1303async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1304    let ns = target.namespace().unwrap();
1305    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1306    let name = format!("{}-admin-passwords", target.name_any());
1307    match api
1308        .get_opt(&name)
1309        .await
1310        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1311    {
1312        None => Ok(true),
1313        Some(secret) => {
1314            if secret.metadata.deletion_timestamp.is_some() {
1315                return Ok(false);
1316            }
1317            match api.delete(&name, &DeleteParams::default()).await {
1318                Ok(_) => {
1319                    debug!(secret = %name, "deleting stale admin secret");
1320                    Ok(false)
1321                }
1322                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1323                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1324            }
1325        }
1326    }
1327}
1328
1329fn restore_job_name(restore: &KanidmRestore) -> String {
1330    format!("{}-restore", restore.name_any())
1331}
1332fn verify_job_name(restore: &KanidmRestore) -> String {
1333    format!("{}-verify", restore.name_any())
1334}
1335
1336fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1337    let origin = target
1338        .spec
1339        .origin
1340        .clone()
1341        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1342    vec![
1343        EnvVar {
1344            name: "KANIDM_DB_PATH".to_string(),
1345            value: Some("/data/kanidm.db".to_string()),
1346            ..Default::default()
1347        },
1348        EnvVar {
1349            name: "KANIDM_DOMAIN".to_string(),
1350            value: Some(target.spec.domain.clone()),
1351            ..Default::default()
1352        },
1353        EnvVar {
1354            name: "KANIDM_ORIGIN".to_string(),
1355            value: Some(origin),
1356            ..Default::default()
1357        },
1358        EnvVar {
1359            name: "KANIDM_TLS_CHAIN".to_string(),
1360            value: Some(format!("{TLS_PATH}/tls.crt")),
1361            ..Default::default()
1362        },
1363        EnvVar {
1364            name: "KANIDM_TLS_KEY".to_string(),
1365            value: Some(format!("{TLS_PATH}/tls.key")),
1366            ..Default::default()
1367        },
1368    ]
1369}
1370
1371async fn ensure_database_job(
1372    restore: &KanidmRestore,
1373    target: &Kanidm,
1374    ctx: &RestoreContext,
1375    name: &str,
1376    verify: bool,
1377) -> Result<()> {
1378    let ns = restore.namespace().unwrap();
1379    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1380    if api
1381        .get_opt(name)
1382        .await
1383        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1384        .is_some()
1385    {
1386        return Ok(());
1387    }
1388    let mut command = vec![
1389        "kanidmd".to_string(),
1390        "database".to_string(),
1391        if verify { "backup" } else { "restore" }.to_string(),
1392    ];
1393    if verify {
1394        command.push("/verify/verification.json.gz".to_string());
1395    } else {
1396        if is_remote_source(restore) {
1397            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1398        } else {
1399            let file_name = restore
1400                .spec
1401                .source
1402                .local
1403                .as_ref()
1404                .map(|l| l.file_name.clone())
1405                .unwrap_or_default();
1406            command.push(format!("{BACKUP_PATH}/{file_name}"));
1407        }
1408    }
1409    let job = Job {
1410        metadata: ObjectMeta {
1411            name: Some(name.to_string()),
1412            namespace: Some(ns.clone()),
1413            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1414            ..Default::default()
1415        },
1416        spec: Some(JobSpec {
1417            backoff_limit: Some(0),
1418            template: PodTemplateSpec {
1419                metadata: Some(ObjectMeta::default()),
1420                spec: Some(PodSpec {
1421                    automount_service_account_token: Some(false),
1422                    restart_policy: Some("Never".to_string()),
1423                    security_context: Some(source_job_pod_security_context(target)),
1424                    containers: vec![Container {
1425                        name: if verify { "verify" } else { "restore" }.to_string(),
1426                        image: Some(restore.spec.restore_image.clone()),
1427                        command: Some(command),
1428                        env: Some(build_kanidm_db_env_vars(target)),
1429                        security_context: Some(kanidm_job_security_context()),
1430                        resources: Some(job_resource_requirements()),
1431                        volume_mounts: Some(vec![
1432                            VolumeMount {
1433                                name: DATA_VOLUME.to_string(),
1434                                mount_path: DATA_PATH.to_string(),
1435                                ..Default::default()
1436                            },
1437                            VolumeMount {
1438                                name: TLS_VOLUME.to_string(),
1439                                mount_path: TLS_PATH.to_string(),
1440                                read_only: Some(true),
1441                                ..Default::default()
1442                            },
1443                            VolumeMount {
1444                                name: "verification-output".to_string(),
1445                                mount_path: "/verify".to_string(),
1446                                ..Default::default()
1447                            },
1448                        ]),
1449                        ..Default::default()
1450                    }],
1451                    volumes: Some(vec![
1452                        Volume {
1453                            name: DATA_VOLUME.to_string(),
1454                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1455                                claim_name: primary_pvc_name(target)?,
1456                                read_only: Some(false),
1457                            }),
1458                            ..Default::default()
1459                        },
1460                        Volume {
1461                            name: TLS_VOLUME.to_string(),
1462                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1463                                secret_name: Some(target.effective_tls_secret_name()),
1464                                ..Default::default()
1465                            }),
1466                            ..Default::default()
1467                        },
1468                        Volume {
1469                            name: "verification-output".to_string(),
1470                            empty_dir: Some(Default::default()),
1471                            ..Default::default()
1472                        },
1473                    ]),
1474                    ..Default::default()
1475                }),
1476            },
1477            ..Default::default()
1478        }),
1479        ..Default::default()
1480    };
1481    api.create(&PostParams::default(), &job)
1482        .await
1483        .map(|_| ())
1484        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1485}
1486
1487enum JobState {
1488    Running,
1489    Complete,
1490    Failed,
1491}
1492
1493async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1494    let ns = restore.namespace().unwrap();
1495    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1496        .get(name)
1497        .await
1498        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1499    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1500        return Ok(JobState::Failed);
1501    }
1502    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1503        return Ok(JobState::Complete);
1504    }
1505    Ok(JobState::Running)
1506}
1507
1508fn is_remote_source(restore: &KanidmRestore) -> bool {
1509    restore.spec.source.backup_ref.is_some()
1510}
1511
1512fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1513    let safety = restore.spec.safety_backup.as_ref();
1514    let skip = safety.is_some_and(|s| s.skip);
1515    !skip
1516}
1517
1518fn validate_source(restore: &KanidmRestore) -> Result<()> {
1519    let has_local = restore.spec.source.local.is_some();
1520    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1521    if has_local && has_backup_ref {
1522        return Err(Error::MissingData(
1523            "source.local and source.backupRef are mutually exclusive".to_string(),
1524        ));
1525    }
1526    if !has_local && !has_backup_ref {
1527        return Err(Error::MissingData(
1528            "source must specify either local or backupRef".to_string(),
1529        ));
1530    }
1531    if let Some(local) = &restore.spec.source.local {
1532        if !safe_basename(&local.file_name) {
1533            return Err(Error::MissingData(
1534                "restore source fileName must be a safe basename".to_string(),
1535            ));
1536        }
1537    }
1538    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1539        if backup_ref.name.is_empty() {
1540            return Err(Error::MissingData(
1541                "source.backupRef.name must not be empty".to_string(),
1542            ));
1543        }
1544    }
1545    Ok(())
1546}
1547
1548fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1549    restore
1550        .metadata
1551        .annotations
1552        .as_ref()
1553        .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1554        .is_some_and(|value| value == "true")
1555}
1556
1557fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1558    let annotations = restore.metadata.annotations.as_ref();
1559    let reason = annotations
1560        .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1561        .map(String::as_str)
1562        .filter(|value| !value.trim().is_empty());
1563    let approved_by = annotations
1564        .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1565        .map(String::as_str)
1566        .filter(|value| !value.trim().is_empty());
1567
1568    if reason.is_none() || approved_by.is_none() {
1569        return Err(Error::MissingData(format!(
1570            "disaster-recovery restore requires non-empty {} and {} annotations",
1571            BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1572        )));
1573    }
1574    Ok(())
1575}
1576
1577async fn record_disaster_recovery_override(
1578    restore: &KanidmRestore,
1579    target: &Kanidm,
1580    ctx: &RestoreContext,
1581    source: &DisasterRecoverySource,
1582) {
1583    ctx.metrics.disaster_recovery_total.add(1, &[]);
1584    let reason = restore
1585        .metadata
1586        .annotations
1587        .as_ref()
1588        .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1589        .cloned()
1590        .unwrap_or_default();
1591    let approved_by = restore
1592        .metadata
1593        .annotations
1594        .as_ref()
1595        .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1596        .cloned()
1597        .unwrap_or_default();
1598    let target_uid = target.uid().unwrap_or_default();
1599    let note = format!(
1600        "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1601        source.name,
1602        source.uid,
1603        target.name_any(),
1604        target_uid,
1605        reason,
1606        approved_by
1607    );
1608    if let Err(error) = ctx
1609        .recorder
1610        .publish(
1611            &Event {
1612                type_: EventType::Warning,
1613                reason: "DisasterRecoveryOverride".to_string(),
1614                note: Some(note.clone()),
1615                action: "Restore".to_string(),
1616                secondary: None,
1617            },
1618            &restore.object_ref(&()),
1619        )
1620        .await
1621    {
1622        warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1623    }
1624    warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1625}
1626
1627fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1628    let safety = restore.spec.safety_backup.as_ref();
1629    let skip = safety.is_some_and(|s| s.skip);
1630    if skip {
1631        let annotations = restore.metadata.annotations.as_ref();
1632        let reason = annotations
1633            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1634            .map(|s| s.trim())
1635            .unwrap_or("");
1636        let approver = annotations
1637            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1638            .map(|s| s.trim())
1639            .unwrap_or("");
1640        if reason.is_empty() {
1641            return Err(Error::MissingData(format!(
1642                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1643            )));
1644        }
1645        if approver.is_empty() {
1646            return Err(Error::MissingData(format!(
1647                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1648            )));
1649        }
1650    }
1651    if !skip {
1652        let has_repo = safety
1653            .and_then(|s| s.repository_ref.as_ref())
1654            .is_some_and(|r| !r.name.is_empty());
1655        if !has_repo {
1656            return Err(Error::MissingData(
1657                "safety backup requires safetyBackup.repositoryRef when skip is not set"
1658                    .to_string(),
1659            ));
1660        }
1661    }
1662    Ok(())
1663}
1664
1665fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1666    target
1667        .spec
1668        .replica_groups
1669        .iter()
1670        .map(|rg| ReplicaCountEntry {
1671            group: rg.name.clone(),
1672            replicas: rg.replicas,
1673        })
1674        .collect()
1675}
1676
1677async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1678    if let Ok(target) = get_target(restore, ctx).await {
1679        let owns_maintenance =
1680            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1681        if owns_maintenance {
1682            scale_desired(&target, ctx).await?;
1683            clear_restoring(restore, &target, ctx).await?;
1684        }
1685    }
1686    Ok(())
1687}
1688
1689async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1690    ctx.metrics.break_glass_total.add(1, &[]);
1691    let reason = restore
1692        .metadata
1693        .annotations
1694        .as_ref()
1695        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1696        .cloned()
1697        .unwrap_or_default();
1698    let approver = restore
1699        .metadata
1700        .annotations
1701        .as_ref()
1702        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1703        .cloned()
1704        .unwrap_or_default();
1705    let mut status = restore.status.clone().unwrap_or_default();
1706    let break_glass_condition = restore_condition(
1707        &status.conditions,
1708        CONDITION_BREAK_GLASS,
1709        CONDITION_TRUE,
1710        "BreakGlassOverride",
1711        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1712        restore.metadata.generation,
1713    );
1714    status
1715        .conditions
1716        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1717    status.conditions.push(break_glass_condition);
1718    patch_status(restore, ctx, status).await?;
1719    if let Err(error) = ctx
1720        .recorder
1721        .publish(
1722            &Event {
1723                type_: EventType::Warning,
1724                reason: "SafetyBackupSkipped".to_string(),
1725                note: Some(format!(
1726                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1727                )),
1728                action: "Restore".to_string(),
1729                secondary: None,
1730            },
1731            &restore.object_ref(&()),
1732        )
1733        .await
1734    {
1735        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1736    }
1737    warn!(
1738        restore = %restore.name_any(),
1739        reason = %reason,
1740        approver = %approver,
1741        "break-glass safety backup override activated"
1742    );
1743    Ok(())
1744}
1745
1746async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1747    let status = restore.status.as_ref().cloned().unwrap_or_default();
1748    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1749        if let Ok(start_ts) = start.parse::<Timestamp>() {
1750            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1751            ctx.metrics
1752                .safety_backup_duration_seconds
1753                .record(elapsed, &[]);
1754        }
1755    }
1756}
1757
1758fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1759    format!("{}-safety-backup", restore.name_any())
1760}
1761
1762fn source_prep_job_name(restore: &KanidmRestore) -> String {
1763    format!("{}-source-prep", restore.name_any())
1764}
1765
1766fn source_check_job_name(restore: &KanidmRestore) -> String {
1767    format!("{}-source-check", restore.name_any())
1768}
1769
1770async fn ensure_source_check_job(
1771    restore: &KanidmRestore,
1772    target: &Kanidm,
1773    ctx: &RestoreContext,
1774    name: &str,
1775) -> Result<()> {
1776    let ns = restore.namespace().unwrap();
1777    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1778    if api
1779        .get_opt(name)
1780        .await
1781        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1782        .is_some()
1783    {
1784        return Ok(());
1785    }
1786    let file_name = restore
1787        .spec
1788        .source
1789        .local
1790        .as_ref()
1791        .map(|l| l.file_name.clone())
1792        .unwrap_or_default();
1793    let operation_doc = serde_json::json!({
1794        "apiVersion": "backup.kaniop.rs/v1alpha1",
1795        "kind": "OperationDocument",
1796        "operation": "check",
1797        "path": format!("{BACKUP_PATH}/{file_name}"),
1798        "resultPath": "/run/kaniop-result/result.json",
1799        "format": "kanidmJsonGzip",
1800    })
1801    .to_string();
1802    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1803    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1804    let data_mover_image = data_mover_image();
1805    let job = Job {
1806        metadata: ObjectMeta {
1807            name: Some(name.to_string()),
1808            namespace: Some(ns.clone()),
1809            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1810            ..Default::default()
1811        },
1812        spec: Some(JobSpec {
1813            backoff_limit: Some(0),
1814            template: PodTemplateSpec {
1815                metadata: Some(ObjectMeta::default()),
1816                spec: Some(PodSpec {
1817                    automount_service_account_token: Some(false),
1818                    restart_policy: Some("Never".to_string()),
1819                    security_context: Some(source_job_pod_security_context(target)),
1820                    containers: vec![Container {
1821                        name: "source-check".to_string(),
1822                        image: Some(data_mover_image),
1823                        command: Some(vec![
1824                            "/bin/kaniop-data-mover".to_string(),
1825                            "check".to_string(),
1826                            "--operation-doc".to_string(),
1827                            "/run/kaniop/operation.json".to_string(),
1828                        ]),
1829                        security_context: Some(hardened_security_context()),
1830                        resources: Some(job_resource_requirements()),
1831                        env: Some(vec![EnvVar {
1832                            name: "RUST_LOG".to_string(),
1833                            value: Some("info".to_string()),
1834                            ..Default::default()
1835                        }]),
1836                        volume_mounts: Some(vec![
1837                            VolumeMount {
1838                                name: DATA_VOLUME.to_string(),
1839                                mount_path: DATA_PATH.to_string(),
1840                                read_only: Some(true),
1841                                ..Default::default()
1842                            },
1843                            VolumeMount {
1844                                name: "operation".to_string(),
1845                                mount_path: "/run/kaniop".to_string(),
1846                                read_only: Some(true),
1847                                ..Default::default()
1848                            },
1849                            VolumeMount {
1850                                name: "result".to_string(),
1851                                mount_path: "/run/kaniop-result".to_string(),
1852                                ..Default::default()
1853                            },
1854                        ]),
1855                        ..Default::default()
1856                    }],
1857                    volumes: Some(vec![
1858                        Volume {
1859                            name: DATA_VOLUME.to_string(),
1860                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1861                                claim_name: primary_pvc_name(target)?,
1862                                read_only: Some(true),
1863                            }),
1864                            ..Default::default()
1865                        },
1866                        Volume {
1867                            name: "operation".to_string(),
1868                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1869                                name: operation_cm_name,
1870                                ..Default::default()
1871                            }),
1872                            ..Default::default()
1873                        },
1874                        Volume {
1875                            name: "result".to_string(),
1876                            empty_dir: Some(Default::default()),
1877                            ..Default::default()
1878                        },
1879                    ]),
1880                    ..Default::default()
1881                }),
1882            },
1883            ..Default::default()
1884        }),
1885        ..Default::default()
1886    };
1887    api.create(&PostParams::default(), &job)
1888        .await
1889        .map(|_| ())
1890        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1891}
1892
1893const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1894
1895fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1896    let ns = restore.namespace().unwrap_or_default();
1897    let uid = restore.uid().ok_or_else(|| {
1898        Error::MissingData(
1899            "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1900                .to_string(),
1901        )
1902    })?;
1903    let namespace_uuid =
1904        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1905    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1906    Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1907}
1908
1909/// Compute the safety-backup manifest key.
1910///
1911/// This key is hand-rolled with `format!` instead of going through
1912/// [`kaniop_backup_core::paths::RepositoryPath`], so it is the site most exposed to keyspace drift:
1913/// it must stay byte-identical to `RepositoryPath::manifest_key`, and its `tenants/{ns}` segment is
1914/// the Kubernetes namespace NAME (`restore.namespace()`), not the Namespace object UID, matching the
1915/// `namespaceUid` wire field the upload operation emits (misnomer tracked in issue #1084).
1916fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1917    let ns = restore.namespace().unwrap_or_default();
1918    let kanidm_uid = &restore.spec.target_ref.uid;
1919    let backup_id = compute_safety_backup_id(restore)?;
1920    Ok(format!(
1921        "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1922    ))
1923}
1924
1925struct VerifiedSafetyBackup {
1926    manifest_key: String,
1927    payload_sha256: String,
1928}
1929
1930async fn read_safety_backup_result(
1931    restore: &KanidmRestore,
1932    ctx: &RestoreContext,
1933    job_name: &str,
1934    expected_backup_id: &str,
1935) -> Result<VerifiedSafetyBackup> {
1936    let ns = restore.namespace().unwrap();
1937    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1938    let pod_list = pods
1939        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1940        .await
1941        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1942    let pod = pod_list.items.first().ok_or_else(|| {
1943        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1944    })?;
1945    let pod_name = pod.name_any();
1946    let container_status = pod
1947        .status
1948        .as_ref()
1949        .and_then(|s| s.container_statuses.as_ref())
1950        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1951        .ok_or_else(|| {
1952            Error::MissingData(format!(
1953                "safety-upload container status not found in pod {pod_name}"
1954            ))
1955        })?;
1956    let termination_message = container_status
1957        .state
1958        .as_ref()
1959        .and_then(|state| state.terminated.as_ref())
1960        .and_then(|t| t.message.as_ref())
1961        .ok_or_else(|| {
1962            Error::MissingData(format!(
1963                "termination message not found in pod {pod_name}; result transport failed"
1964            ))
1965        })?;
1966    if termination_message.is_empty() {
1967        return Err(Error::MissingData(format!(
1968            "termination message is empty in pod {pod_name}; result document was not written"
1969        )));
1970    }
1971    let result_doc = parse_result_document(termination_message).map_err(|e| {
1972        Error::ParseError(format!(
1973            "failed to parse result document from pod {pod_name}: {e}"
1974        ))
1975    })?;
1976    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1977        return Err(Error::ParseError(format!(
1978            "unsupported result document version: {}",
1979            result_doc.api_version
1980        )));
1981    }
1982    if result_doc.kind != "ResultDocument" {
1983        return Err(Error::ParseError(format!(
1984            "invalid result document kind: {}",
1985            result_doc.kind
1986        )));
1987    }
1988    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1989        return Err(Error::ParseError(format!(
1990            "result document operation mismatch: expected '{}', got '{}'",
1991            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1992        )));
1993    }
1994    if !result_doc.success {
1995        return Err(Error::ParseError(format!(
1996            "safety backup reported failure with exit code {:?}",
1997            result_doc.exit_code
1998        )));
1999    }
2000    if result_doc.exit_code != ExitCode::Success {
2001        return Err(Error::ParseError(format!(
2002            "safety backup exit code is not Success: {:?}",
2003            result_doc.exit_code
2004        )));
2005    }
2006    let doc_backup_id = result_doc
2007        .backup_id
2008        .as_deref()
2009        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2010    if doc_backup_id != expected_backup_id {
2011        return Err(Error::ParseError(format!(
2012            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2013        )));
2014    }
2015    let manifest_key = result_doc
2016        .manifest_key
2017        .clone()
2018        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2019    if manifest_key.is_empty() {
2020        return Err(Error::ParseError(
2021            "result document manifestKey is empty".to_string(),
2022        ));
2023    }
2024    let payload_sha256 = result_doc
2025        .payload_sha256
2026        .clone()
2027        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2028    if payload_sha256.is_empty() {
2029        return Err(Error::ParseError(
2030            "result document payloadSha256 is empty".to_string(),
2031        ));
2032    }
2033    info!(
2034        restore = %restore.name_any(),
2035        backup_id = %expected_backup_id,
2036        manifest_key = %manifest_key,
2037        "safety backup result verified"
2038    );
2039    Ok(VerifiedSafetyBackup {
2040        manifest_key,
2041        payload_sha256,
2042    })
2043}
2044
2045struct VerifiedSourcePrepResult {
2046    #[allow(dead_code)]
2047    manifest_key: String,
2048    payload_sha256: String,
2049}
2050
2051async fn read_source_prep_result(
2052    restore: &KanidmRestore,
2053    ctx: &RestoreContext,
2054    job_name: &str,
2055    expected_backup_id: &str,
2056) -> Result<VerifiedSourcePrepResult> {
2057    let ns = restore.namespace().unwrap();
2058    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2059    let pod_list = pods
2060        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2061        .await
2062        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2063    let pod = pod_list.items.first().ok_or_else(|| {
2064        Error::MissingData(format!("no pod found for source prep job {job_name}"))
2065    })?;
2066    let pod_name = pod.name_any();
2067    let container_status = pod
2068        .status
2069        .as_ref()
2070        .and_then(|s| s.container_statuses.as_ref())
2071        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2072        .ok_or_else(|| {
2073            Error::MissingData(format!(
2074                "source-prep container status not found in pod {pod_name}"
2075            ))
2076        })?;
2077    let termination_message = container_status
2078        .state
2079        .as_ref()
2080        .and_then(|state| state.terminated.as_ref())
2081        .and_then(|t| t.message.as_ref())
2082        .ok_or_else(|| {
2083            Error::MissingData(format!(
2084                "termination message not found in pod {pod_name}; result transport failed"
2085            ))
2086        })?;
2087    if termination_message.is_empty() {
2088        return Err(Error::MissingData(format!(
2089            "termination message is empty in pod {pod_name}; result document was not written"
2090        )));
2091    }
2092    let result_doc = parse_result_document(termination_message).map_err(|e| {
2093        Error::ParseError(format!(
2094            "failed to parse result document from pod {pod_name}: {e}"
2095        ))
2096    })?;
2097    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2098        return Err(Error::ParseError(format!(
2099            "unsupported result document version: {}",
2100            result_doc.api_version
2101        )));
2102    }
2103    if result_doc.kind != "ResultDocument" {
2104        return Err(Error::ParseError(format!(
2105            "invalid result document kind: {}",
2106            result_doc.kind
2107        )));
2108    }
2109    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2110        return Err(Error::ParseError(format!(
2111            "result document operation mismatch: expected '{}', got '{}'",
2112            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2113        )));
2114    }
2115    if !result_doc.success {
2116        return Err(Error::ParseError(format!(
2117            "source prep reported failure with exit code {:?}",
2118            result_doc.exit_code
2119        )));
2120    }
2121    if result_doc.exit_code != ExitCode::Success {
2122        return Err(Error::ParseError(format!(
2123            "source prep exit code is not Success: {:?}",
2124            result_doc.exit_code
2125        )));
2126    }
2127    let doc_backup_id = result_doc
2128        .backup_id
2129        .as_deref()
2130        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2131    if doc_backup_id != expected_backup_id {
2132        return Err(Error::ParseError(format!(
2133            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2134        )));
2135    }
2136    let manifest_key = result_doc
2137        .manifest_key
2138        .clone()
2139        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2140    if manifest_key.is_empty() {
2141        return Err(Error::ParseError(
2142            "result document manifestKey is empty".to_string(),
2143        ));
2144    }
2145    let payload_sha256 = result_doc
2146        .payload_sha256
2147        .clone()
2148        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2149    if payload_sha256.is_empty() {
2150        return Err(Error::ParseError(
2151            "result document payloadSha256 is empty".to_string(),
2152        ));
2153    }
2154    info!(
2155        restore = %restore.name_any(),
2156        backup_id = %expected_backup_id,
2157        manifest_key = %manifest_key,
2158        payload_sha256 = %payload_sha256,
2159        "source prep result verified"
2160    );
2161    Ok(VerifiedSourcePrepResult {
2162        manifest_key,
2163        payload_sha256,
2164    })
2165}
2166
2167fn source_job_pod_security_context(
2168    target: &Kanidm,
2169) -> k8s_openapi::api::core::v1::PodSecurityContext {
2170    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2171    security_context.seccomp_profile = Some(SeccompProfile {
2172        type_: "RuntimeDefault".to_string(),
2173        ..Default::default()
2174    });
2175    security_context
2176}
2177
2178fn hardened_security_context() -> SecurityContext {
2179    SecurityContext {
2180        allow_privilege_escalation: Some(false),
2181        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2182            drop: Some(vec!["ALL".to_string()]),
2183            ..Default::default()
2184        }),
2185        read_only_root_filesystem: Some(true),
2186        run_as_non_root: Some(true),
2187        seccomp_profile: Some(SeccompProfile {
2188            type_: "RuntimeDefault".to_string(),
2189            ..Default::default()
2190        }),
2191        ..Default::default()
2192    }
2193}
2194
2195fn kanidm_job_security_context() -> SecurityContext {
2196    SecurityContext {
2197        allow_privilege_escalation: Some(false),
2198        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2199            drop: Some(vec!["ALL".to_string()]),
2200            ..Default::default()
2201        }),
2202        seccomp_profile: Some(SeccompProfile {
2203            type_: "RuntimeDefault".to_string(),
2204            ..Default::default()
2205        }),
2206        ..Default::default()
2207    }
2208}
2209
2210fn job_resource_requirements() -> ResourceRequirements {
2211    ResourceRequirements {
2212        requests: Some(std::collections::BTreeMap::from([
2213            ("cpu".to_string(), Quantity("100m".to_string())),
2214            ("memory".to_string(), Quantity("128Mi".to_string())),
2215        ])),
2216        limits: Some(std::collections::BTreeMap::from([
2217            ("cpu".to_string(), Quantity("2".to_string())),
2218            ("memory".to_string(), Quantity("2Gi".to_string())),
2219        ])),
2220        ..Default::default()
2221    }
2222}
2223
2224// Status absence is not safe for new repositories. Legacy repositories with
2225// observedGeneration but no usageStarted remain locked by the admission webhook.
2226fn is_usage_lock_persisted(status: Option<&KanidmBackupRepositoryStatus>) -> bool {
2227    status.is_some_and(|status| {
2228        status.usage_started == Some(true)
2229            || (status.usage_started.is_none() && status.observed_generation.is_some())
2230    })
2231}
2232
2233async fn safety_repository_usage_locked(
2234    restore: &KanidmRestore,
2235    ctx: &RestoreContext,
2236) -> Result<bool> {
2237    let repository_name = restore
2238        .spec
2239        .safety_backup
2240        .as_ref()
2241        .and_then(|safety| safety.repository_ref.as_ref())
2242        .map(|reference| reference.name.as_str())
2243        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2244    let namespace = restore.namespace().unwrap_or_default();
2245    let repository = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &namespace)
2246        .get(repository_name)
2247        .await
2248        .map_err(|error| {
2249            Error::kube_error(
2250                "get",
2251                "KanidmBackupRepository",
2252                &namespace,
2253                repository_name,
2254                error,
2255            )
2256        })?;
2257    Ok(is_usage_lock_persisted(repository.status.as_ref()))
2258}
2259
2260async fn ensure_safety_backup_job(
2261    restore: &KanidmRestore,
2262    target: &Kanidm,
2263    ctx: &RestoreContext,
2264    name: &str,
2265    backup_id: &str,
2266    manifest_key: &str,
2267) -> Result<()> {
2268    let ns = restore.namespace().unwrap();
2269    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2270    if api
2271        .get_opt(name)
2272        .await
2273        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2274        .is_some()
2275    {
2276        return Ok(());
2277    }
2278    let safety = restore
2279        .spec
2280        .safety_backup
2281        .as_ref()
2282        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2283    let repo_ref = safety
2284        .repository_ref
2285        .as_ref()
2286        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2287    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2288        .get(&repo_ref.name)
2289        .await
2290        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2291    let _kanidm_uid = &restore.spec.target_ref.uid;
2292    let endpoint = &repo.spec.s3.endpoint;
2293    let region = &repo.spec.s3.region;
2294    let operation_doc = build_safety_upload_operation_doc(
2295        restore,
2296        target,
2297        backup_id,
2298        manifest_key,
2299        &repo.spec.s3.bucket,
2300        &repo.spec.s3.prefix,
2301        endpoint,
2302        region,
2303        repo.spec.s3.force_path_style,
2304        repo.spec.s3.insecure,
2305        repo.spec
2306            .s3
2307            .ca_bundle_ref
2308            .as_ref()
2309            .map(|_| ca_bundle_path())
2310            .as_deref(),
2311        repo.spec.encryption.as_ref(),
2312    )?;
2313    let operation_cm_name = format!("{name}-op");
2314    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2315    let auth_method = &repo.spec.authentication.writer;
2316    let data_mover_image = data_mover_image();
2317    let init_command = vec![
2318        "kanidmd".to_string(),
2319        "database".to_string(),
2320        "backup".to_string(),
2321        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2322    ];
2323    let upload_script = format!(
2324        r#"set -eu
2325/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2326RESULT_FILE="/run/kaniop-result/result.json"
2327TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2328if [ ! -f "$RESULT_FILE" ]; then
2329  echo "ERROR: result document not found at $RESULT_FILE"
2330  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2331  exit 1
2332fi
2333if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2334  echo "ERROR: safety backup upload reported failure"
2335  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2336  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2337  exit 1
2338fi
2339dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2340echo "safety backup upload completed successfully"
2341"#
2342    );
2343    let job = Job {
2344        metadata: ObjectMeta {
2345            name: Some(name.to_string()),
2346            namespace: Some(ns.clone()),
2347            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2348            ..Default::default()
2349        },
2350        spec: Some(JobSpec {
2351            backoff_limit: Some(0),
2352            template: PodTemplateSpec {
2353                metadata: Some(ObjectMeta::default()),
2354                spec: Some(PodSpec {
2355                    automount_service_account_token: Some(false),
2356                    restart_policy: Some("Never".to_string()),
2357                    security_context: Some(source_job_pod_security_context(target)),
2358                    init_containers: Some(vec![Container {
2359                        name: "safety-backup".to_string(),
2360                        image: Some(restore.spec.restore_image.clone()),
2361                        command: Some(init_command),
2362                        security_context: Some(SecurityContext {
2363                            allow_privilege_escalation: Some(false),
2364                            capabilities: Some(Capabilities {
2365                                drop: Some(vec!["ALL".to_string()]),
2366                                ..Default::default()
2367                            }),
2368                            read_only_root_filesystem: Some(true),
2369                            seccomp_profile: Some(SeccompProfile {
2370                                type_: "RuntimeDefault".to_string(),
2371                                ..Default::default()
2372                            }),
2373                            ..Default::default()
2374                        }),
2375                        resources: Some(job_resource_requirements()),
2376                        env: Some(build_kanidm_db_env_vars(target)),
2377                        volume_mounts: Some(vec![
2378                            VolumeMount {
2379                                name: DATA_VOLUME.to_string(),
2380                                mount_path: DATA_PATH.to_string(),
2381                                read_only: Some(false),
2382                                ..Default::default()
2383                            },
2384                            VolumeMount {
2385                                name: TLS_VOLUME.to_string(),
2386                                mount_path: TLS_PATH.to_string(),
2387                                read_only: Some(true),
2388                                ..Default::default()
2389                            },
2390                            VolumeMount {
2391                                name: SHARED_VOLUME.to_string(),
2392                                mount_path: SHARED_VOL_PATH.to_string(),
2393                                ..Default::default()
2394                            },
2395                        ]),
2396                        ..Default::default()
2397                    }]),
2398                    containers: vec![Container {
2399                        name: "safety-upload".to_string(),
2400                        image: Some(data_mover_image),
2401                        command: Some(vec!["/bin/sh".to_string()]),
2402                        args: Some(vec!["-c".to_string(), upload_script]),
2403                        security_context: Some(hardened_security_context()),
2404                        resources: Some(job_resource_requirements()),
2405                        env: {
2406                            let mut env_vars =
2407                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2408                            env_vars.push(EnvVar {
2409                                name: "RUST_LOG".to_string(),
2410                                value: Some("info".to_string()),
2411                                ..Default::default()
2412                            });
2413                            if repo.spec.s3.ca_bundle_ref.is_some() {
2414                                env_vars.push(ca_bundle_env_var());
2415                            }
2416                            env_vars.extend(build_encryption_env_vars(
2417                                repo.spec
2418                                    .encryption
2419                                    .as_ref()
2420                                    .and_then(|e| e.key_ref.as_ref()),
2421                            ));
2422                            Some(env_vars)
2423                        },
2424                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2425                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2426                        volume_mounts: {
2427                            let mut mounts = vec![
2428                                VolumeMount {
2429                                    name: SHARED_VOLUME.to_string(),
2430                                    mount_path: SHARED_VOL_PATH.to_string(),
2431                                    read_only: Some(true),
2432                                    ..Default::default()
2433                                },
2434                                VolumeMount {
2435                                    name: "operation".to_string(),
2436                                    mount_path: "/run/kaniop".to_string(),
2437                                    read_only: Some(true),
2438                                    ..Default::default()
2439                                },
2440                                VolumeMount {
2441                                    name: "result".to_string(),
2442                                    mount_path: "/run/kaniop-result".to_string(),
2443                                    ..Default::default()
2444                                },
2445                            ];
2446                            mounts.extend(build_auth_volume_mounts(auth_method));
2447                            if repo.spec.s3.ca_bundle_ref.is_some() {
2448                                mounts.push(build_ca_bundle_volume_mount());
2449                            }
2450                            Some(mounts)
2451                        },
2452                        ..Default::default()
2453                    }],
2454                    volumes: {
2455                        let mut vols = vec![
2456                            Volume {
2457                                name: DATA_VOLUME.to_string(),
2458                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2459                                    claim_name: primary_pvc_name(target)?,
2460                                    read_only: Some(false),
2461                                }),
2462                                ..Default::default()
2463                            },
2464                            Volume {
2465                                name: TLS_VOLUME.to_string(),
2466                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2467                                    secret_name: Some(target.effective_tls_secret_name()),
2468                                    ..Default::default()
2469                                }),
2470                                ..Default::default()
2471                            },
2472                            Volume {
2473                                name: SHARED_VOLUME.to_string(),
2474                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2475                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2476                                    ..Default::default()
2477                                }),
2478                                ..Default::default()
2479                            },
2480                            Volume {
2481                                name: "operation".to_string(),
2482                                config_map: Some(ConfigMapVolumeSource {
2483                                    name: operation_cm_name,
2484                                    ..Default::default()
2485                                }),
2486                                ..Default::default()
2487                            },
2488                            Volume {
2489                                name: "result".to_string(),
2490                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2491                                    size_limit: Some(Quantity("16Mi".to_string())),
2492                                    ..Default::default()
2493                                }),
2494                                ..Default::default()
2495                            },
2496                        ];
2497                        vols.extend(build_auth_volumes(auth_method));
2498                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2499                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2500                        }
2501                        Some(vols)
2502                    },
2503                    ..Default::default()
2504                }),
2505            },
2506            ..Default::default()
2507        }),
2508        ..Default::default()
2509    };
2510    api.create(&PostParams::default(), &job)
2511        .await
2512        .map(|_| ())
2513        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2514}
2515
2516async fn ensure_source_prep_job(
2517    restore: &KanidmRestore,
2518    target: &Kanidm,
2519    ctx: &RestoreContext,
2520    name: &str,
2521) -> Result<()> {
2522    let ns = restore.namespace().unwrap();
2523    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2524    if api
2525        .get_opt(name)
2526        .await
2527        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2528        .is_some()
2529    {
2530        return Ok(());
2531    }
2532    let backup_name = restore
2533        .spec
2534        .source
2535        .backup_ref
2536        .as_ref()
2537        .map(|b| b.name.clone())
2538        .unwrap_or_default();
2539    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2540        .get(&backup_name)
2541        .await
2542        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2543    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2544        .get(&backup.spec.repository_ref.name)
2545        .await
2546        .map_err(|e| {
2547            Error::kube_error(
2548                "get",
2549                "KanidmBackupRepository",
2550                &ns,
2551                &backup.spec.repository_ref.name,
2552                e,
2553            )
2554        })?;
2555    let endpoint = &repo.spec.s3.endpoint;
2556    let region = &repo.spec.s3.region;
2557    let operation_doc = build_download_operation_doc(
2558        target,
2559        &backup.spec.manifest_key,
2560        &backup.spec.backup_id,
2561        &backup.spec.kanidm_ref.uid,
2562        &repo.spec.s3.bucket,
2563        &repo.spec.s3.prefix,
2564        endpoint,
2565        region,
2566        repo.spec.s3.force_path_style,
2567        repo.spec.s3.insecure,
2568        repo.spec
2569            .s3
2570            .ca_bundle_ref
2571            .as_ref()
2572            .map(|_| ca_bundle_path())
2573            .as_deref(),
2574        repo.spec.encryption.as_ref(),
2575    );
2576    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2577    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2578    let auth_method = &repo.spec.authentication.reader;
2579    let data_mover_image = data_mover_image();
2580    let download_script = format!(
2581        r#"set -eu
2582/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2583RESULT_FILE="/run/kaniop-result/result.json"
2584TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2585if [ ! -f "$RESULT_FILE" ]; then
2586  echo "ERROR: result document not found at $RESULT_FILE"
2587  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2588  exit 1
2589fi
2590if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2591  echo "ERROR: source preparation download reported failure"
2592  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2593  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2594  exit 1
2595fi
2596dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2597echo "source preparation download completed successfully"
2598"#
2599    );
2600    let job = Job {
2601        metadata: ObjectMeta {
2602            name: Some(name.to_string()),
2603            namespace: Some(ns.clone()),
2604            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2605            ..Default::default()
2606        },
2607        spec: Some(JobSpec {
2608            backoff_limit: Some(0),
2609            template: PodTemplateSpec {
2610                metadata: Some(ObjectMeta::default()),
2611                spec: Some(PodSpec {
2612                    automount_service_account_token: Some(false),
2613                    restart_policy: Some("Never".to_string()),
2614                    security_context: Some(source_job_pod_security_context(target)),
2615                    containers: vec![Container {
2616                        name: "source-prep".to_string(),
2617                        image: Some(data_mover_image),
2618                        command: Some(vec!["/bin/sh".to_string()]),
2619                        args: Some(vec!["-c".to_string(), download_script]),
2620                        security_context: Some(hardened_security_context()),
2621                        resources: Some(job_resource_requirements()),
2622                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2623                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2624                        env: {
2625                            let mut env_vars = build_auth_env_vars(
2626                                auth_method,
2627                                &backup.spec.repository_ref.name,
2628                                AuthRole::Reader,
2629                            );
2630                            env_vars.push(EnvVar {
2631                                name: "RUST_LOG".to_string(),
2632                                value: Some("info".to_string()),
2633                                ..Default::default()
2634                            });
2635                            if repo.spec.s3.ca_bundle_ref.is_some() {
2636                                env_vars.push(ca_bundle_env_var());
2637                            }
2638                            env_vars.extend(build_encryption_env_vars(
2639                                repo.spec
2640                                    .encryption
2641                                    .as_ref()
2642                                    .and_then(|e| e.key_ref.as_ref()),
2643                            ));
2644                            Some(env_vars)
2645                        },
2646                        volume_mounts: {
2647                            let mut mounts = vec![
2648                                VolumeMount {
2649                                    name: DATA_VOLUME.to_string(),
2650                                    mount_path: DATA_PATH.to_string(),
2651                                    ..Default::default()
2652                                },
2653                                VolumeMount {
2654                                    name: "operation".to_string(),
2655                                    mount_path: "/run/kaniop".to_string(),
2656                                    read_only: Some(true),
2657                                    ..Default::default()
2658                                },
2659                                VolumeMount {
2660                                    name: "result".to_string(),
2661                                    mount_path: "/run/kaniop-result".to_string(),
2662                                    ..Default::default()
2663                                },
2664                            ];
2665                            mounts.extend(build_auth_volume_mounts(auth_method));
2666                            if repo.spec.s3.ca_bundle_ref.is_some() {
2667                                mounts.push(build_ca_bundle_volume_mount());
2668                            }
2669                            Some(mounts)
2670                        },
2671                        ..Default::default()
2672                    }],
2673                    volumes: {
2674                        let mut vols = vec![
2675                            Volume {
2676                                name: DATA_VOLUME.to_string(),
2677                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2678                                    claim_name: primary_pvc_name(target)?,
2679                                    read_only: Some(false),
2680                                }),
2681                                ..Default::default()
2682                            },
2683                            Volume {
2684                                name: "operation".to_string(),
2685                                config_map: Some(ConfigMapVolumeSource {
2686                                    name: operation_cm_name,
2687                                    ..Default::default()
2688                                }),
2689                                ..Default::default()
2690                            },
2691                            Volume {
2692                                name: "result".to_string(),
2693                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2694                                    size_limit: Some(Quantity("16Mi".to_string())),
2695                                    ..Default::default()
2696                                }),
2697                                ..Default::default()
2698                            },
2699                        ];
2700                        vols.extend(build_auth_volumes(auth_method));
2701                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2702                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2703                        }
2704                        Some(vols)
2705                    },
2706                    ..Default::default()
2707                }),
2708            },
2709            ..Default::default()
2710        }),
2711        ..Default::default()
2712    };
2713    api.create(&PostParams::default(), &job)
2714        .await
2715        .map(|_| ())
2716        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2717}
2718
2719#[allow(clippy::too_many_arguments)]
2720fn build_safety_upload_operation_doc(
2721    restore: &KanidmRestore,
2722    target: &Kanidm,
2723    backup_id: &str,
2724    manifest_key: &str,
2725    bucket: &str,
2726    prefix: &str,
2727    endpoint: &str,
2728    region: &str,
2729    force_path_style: bool,
2730    insecure: bool,
2731    ca_bundle_path: Option<&str>,
2732    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2733) -> Result<String> {
2734    let enc_mode =
2735        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2736    let enc_key_id = encryption
2737        .and_then(|e| e.key_id.as_ref())
2738        .map(|s| serde_json::Value::String(s.clone()));
2739    let op = UploadOperation {
2740        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2741        bucket: bucket.to_string(),
2742        prefix: prefix.to_string(),
2743        endpoint: endpoint.to_string(),
2744        region: region.to_string(),
2745        force_path_style,
2746        insecure,
2747        ca_bundle_path: ca_bundle_path.map(str::to_string),
2748        backup_id: backup_id.to_string(),
2749        // `UploadOperation.namespace_uid` carries the Kubernetes namespace NAME (`restore.namespace()`),
2750        // not the Namespace object UID: it selects the `tenants/<x>` keyspace segment and matches the
2751        // `namespaceUid` wire field name, which is frozen for compatibility (issue #1084).
2752        namespace_uid: restore.namespace().unwrap_or_default(),
2753        kanidm_uid: restore.spec.target_ref.uid.clone(),
2754        kanidm_name: restore.spec.target_ref.name.clone(),
2755        domain: target.spec.domain.clone(),
2756        kanidm_version: target
2757            .status
2758            .as_ref()
2759            .and_then(|s| s.version.as_ref())
2760            .map(|v| v.image_tag.clone())
2761            .unwrap_or_default(),
2762        image_digest: None,
2763        consistency: "kanidm-offline".to_string(),
2764        reason: "restore-safety".to_string(),
2765        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2766        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2767        result_path: "/run/kaniop-result/result.json".to_string(),
2768        max_concurrent_parts: 4,
2769        max_retries: 3,
2770    };
2771    let _ = manifest_key;
2772    let doc = OperationDocument {
2773        api_version: OPERATION_DOC_VERSION.to_string(),
2774        kind: "OperationDocument".to_string(),
2775        spec: OperationSpec::Upload(op),
2776    };
2777    serde_json::to_string(&doc)
2778        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2779}
2780
2781async fn validate_backup_ref(
2782    restore: &KanidmRestore,
2783    target: &Kanidm,
2784    ctx: &RestoreContext,
2785) -> Result<Option<DisasterRecoverySource>> {
2786    let backup_ref = restore
2787        .spec
2788        .source
2789        .backup_ref
2790        .as_ref()
2791        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2792    let ns = restore.namespace().unwrap();
2793    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2794        .get(&backup_ref.name)
2795        .await
2796        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2797    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2798        return Err(Error::MissingData(format!(
2799            "KanidmBackup '{}' is not Ready (phase: {:?})",
2800            backup_ref.name,
2801            backup.status.as_ref().map(|s| &s.phase)
2802        )));
2803    }
2804    let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2805        || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2806    let disaster_recovery_source = if source_identity_mismatch {
2807        if !disaster_recovery_requested(restore) {
2808            return Err(Error::MissingData(format!(
2809                "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2810                backup.spec.kanidm_ref.name,
2811                backup.spec.kanidm_ref.uid,
2812                restore.spec.target_ref.name,
2813                restore.spec.target_ref.uid,
2814                DISASTER_RECOVERY_ANNOTATION,
2815                BREAK_GLASS_REASON_ANNOTATION,
2816                BREAK_GLASS_APPROVED_BY_ANNOTATION,
2817            )));
2818        }
2819        validate_disaster_recovery_approval(restore)?;
2820        Some(DisasterRecoverySource {
2821            name: backup.spec.kanidm_ref.name.clone(),
2822            uid: backup.spec.kanidm_ref.uid.clone(),
2823        })
2824    } else {
2825        None
2826    };
2827    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2828        .get(&backup.spec.repository_ref.name)
2829        .await
2830        .map_err(|e| {
2831            Error::kube_error(
2832                "get",
2833                "KanidmBackupRepository",
2834                &ns,
2835                &backup.spec.repository_ref.name,
2836                e,
2837            )
2838        })?;
2839    if !has_accepted_condition(
2840        &repo
2841            .status
2842            .as_ref()
2843            .map(|s| &s.conditions)
2844            .cloned()
2845            .unwrap_or_default(),
2846    ) {
2847        return Err(Error::MissingData(format!(
2848            "KanidmBackupRepository '{}' configuration has not been accepted",
2849            backup.spec.repository_ref.name
2850        )));
2851    }
2852    validate_backup_compatibility(restore, target, &backup)?;
2853    Ok(disaster_recovery_source)
2854}
2855
2856fn has_accepted_condition(conditions: &[Condition]) -> bool {
2857    conditions
2858        .iter()
2859        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2860}
2861
2862fn validate_backup_compatibility(
2863    restore: &KanidmRestore,
2864    target: &Kanidm,
2865    backup: &KanidmBackup,
2866) -> Result<()> {
2867    let status = backup.status.as_ref();
2868    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2869        let target_version = target
2870            .status
2871            .as_ref()
2872            .and_then(|s| s.version.as_ref())
2873            .map(|v| v.image_tag.as_str());
2874        if let Some(tv) = target_version {
2875            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2876                return Err(Error::MissingData(format!(
2877                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2878                )));
2879            }
2880        }
2881    }
2882    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2883        if !backup_digest.is_empty()
2884            && restore.spec.restore_image.contains('@')
2885            && !restore.spec.restore_image.contains(backup_digest)
2886        {
2887            return Err(Error::MissingData(format!(
2888                "restore image digest does not match backup image digest '{backup_digest}'"
2889            )));
2890        }
2891    }
2892    Ok(())
2893}
2894
2895#[allow(clippy::too_many_arguments)]
2896fn build_download_operation_doc(
2897    target: &Kanidm,
2898    manifest_key: &str,
2899    expected_backup_id: &str,
2900    expected_source_uid: &str,
2901    bucket: &str,
2902    prefix: &str,
2903    endpoint: &str,
2904    region: &str,
2905    force_path_style: bool,
2906    insecure: bool,
2907    ca_bundle_path: Option<&str>,
2908    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2909) -> String {
2910    let enc_mode =
2911        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2912    let enc_key_id = encryption
2913        .and_then(|e| e.key_id.as_ref())
2914        .map(|s| serde_json::Value::String(s.clone()));
2915    serde_json::json!({
2916        "apiVersion": "backup.kaniop.rs/v1alpha1",
2917        "kind": "OperationDocument",
2918        "operation": "download",
2919        "manifestKey": manifest_key,
2920        "bucket": bucket,
2921        "prefix": prefix,
2922        "endpoint": endpoint,
2923        "region": region,
2924        "forcePathStyle": force_path_style,
2925        "insecure": insecure,
2926        "caBundlePath": ca_bundle_path,
2927        "expectedBackupId": expected_backup_id,
2928        "expectedKanidmUid": expected_source_uid,
2929        "expectedDomain": target.spec.domain,
2930        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2931        "resultPath": "/run/kaniop-result/result.json",
2932        "maxRetries": 3,
2933        "encryptionMode": enc_mode,
2934        "encryptionKeyId": enc_key_id,
2935    })
2936    .to_string()
2937}
2938
2939async fn ensure_operation_configmap(
2940    restore: &KanidmRestore,
2941    name: &str,
2942    data: &str,
2943    ns: &str,
2944    ctx: &RestoreContext,
2945) -> Result<()> {
2946    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2947    if api
2948        .get_opt(name)
2949        .await
2950        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2951        .is_some()
2952    {
2953        return Ok(());
2954    }
2955    let cm = ConfigMap {
2956        metadata: ObjectMeta {
2957            name: Some(name.to_string()),
2958            namespace: Some(ns.to_string()),
2959            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2960            ..Default::default()
2961        },
2962        data: Some(std::collections::BTreeMap::from([(
2963            "operation.json".to_string(),
2964            data.to_string(),
2965        )])),
2966        ..Default::default()
2967    };
2968    api.create(&PostParams::default(), &cm)
2969        .await
2970        .map(|_| ())
2971        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2972}
2973
2974#[cfg(test)]
2975mod tests {
2976    use super::{
2977        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2978        CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2979        DISASTER_RECOVERY_ANNOTATION,
2980        KanidmRestore, KanidmRestorePhase,
2981        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2982        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2983        disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2984        is_usage_lock_persisted,
2985        is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2986        restore_job_name, safe_basename, validate_disaster_recovery_approval,
2987        validate_safety_backup_config, validate_source, verify_job_name,
2988    };
2989    use kaniop_backup_core::crd::KanidmBackupRepositoryStatus;
2990    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2991    use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2992    use k8s_openapi::jiff::Timestamp;
2993    use kube::api::ObjectMeta as ApiObjectMeta;
2994    use std::collections::BTreeMap;
2995
2996    fn make_restore(
2997        source: KanidmRestoreSource,
2998        safety: Option<SafetyBackupConfig>,
2999    ) -> KanidmRestore {
3000        KanidmRestore {
3001            metadata: ApiObjectMeta {
3002                name: Some("test-restore".to_string()),
3003                namespace: Some("default".to_string()),
3004                uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
3005                ..Default::default()
3006            },
3007            spec: KanidmRestoreSpec {
3008                target_ref: KanidmRestoreTargetRef {
3009                    name: "corp-idm".to_string(),
3010                    uid: "test-uid-123".to_string(),
3011                },
3012                source,
3013                restore_image: "kanidm/server@sha256:abc".to_string(),
3014                safety_backup: safety,
3015            },
3016            status: None,
3017        }
3018    }
3019
3020    #[test]
3021    fn safety_backup_waits_for_persisted_repository_lock() {
3022        assert!(!is_usage_lock_persisted(None));
3023        let mut status = KanidmBackupRepositoryStatus {
3024            observed_generation: Some(1),
3025            usage_started: Some(false),
3026            ..Default::default()
3027        };
3028        assert!(!is_usage_lock_persisted(Some(&status)));
3029        status.usage_started = Some(true);
3030        assert!(is_usage_lock_persisted(Some(&status)));
3031        // Existing repositories remain fail-closed if the new field is absent.
3032        status.usage_started = None;
3033        assert!(is_usage_lock_persisted(Some(&status)));
3034        status.observed_generation = None;
3035        assert!(!is_usage_lock_persisted(Some(&status)));
3036    }
3037
3038    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
3039        super::super::crd::Kanidm {
3040            status: Some(super::super::crd::KanidmStatus {
3041                available_replicas: 1,
3042                replicas: 1,
3043                unavailable_replicas: 0,
3044                updated_replicas: 1,
3045                replica_statuses: vec![],
3046                replica_column: "1/1".to_string(),
3047                secret_name: None,
3048                version: Some(super::super::crd::KanidmVersionStatus {
3049                    image_tag: image_tag.to_string(),
3050                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
3051                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
3052                }),
3053                domain_appearance_image: None,
3054                mail_sender: None,
3055                conditions: None,
3056            }),
3057            ..Default::default()
3058        }
3059    }
3060
3061    #[test]
3062    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
3063        assert!(!KanidmRestoreStatus::default().database_mutation_started);
3064    }
3065
3066    #[test]
3067    fn completed_restore_sets_ready_condition() {
3068        let mut status = KanidmRestoreStatus {
3069            phase: KanidmRestorePhase::Completed,
3070            ..Default::default()
3071        };
3072        super::update_restore_conditions(&mut status, Some(7));
3073        assert!(status.conditions.iter().any(|condition| {
3074            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3075        }));
3076        assert!(!status.conditions.iter().any(|condition| {
3077            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3078        }));
3079    }
3080
3081    #[test]
3082    fn disaster_recovery_condition_survives_phase_updates() {
3083        let dr_condition = Condition {
3084            type_: CONDITION_DISASTER_RECOVERY.to_string(),
3085            status: CONDITION_TRUE.to_string(),
3086            reason: "ApprovedSourceIdentityMismatch".to_string(),
3087            message: "approved DR".to_string(),
3088            last_transition_time: Time(Timestamp::now()),
3089            observed_generation: Some(7),
3090        };
3091        let mut status = KanidmRestoreStatus {
3092            phase: KanidmRestorePhase::Quiescing,
3093            conditions: vec![dr_condition.clone()],
3094            ..Default::default()
3095        };
3096
3097        super::update_restore_conditions(&mut status, Some(8));
3098
3099        assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3100    }
3101
3102    #[test]
3103    fn failed_restore_sets_failed_condition() {
3104        let mut status = KanidmRestoreStatus {
3105            phase: KanidmRestorePhase::Failed,
3106            message: Some("verification failed".to_string()),
3107            ..Default::default()
3108        };
3109        super::update_restore_conditions(&mut status, Some(8));
3110        assert!(status.conditions.iter().any(|condition| {
3111            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3112        }));
3113    }
3114
3115    #[test]
3116    fn restore_filename_is_confined_to_backup_directory() {
3117        assert!(safe_basename("backup.json.gz"));
3118        assert!(!safe_basename("../backup.json.gz"));
3119        assert!(!safe_basename("a/b"));
3120        assert!(!safe_basename(""));
3121    }
3122
3123    #[test]
3124    fn restore_image_rejects_latest_and_unpinned_names() {
3125        assert!(mutable_image("kanidm/server:latest"));
3126        assert!(mutable_image("kanidm/server"));
3127        assert!(!mutable_image("kanidm/server:1.10.0"));
3128        assert!(!mutable_image("kanidm/server:1.11.2@sha256:serverdigest"));
3129        assert!(!mutable_image("kanidm/server@sha256:abc"));
3130        assert!(mutable_image("registry.example.com:5000/kanidm/server"));
3131    }
3132
3133    #[test]
3134    fn source_validation_rejects_both_local_and_backup_ref() {
3135        let source = KanidmRestoreSource {
3136            local: Some(super::KanidmRestoreLocalSource {
3137                file_name: "backup.json".to_string(),
3138            }),
3139            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3140                name: "some-backup".to_string(),
3141            }),
3142        };
3143        let restore = make_restore(source, None);
3144        assert!(validate_source(&restore).is_err());
3145    }
3146
3147    #[test]
3148    fn source_validation_rejects_empty_source() {
3149        let source = KanidmRestoreSource {
3150            local: None,
3151            backup_ref: None,
3152        };
3153        let restore = make_restore(source, None);
3154        assert!(validate_source(&restore).is_err());
3155    }
3156
3157    #[test]
3158    fn source_validation_accepts_local_source() {
3159        let source = KanidmRestoreSource {
3160            local: Some(super::KanidmRestoreLocalSource {
3161                file_name: "backup.json".to_string(),
3162            }),
3163            backup_ref: None,
3164        };
3165        let restore = make_restore(source, None);
3166        assert!(validate_source(&restore).is_ok());
3167    }
3168
3169    #[test]
3170    fn source_validation_accepts_backup_ref_source() {
3171        let source = KanidmRestoreSource {
3172            local: None,
3173            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3174                name: "corp-idm-019c7c76".to_string(),
3175            }),
3176        };
3177        let restore = make_restore(source, None);
3178        assert!(validate_source(&restore).is_ok());
3179    }
3180
3181    #[test]
3182    fn source_validation_rejects_empty_backup_ref_name() {
3183        let source = KanidmRestoreSource {
3184            local: None,
3185            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3186                name: "".to_string(),
3187            }),
3188        };
3189        let restore = make_restore(source, None);
3190        assert!(validate_source(&restore).is_err());
3191    }
3192
3193    #[test]
3194    fn is_remote_source_true_for_backup_ref() {
3195        let source = KanidmRestoreSource {
3196            local: None,
3197            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3198                name: "backup-1".to_string(),
3199            }),
3200        };
3201        let restore = make_restore(source, None);
3202        assert!(is_remote_source(&restore));
3203    }
3204
3205    #[test]
3206    fn is_remote_source_false_for_local() {
3207        let source = KanidmRestoreSource {
3208            local: Some(super::KanidmRestoreLocalSource {
3209                file_name: "backup.json".to_string(),
3210            }),
3211            backup_ref: None,
3212        };
3213        let restore = make_restore(source, None);
3214        assert!(!is_remote_source(&restore));
3215    }
3216
3217    #[test]
3218    fn requires_safety_backup_default_true() {
3219        let source = KanidmRestoreSource {
3220            local: None,
3221            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3222                name: "backup-1".to_string(),
3223            }),
3224        };
3225        let restore = make_restore(source, None);
3226        assert!(requires_safety_backup(&restore));
3227    }
3228
3229    #[test]
3230    fn requires_safety_backup_false_when_skip() {
3231        let source = KanidmRestoreSource {
3232            local: None,
3233            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3234                name: "backup-1".to_string(),
3235            }),
3236        };
3237        let safety = SafetyBackupConfig {
3238            repository_ref: Some(SafetyBackupRepositoryRef {
3239                name: "offsite".to_string(),
3240            }),
3241            skip: true,
3242        };
3243        let restore = make_restore(source, Some(safety));
3244        assert!(!requires_safety_backup(&restore));
3245    }
3246
3247    #[test]
3248    fn break_glass_validation_rejects_skip_without_reason() {
3249        let source = KanidmRestoreSource {
3250            local: None,
3251            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3252                name: "backup-1".to_string(),
3253            }),
3254        };
3255        let safety = SafetyBackupConfig {
3256            repository_ref: Some(SafetyBackupRepositoryRef {
3257                name: "offsite".to_string(),
3258            }),
3259            skip: true,
3260        };
3261        let mut restore = make_restore(source, Some(safety));
3262        restore.metadata.annotations = Some(BTreeMap::from([(
3263            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3264            "commander@example.com".to_string(),
3265        )]));
3266        assert!(validate_safety_backup_config(&restore).is_err());
3267    }
3268
3269    #[test]
3270    fn break_glass_validation_rejects_skip_without_approver() {
3271        let source = KanidmRestoreSource {
3272            local: None,
3273            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3274                name: "backup-1".to_string(),
3275            }),
3276        };
3277        let safety = SafetyBackupConfig {
3278            repository_ref: Some(SafetyBackupRepositoryRef {
3279                name: "offsite".to_string(),
3280            }),
3281            skip: true,
3282        };
3283        let mut restore = make_restore(source, Some(safety));
3284        restore.metadata.annotations = Some(BTreeMap::from([(
3285            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3286            "PVC is unreadable".to_string(),
3287        )]));
3288        assert!(validate_safety_backup_config(&restore).is_err());
3289    }
3290
3291    #[test]
3292    fn break_glass_validation_accepts_valid_annotations() {
3293        let source = KanidmRestoreSource {
3294            local: None,
3295            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3296                name: "backup-1".to_string(),
3297            }),
3298        };
3299        let safety = SafetyBackupConfig {
3300            repository_ref: Some(SafetyBackupRepositoryRef {
3301                name: "offsite".to_string(),
3302            }),
3303            skip: true,
3304        };
3305        let mut restore = make_restore(source, Some(safety));
3306        restore.metadata.annotations = Some(BTreeMap::from([
3307            (
3308                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3309                "PVC is unreadable".to_string(),
3310            ),
3311            (
3312                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3313                "commander@example.com".to_string(),
3314            ),
3315        ]));
3316        assert!(validate_safety_backup_config(&restore).is_ok());
3317    }
3318
3319    #[test]
3320    fn disaster_recovery_requires_explicit_approval_annotations() {
3321        let source = KanidmRestoreSource {
3322            local: None,
3323            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3324                name: "backup-1".to_string(),
3325            }),
3326        };
3327        let mut restore = make_restore(source, None);
3328        restore.metadata.annotations = Some(BTreeMap::from([(
3329            DISASTER_RECOVERY_ANNOTATION.to_string(),
3330            "true".to_string(),
3331        )]));
3332        assert!(disaster_recovery_requested(&restore));
3333        assert!(validate_disaster_recovery_approval(&restore).is_err());
3334    }
3335
3336    #[test]
3337    fn disaster_recovery_accepts_reason_and_approver() {
3338        let source = KanidmRestoreSource {
3339            local: None,
3340            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3341                name: "backup-1".to_string(),
3342            }),
3343        };
3344        let mut restore = make_restore(source, None);
3345        restore.metadata.annotations = Some(BTreeMap::from([
3346            (
3347                DISASTER_RECOVERY_ANNOTATION.to_string(),
3348                "true".to_string(),
3349            ),
3350            (
3351                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3352                "original cluster lost".to_string(),
3353            ),
3354            (
3355                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3356                "incident-commander".to_string(),
3357            ),
3358        ]));
3359        assert!(disaster_recovery_requested(&restore));
3360        assert!(validate_disaster_recovery_approval(&restore).is_ok());
3361    }
3362
3363    #[test]
3364    fn remote_source_requires_repository_ref_when_not_skipping() {
3365        let source = KanidmRestoreSource {
3366            local: None,
3367            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3368                name: "backup-1".to_string(),
3369            }),
3370        };
3371        let safety = SafetyBackupConfig {
3372            repository_ref: None,
3373            skip: false,
3374        };
3375        let restore = make_restore(source, Some(safety));
3376        assert!(validate_safety_backup_config(&restore).is_err());
3377    }
3378
3379    #[test]
3380    fn remote_source_accepts_repository_ref() {
3381        let source = KanidmRestoreSource {
3382            local: None,
3383            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3384                name: "backup-1".to_string(),
3385            }),
3386        };
3387        let safety = SafetyBackupConfig {
3388            repository_ref: Some(SafetyBackupRepositoryRef {
3389                name: "offsite".to_string(),
3390            }),
3391            skip: false,
3392        };
3393        let restore = make_restore(source, Some(safety));
3394        assert!(validate_safety_backup_config(&restore).is_ok());
3395    }
3396
3397    #[test]
3398    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3399        let phases = vec![
3400            KanidmRestorePhase::Pending,
3401            KanidmRestorePhase::Validating,
3402            KanidmRestorePhase::Quiescing,
3403            KanidmRestorePhase::SafetyBackup,
3404            KanidmRestorePhase::PreparingSource,
3405            KanidmRestorePhase::RestoringPrimary,
3406            KanidmRestorePhase::Verifying,
3407            KanidmRestorePhase::RebuildingReplicas,
3408            KanidmRestorePhase::Resuming,
3409            KanidmRestorePhase::Completed,
3410        ];
3411        assert_eq!(phases.len(), 10);
3412        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3413        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3414    }
3415
3416    #[test]
3417    fn status_default_has_empty_phase_timestamps() {
3418        let status = KanidmRestoreStatus::default();
3419        assert!(status.phase_timestamps.is_empty());
3420    }
3421
3422    #[test]
3423    fn status_default_has_empty_original_replicas() {
3424        let status = KanidmRestoreStatus::default();
3425        assert!(status.original_replicas.is_empty());
3426    }
3427
3428    #[test]
3429    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3430        let mut target = super::super::crd::Kanidm::default();
3431        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3432            run_as_non_root: Some(false),
3433            run_as_user: Some(389),
3434            run_as_group: Some(389),
3435            fs_group: Some(389),
3436            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3437            supplemental_groups: Some(vec![390, 391]),
3438            ..Default::default()
3439        });
3440
3441        let ctx = super::source_job_pod_security_context(&target);
3442
3443        assert_eq!(ctx.run_as_non_root, Some(false));
3444        assert_eq!(ctx.run_as_user, Some(389));
3445        assert_eq!(ctx.run_as_group, Some(389));
3446        assert_eq!(ctx.fs_group, Some(389));
3447        assert_eq!(
3448            ctx.fs_group_change_policy.as_deref(),
3449            Some("OnRootMismatch")
3450        );
3451        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3452        assert_eq!(
3453            ctx.seccomp_profile
3454                .as_ref()
3455                .map(|profile| profile.type_.as_str()),
3456            Some("RuntimeDefault")
3457        );
3458    }
3459
3460    #[test]
3461    fn hardened_security_context_drops_all_capabilities() {
3462        let ctx = hardened_security_context();
3463        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3464        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3465        assert_eq!(ctx.run_as_non_root, Some(true));
3466        let caps = ctx.capabilities.unwrap();
3467        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3468        assert_eq!(
3469            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3470            Some("RuntimeDefault")
3471        );
3472    }
3473
3474    #[test]
3475    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3476        let ctx = kanidm_job_security_context();
3477        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3478        assert!(ctx.run_as_non_root.is_none());
3479        assert!(ctx.read_only_root_filesystem.is_none());
3480        let caps = ctx.capabilities.unwrap();
3481        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3482        assert_eq!(
3483            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3484            Some("RuntimeDefault")
3485        );
3486    }
3487
3488    #[test]
3489    fn safety_backup_job_name_is_deterministic() {
3490        let source = KanidmRestoreSource {
3491            local: Some(super::KanidmRestoreLocalSource {
3492                file_name: "backup.json".to_string(),
3493            }),
3494            backup_ref: None,
3495        };
3496        let restore = make_restore(source, None);
3497        let name1 = super::safety_backup_job_name(&restore);
3498        let name2 = super::safety_backup_job_name(&restore);
3499        assert_eq!(name1, name2);
3500        assert_eq!(name1, "test-restore-safety-backup");
3501    }
3502
3503    #[test]
3504    fn source_prep_job_name_is_deterministic() {
3505        let source = KanidmRestoreSource {
3506            local: None,
3507            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3508                name: "backup-1".to_string(),
3509            }),
3510        };
3511        let restore = make_restore(source, None);
3512        let name = super::source_prep_job_name(&restore);
3513        assert_eq!(name, "test-restore-source-prep");
3514    }
3515
3516    #[test]
3517    fn break_glass_condition_is_recorded_in_status() {
3518        let mut status = KanidmRestoreStatus::default();
3519        let condition = super::restore_condition(
3520            &status.conditions,
3521            CONDITION_BREAK_GLASS,
3522            CONDITION_TRUE,
3523            "BreakGlassOverride",
3524            "Safety backup skipped",
3525            Some(1),
3526        );
3527        status.conditions.push(condition);
3528        assert!(status.conditions.iter().any(|c| {
3529            c.type_ == CONDITION_BREAK_GLASS
3530                && c.status == CONDITION_TRUE
3531                && c.reason == "BreakGlassOverride"
3532        }));
3533    }
3534
3535    #[test]
3536    fn replica_count_entry_serialization() {
3537        let entry = ReplicaCountEntry {
3538            group: "primary".to_string(),
3539            replicas: 3,
3540        };
3541        let json = serde_json::to_string(&entry).unwrap();
3542        assert!(json.contains("\"group\":\"primary\""));
3543        assert!(json.contains("\"replicas\":3"));
3544    }
3545
3546    #[test]
3547    fn local_source_requires_repository_ref_when_not_skipping() {
3548        let source = KanidmRestoreSource {
3549            local: Some(super::KanidmRestoreLocalSource {
3550                file_name: "backup.json".to_string(),
3551            }),
3552            backup_ref: None,
3553        };
3554        let safety = SafetyBackupConfig {
3555            repository_ref: None,
3556            skip: false,
3557        };
3558        let restore = make_restore(source, Some(safety));
3559        assert!(validate_safety_backup_config(&restore).is_err());
3560    }
3561
3562    #[test]
3563    fn local_source_accepts_repository_ref_when_not_skipping() {
3564        let source = KanidmRestoreSource {
3565            local: Some(super::KanidmRestoreLocalSource {
3566                file_name: "backup.json".to_string(),
3567            }),
3568            backup_ref: None,
3569        };
3570        let safety = SafetyBackupConfig {
3571            repository_ref: Some(SafetyBackupRepositoryRef {
3572                name: "offsite".to_string(),
3573            }),
3574            skip: false,
3575        };
3576        let restore = make_restore(source, Some(safety));
3577        assert!(validate_safety_backup_config(&restore).is_ok());
3578    }
3579
3580    #[test]
3581    fn skip_without_break_glass_annotations_is_rejected() {
3582        let source = KanidmRestoreSource {
3583            local: Some(super::KanidmRestoreLocalSource {
3584                file_name: "backup.json".to_string(),
3585            }),
3586            backup_ref: None,
3587        };
3588        let safety = SafetyBackupConfig {
3589            repository_ref: None,
3590            skip: true,
3591        };
3592        let restore = make_restore(source, Some(safety));
3593        assert!(validate_safety_backup_config(&restore).is_err());
3594    }
3595
3596    #[test]
3597    fn has_accepted_condition_true() {
3598        let conditions = vec![Condition {
3599            type_: "Ready".to_string(),
3600            status: "True".to_string(),
3601            reason: "Accepted".to_string(),
3602            message: String::new(),
3603            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3604                k8s_openapi::jiff::Timestamp::now(),
3605            ),
3606            observed_generation: None,
3607        }];
3608        assert!(has_accepted_condition(&conditions));
3609    }
3610
3611    #[test]
3612    fn has_accepted_condition_false_when_not_accepted_reason() {
3613        let conditions = vec![Condition {
3614            type_: "Ready".to_string(),
3615            status: "True".to_string(),
3616            reason: "SomeOtherReason".to_string(),
3617            message: String::new(),
3618            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3619                k8s_openapi::jiff::Timestamp::now(),
3620            ),
3621            observed_generation: None,
3622        }];
3623        assert!(!has_accepted_condition(&conditions));
3624    }
3625
3626    #[test]
3627    fn has_accepted_condition_false_when_status_false() {
3628        let conditions = vec![Condition {
3629            type_: "Ready".to_string(),
3630            status: "False".to_string(),
3631            reason: "Accepted".to_string(),
3632            message: String::new(),
3633            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3634                k8s_openapi::jiff::Timestamp::now(),
3635            ),
3636            observed_generation: None,
3637        }];
3638        assert!(!has_accepted_condition(&conditions));
3639    }
3640
3641    #[test]
3642    fn has_accepted_condition_empty() {
3643        assert!(!has_accepted_condition(&[]));
3644    }
3645
3646    #[test]
3647    fn safety_backup_upload_operation_doc_is_valid() {
3648        let source = KanidmRestoreSource {
3649            local: None,
3650            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3651                name: "backup-1".to_string(),
3652            }),
3653        };
3654        let mut restore = make_restore(source, None);
3655        restore.metadata.namespace = Some("test-ns".to_string());
3656        let target = super::super::crd::Kanidm::default();
3657        let doc_str = super::build_safety_upload_operation_doc(
3658            &restore,
3659            &target,
3660            "test-backup-id",
3661            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3662            "my-bucket",
3663            "prod",
3664            "https://s3.example.com",
3665            "us-east-1",
3666            false,
3667            false,
3668            None,
3669            None,
3670        )
3671        .unwrap();
3672        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3673        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3674        assert_eq!(parsed["kind"], "OperationDocument");
3675        assert_eq!(parsed["operation"], "upload");
3676        assert_eq!(parsed["bucket"], "my-bucket");
3677        assert_eq!(parsed["prefix"], "prod");
3678        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3679        assert_eq!(parsed["region"], "us-east-1");
3680        assert_eq!(parsed["backupId"], "test-backup-id");
3681        assert_eq!(parsed["consistency"], "kanidm-offline");
3682        assert_eq!(parsed["reason"], "restore-safety");
3683        assert_eq!(
3684            parsed["payloadPath"],
3685            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3686        );
3687        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3688    }
3689
3690    #[test]
3691    fn download_operation_doc_uses_real_repository_data() {
3692        let source = KanidmRestoreSource {
3693            local: None,
3694            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3695                name: "backup-1".to_string(),
3696            }),
3697        };
3698        let _restore = make_restore(source, None);
3699        let target = super::super::crd::Kanidm::default();
3700        let doc_str = super::build_download_operation_doc(
3701            &target,
3702            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3703            "019c7c76-f423-7a12-8f41-2bea7588a303",
3704            "source-uid-456",
3705            "real-bucket",
3706            "real-prefix",
3707            "https://real-endpoint.com",
3708            "eu-west-1",
3709            true,
3710            false,
3711            None,
3712            None,
3713        );
3714        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3715        assert_eq!(parsed["bucket"], "real-bucket");
3716        assert_eq!(parsed["prefix"], "real-prefix");
3717        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3718        assert_eq!(parsed["region"], "eu-west-1");
3719        assert_eq!(parsed["forcePathStyle"], true);
3720        assert_eq!(
3721            parsed["manifestKey"],
3722            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3723        );
3724        assert_eq!(
3725            parsed["expectedBackupId"],
3726            "019c7c76-f423-7a12-8f41-2bea7588a303"
3727        );
3728        assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3729    }
3730
3731    #[test]
3732    fn safety_backup_ref_is_set_after_verification() {
3733        let status = KanidmRestoreStatus {
3734            phase: KanidmRestorePhase::SafetyBackup,
3735            safety_backup_job_name: Some("test-safety-backup".to_string()),
3736            safety_backup_ref: Some("safety-test-restore".to_string()),
3737            safety_backup_expected_backup_id: Some(
3738                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3739            ),
3740            safety_backup_manifest_key: Some(
3741                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3742            ),
3743            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3744            ..Default::default()
3745        };
3746        assert!(status.safety_backup_ref.is_some());
3747        assert!(
3748            status
3749                .safety_backup_ref
3750                .as_ref()
3751                .unwrap()
3752                .starts_with("safety-")
3753        );
3754        assert!(status.safety_backup_expected_backup_id.is_some());
3755        assert!(status.safety_backup_manifest_key.is_some());
3756        assert!(status.safety_backup_payload_sha256.is_some());
3757    }
3758
3759    #[test]
3760    fn database_mutation_not_started_without_safety_backup_ref() {
3761        let status = KanidmRestoreStatus {
3762            phase: KanidmRestorePhase::PreparingSource,
3763            safety_backup_ref: None,
3764            database_mutation_started: false,
3765            ..Default::default()
3766        };
3767        assert!(!status.database_mutation_started);
3768        assert!(status.safety_backup_ref.is_none());
3769    }
3770
3771    #[test]
3772    fn pre_boundary_failure_does_not_set_mutation_started() {
3773        let status = KanidmRestoreStatus {
3774            phase: KanidmRestorePhase::Failed,
3775            message: Some(
3776                "safety backup job failed; target restored to original state".to_string(),
3777            ),
3778            database_mutation_started: false,
3779            ..Default::default()
3780        };
3781        assert!(!status.database_mutation_started);
3782    }
3783
3784    #[test]
3785    fn shared_volume_constants_are_distinct() {
3786        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3787        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3788    }
3789
3790    #[test]
3791    fn validate_backup_compatibility_matching_versions_passes() {
3792        let source = KanidmRestoreSource {
3793            local: None,
3794            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3795                name: "backup-1".to_string(),
3796            }),
3797        };
3798        let restore = make_restore(source, None);
3799        let target = kanidm_with_version("1.10.4");
3800        let backup = kaniop_backup_core::crd::KanidmBackup {
3801            metadata: kube::api::ObjectMeta::default(),
3802            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3803                backup_id: "id".to_string(),
3804                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3805                    name: "k".to_string(),
3806                    uid: "uid".to_string(),
3807                },
3808                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3809                    name: "repo".to_string(),
3810                },
3811                manifest_key: "key".to_string(),
3812            },
3813            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3814                kanidm_version: Some("1.10.4".to_string()),
3815                ..Default::default()
3816            }),
3817        };
3818        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3819    }
3820
3821    #[test]
3822    fn validate_backup_compatibility_version_mismatch_fails() {
3823        let source = KanidmRestoreSource {
3824            local: None,
3825            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3826                name: "backup-1".to_string(),
3827            }),
3828        };
3829        let restore = make_restore(source, None);
3830        let target = kanidm_with_version("1.11.0");
3831        let backup = kaniop_backup_core::crd::KanidmBackup {
3832            metadata: kube::api::ObjectMeta::default(),
3833            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3834                backup_id: "id".to_string(),
3835                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3836                    name: "k".to_string(),
3837                    uid: "uid".to_string(),
3838                },
3839                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3840                    name: "repo".to_string(),
3841                },
3842                manifest_key: "key".to_string(),
3843            },
3844            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3845                kanidm_version: Some("1.10.4".to_string()),
3846                ..Default::default()
3847            }),
3848        };
3849        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3850    }
3851
3852    #[test]
3853    fn validate_backup_compatibility_no_backup_version_passes() {
3854        let source = KanidmRestoreSource {
3855            local: None,
3856            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3857                name: "backup-1".to_string(),
3858            }),
3859        };
3860        let restore = make_restore(source, None);
3861        let target = super::super::crd::Kanidm::default();
3862        let backup = kaniop_backup_core::crd::KanidmBackup {
3863            metadata: kube::api::ObjectMeta::default(),
3864            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3865                backup_id: "id".to_string(),
3866                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3867                    name: "k".to_string(),
3868                    uid: "uid".to_string(),
3869                },
3870                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3871                    name: "repo".to_string(),
3872                },
3873                manifest_key: "key".to_string(),
3874            },
3875            status: None,
3876        };
3877        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3878    }
3879
3880    #[test]
3881    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3882        let source = KanidmRestoreSource {
3883            local: None,
3884            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3885                name: "backup-1".to_string(),
3886            }),
3887        };
3888        let mut restore = make_restore(source, None);
3889        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3890        let target = super::super::crd::Kanidm::default();
3891        let backup = kaniop_backup_core::crd::KanidmBackup {
3892            metadata: kube::api::ObjectMeta::default(),
3893            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3894                backup_id: "id".to_string(),
3895                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3896                    name: "k".to_string(),
3897                    uid: "uid".to_string(),
3898                },
3899                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3900                    name: "repo".to_string(),
3901                },
3902                manifest_key: "key".to_string(),
3903            },
3904            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3905                image_digest: Some("sha256:originaldigest".to_string()),
3906                ..Default::default()
3907            }),
3908        };
3909        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3910    }
3911
3912    #[test]
3913    fn validate_backup_compatibility_image_digest_match_passes() {
3914        let source = KanidmRestoreSource {
3915            local: None,
3916            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3917                name: "backup-1".to_string(),
3918            }),
3919        };
3920        let mut restore = make_restore(source, None);
3921        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3922        let target = super::super::crd::Kanidm::default();
3923        let backup = kaniop_backup_core::crd::KanidmBackup {
3924            metadata: kube::api::ObjectMeta::default(),
3925            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3926                backup_id: "id".to_string(),
3927                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3928                    name: "k".to_string(),
3929                    uid: "uid".to_string(),
3930                },
3931                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3932                    name: "repo".to_string(),
3933                },
3934                manifest_key: "key".to_string(),
3935            },
3936            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3937                image_digest: Some("sha256:abc123".to_string()),
3938                ..Default::default()
3939            }),
3940        };
3941        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3942    }
3943
3944    #[test]
3945    fn safety_backup_id_is_deterministic() {
3946        let source = KanidmRestoreSource {
3947            local: None,
3948            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3949                name: "backup-1".to_string(),
3950            }),
3951        };
3952        let restore = make_restore(source, None);
3953        let id1 = super::compute_safety_backup_id(&restore).unwrap();
3954        let id2 = super::compute_safety_backup_id(&restore).unwrap();
3955        assert_eq!(id1, id2);
3956        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3957    }
3958
3959    #[test]
3960    fn safety_manifest_key_is_deterministic() {
3961        let source = KanidmRestoreSource {
3962            local: None,
3963            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3964                name: "backup-1".to_string(),
3965            }),
3966        };
3967        let restore = make_restore(source, None);
3968        let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3969        let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3970        assert_eq!(key1, key2);
3971        assert!(key1.contains("manifest.json"));
3972        assert!(key1.starts_with("v1/tenants/"));
3973        // The tenant segment is the Kubernetes namespace NAME from `make_restore`, never the
3974        // Namespace object UID or the restore UID.
3975        assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3976        assert!(
3977            !key1.contains("a1b2c3d4"),
3978            "safety manifest key must not key the tenant segment by object UID: {key1}"
3979        );
3980    }
3981
3982    /// `compute_safety_manifest_key` hand-rolls the keyspace with `format!` instead of going through
3983    /// `kaniop_backup_core::paths::RepositoryPath`, so it is pinned here byte-for-byte against the
3984    /// deployed layout: `v1/tenants/<namespace-name>/clusters/<kanidm-uid>/backups/<backup-id>/`
3985    /// `manifest.json`. The `<backup-id>` is the UUIDv5 chain `DNS -> namespace name -> restore UID`
3986    /// and is load-bearing for transport re-upload idempotency, so both the segment and the derived
3987    /// ID are asserted as literals. The tenant segment carries the namespace NAME even though the
3988    /// `namespaceUid` wire field it mirrors is named like a UID (misnomer tracked in issue #1084).
3989    #[test]
3990    fn safety_manifest_key_matches_deployed_keyspace_layout() {
3991        let source = KanidmRestoreSource {
3992            local: None,
3993            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3994                name: "backup-1".to_string(),
3995            }),
3996        };
3997        let restore = make_restore(source, None);
3998        let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3999        assert_eq!(
4000            super::compute_safety_backup_id(&restore).unwrap(),
4001            expected_backup_id
4002        );
4003        assert_eq!(
4004            super::compute_safety_manifest_key(&restore).unwrap(),
4005            format!(
4006                "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
4007            )
4008        );
4009    }
4010
4011    #[test]
4012    fn safety_backup_id_survives_restart() {
4013        let source = KanidmRestoreSource {
4014            local: None,
4015            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4016                name: "backup-1".to_string(),
4017            }),
4018        };
4019        let mut restore = make_restore(source, None);
4020        let original_id = super::compute_safety_backup_id(&restore).unwrap();
4021        // Pinning the derived ID pins its UUIDv5 inputs: the namespace NAME and the restore UID.
4022        // Feeding the Namespace object UID here would change every safety backup ID and orphan an
4023        // in-flight safety backup (issue #1084).
4024        assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
4025        restore.status = Some(KanidmRestoreStatus {
4026            phase: KanidmRestorePhase::SafetyBackup,
4027            safety_backup_expected_backup_id: Some(original_id.clone()),
4028            ..Default::default()
4029        });
4030        let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
4031        assert_eq!(original_id, recomputed_id);
4032    }
4033
4034    #[test]
4035    fn safety_backup_id_derived_from_uid_not_name() {
4036        let source = KanidmRestoreSource {
4037            local: None,
4038            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4039                name: "backup-1".to_string(),
4040            }),
4041        };
4042        let restore1 = make_restore(source.clone(), None);
4043        let mut restore2 = make_restore(source, None);
4044        restore2.metadata.name = Some("different-name-same-uid".to_string());
4045        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
4046        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
4047        assert_eq!(id1, id2);
4048    }
4049
4050    #[test]
4051    fn safety_backup_id_differs_for_different_uids() {
4052        let source = KanidmRestoreSource {
4053            local: None,
4054            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4055                name: "backup-1".to_string(),
4056            }),
4057        };
4058        let restore1 = make_restore(source.clone(), None);
4059        let mut restore2 = make_restore(source, None);
4060        restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
4061        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
4062        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
4063        assert_ne!(id1, id2);
4064    }
4065
4066    #[test]
4067    fn safety_backup_id_errors_without_uid() {
4068        let source = KanidmRestoreSource {
4069            local: None,
4070            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4071                name: "backup-1".to_string(),
4072            }),
4073        };
4074        let mut restore = make_restore(source, None);
4075        restore.metadata.uid = None;
4076        assert!(super::compute_safety_backup_id(&restore).is_err());
4077    }
4078
4079    #[test]
4080    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4081        let source = KanidmRestoreSource {
4082            local: None,
4083            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4084                name: "backup-1".to_string(),
4085            }),
4086        };
4087        let restore = make_restore(source, None);
4088        assert!(super::requires_safety_backup(&restore));
4089        let status_without_ref = KanidmRestoreStatus {
4090            phase: KanidmRestorePhase::RestoringPrimary,
4091            safety_backup_ref: None,
4092            database_mutation_started: false,
4093            ..Default::default()
4094        };
4095        assert!(status_without_ref.safety_backup_ref.is_none());
4096        assert!(!status_without_ref.database_mutation_started);
4097    }
4098
4099    #[test]
4100    fn mutation_boundary_allows_progression_when_safety_ref_set() {
4101        let status_with_ref = KanidmRestoreStatus {
4102            phase: KanidmRestorePhase::RestoringPrimary,
4103            safety_backup_ref: Some("safety-test-restore".to_string()),
4104            safety_backup_manifest_key: Some(
4105                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4106            ),
4107            safety_backup_payload_sha256: Some("abc123".to_string()),
4108            database_mutation_started: false,
4109            ..Default::default()
4110        };
4111        assert!(status_with_ref.safety_backup_ref.is_some());
4112        assert!(!status_with_ref.database_mutation_started);
4113    }
4114
4115    #[test]
4116    fn result_document_verification_rejects_missing_backup_id() {
4117        let result_json = r#"{
4118            "apiVersion": "backup.kaniop.rs/v1alpha1",
4119            "kind": "ResultDocument",
4120            "operation": "upload",
4121            "success": true,
4122            "exitCode": "success",
4123            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4124            "payloadSha256": "abc123"
4125        }"#;
4126        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4127        assert!(result.backup_id.is_none());
4128    }
4129
4130    #[test]
4131    fn result_document_verification_rejects_missing_manifest_key() {
4132        let result_json = r#"{
4133            "apiVersion": "backup.kaniop.rs/v1alpha1",
4134            "kind": "ResultDocument",
4135            "operation": "upload",
4136            "success": true,
4137            "exitCode": "success",
4138            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4139            "payloadSha256": "abc123"
4140        }"#;
4141        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4142        assert!(result.manifest_key.is_none());
4143    }
4144
4145    #[test]
4146    fn result_document_verification_rejects_missing_payload_sha256() {
4147        let result_json = r#"{
4148            "apiVersion": "backup.kaniop.rs/v1alpha1",
4149            "kind": "ResultDocument",
4150            "operation": "upload",
4151            "success": true,
4152            "exitCode": "success",
4153            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4154            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4155        }"#;
4156        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4157        assert!(result.payload_sha256.is_none());
4158    }
4159
4160    #[test]
4161    fn result_document_verification_rejects_failure_exit_code() {
4162        let result_json = r#"{
4163            "apiVersion": "backup.kaniop.rs/v1alpha1",
4164            "kind": "ResultDocument",
4165            "operation": "upload",
4166            "success": false,
4167            "exitCode": "integrity",
4168            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4169            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4170            "payloadSha256": "abc123"
4171        }"#;
4172        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4173        assert!(!result.success);
4174        assert_eq!(
4175            result.exit_code,
4176            kaniop_backup_core::result::ExitCode::Integrity
4177        );
4178    }
4179
4180    #[test]
4181    fn result_document_verification_rejects_wrong_operation() {
4182        let result_json = r#"{
4183            "apiVersion": "backup.kaniop.rs/v1alpha1",
4184            "kind": "ResultDocument",
4185            "operation": "download",
4186            "success": true,
4187            "exitCode": "success",
4188            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4189            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4190            "payloadSha256": "abc123"
4191        }"#;
4192        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4193        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4194    }
4195
4196    #[test]
4197    fn result_document_verification_accepts_valid_document() {
4198        let result_json = r#"{
4199            "apiVersion": "backup.kaniop.rs/v1alpha1",
4200            "kind": "ResultDocument",
4201            "operation": "upload",
4202            "success": true,
4203            "exitCode": "success",
4204            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4205            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4206            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4207        }"#;
4208        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4209        assert!(result.success);
4210        assert_eq!(
4211            result.exit_code,
4212            kaniop_backup_core::result::ExitCode::Success
4213        );
4214        assert_eq!(result.operation, "upload");
4215        assert!(result.backup_id.is_some());
4216        assert!(result.manifest_key.is_some());
4217        assert!(result.payload_sha256.is_some());
4218    }
4219
4220    #[test]
4221    fn termination_message_path_constant_is_valid() {
4222        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4223        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4224    }
4225
4226    #[test]
4227    fn status_tracks_safety_backup_evidence() {
4228        let status = KanidmRestoreStatus {
4229            phase: KanidmRestorePhase::PreparingSource,
4230            safety_backup_ref: Some("safety-test-restore".to_string()),
4231            safety_backup_expected_backup_id: Some(
4232                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4233            ),
4234            safety_backup_manifest_key: Some(
4235                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4236            ),
4237            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4238            ..Default::default()
4239        };
4240        assert!(status.safety_backup_ref.is_some());
4241        assert!(status.safety_backup_expected_backup_id.is_some());
4242        assert!(status.safety_backup_manifest_key.is_some());
4243        assert!(status.safety_backup_payload_sha256.is_some());
4244    }
4245
4246    #[test]
4247    fn fail_closed_semantics_preserved_on_result_parse_failure() {
4248        let invalid_json = "not valid json";
4249        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4250        assert!(result.is_err());
4251    }
4252
4253    #[test]
4254    fn fail_closed_semantics_preserved_on_oversized_result() {
4255        let oversized_json = format!(
4256            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4257            kaniop_backup_core::result::RESULT_DOC_VERSION,
4258            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4259        );
4260        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4261        assert!(result.is_err());
4262    }
4263
4264    #[test]
4265    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4266        let source = KanidmRestoreSource {
4267            local: None,
4268            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4269                name: "backup-1".to_string(),
4270            }),
4271        };
4272        let mut restore = make_restore(source, None);
4273        restore.metadata.namespace = Some("test-ns".to_string());
4274        let target = super::super::crd::Kanidm::default();
4275        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4276        let doc_str = super::build_safety_upload_operation_doc(
4277            &restore,
4278            &target,
4279            "test-backup-id",
4280            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4281            "my-bucket",
4282            "prod",
4283            "https://s3.example.com",
4284            "us-east-1",
4285            false,
4286            false,
4287            Some(&ca_path),
4288            None,
4289        )
4290        .unwrap();
4291        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4292        assert_eq!(parsed["caBundlePath"], ca_path);
4293    }
4294
4295    #[test]
4296    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4297        let source = KanidmRestoreSource {
4298            local: None,
4299            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4300                name: "backup-1".to_string(),
4301            }),
4302        };
4303        let mut restore = make_restore(source, None);
4304        restore.metadata.namespace = Some("test-ns".to_string());
4305        let target = super::super::crd::Kanidm::default();
4306        let doc_str = super::build_safety_upload_operation_doc(
4307            &restore,
4308            &target,
4309            "test-backup-id",
4310            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4311            "my-bucket",
4312            "prod",
4313            "https://s3.example.com",
4314            "us-east-1",
4315            false,
4316            false,
4317            None,
4318            None,
4319        )
4320        .unwrap();
4321        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4322        assert!(parsed["caBundlePath"].is_null());
4323    }
4324
4325    #[test]
4326    fn download_operation_doc_includes_ca_bundle_path_when_set() {
4327        let source = KanidmRestoreSource {
4328            local: None,
4329            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4330                name: "backup-1".to_string(),
4331            }),
4332        };
4333        let _restore = make_restore(source, None);
4334        let target = super::super::crd::Kanidm::default();
4335        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4336        let doc_str = super::build_download_operation_doc(
4337            &target,
4338            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4339            "019c7c76-f423-7a12-8f41-2bea7588a303",
4340            "source-uid-456",
4341            "real-bucket",
4342            "real-prefix",
4343            "https://real-endpoint.com",
4344            "eu-west-1",
4345            true,
4346            false,
4347            Some(&ca_path),
4348            None,
4349        );
4350        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4351        assert_eq!(parsed["caBundlePath"], ca_path);
4352    }
4353
4354    #[test]
4355    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4356        let source = KanidmRestoreSource {
4357            local: None,
4358            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4359                name: "backup-1".to_string(),
4360            }),
4361        };
4362        let _restore = make_restore(source, None);
4363        let target = super::super::crd::Kanidm::default();
4364        let doc_str = super::build_download_operation_doc(
4365            &target,
4366            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4367            "019c7c76-f423-7a12-8f41-2bea7588a303",
4368            "source-uid-456",
4369            "real-bucket",
4370            "real-prefix",
4371            "https://real-endpoint.com",
4372            "eu-west-1",
4373            true,
4374            false,
4375            None,
4376            None,
4377        );
4378        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4379        assert!(parsed["caBundlePath"].is_null());
4380    }
4381
4382    #[test]
4383    fn source_prep_result_operation_is_download() {
4384        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4385        assert_ne!(
4386            super::SOURCE_PREP_RESULT_OPERATION,
4387            super::SAFETY_BACKUP_RESULT_OPERATION
4388        );
4389    }
4390
4391    #[test]
4392    fn download_result_document_verification_accepts_valid_document() {
4393        let result_json = r#"{
4394            "apiVersion": "backup.kaniop.rs/v1alpha1",
4395            "kind": "ResultDocument",
4396            "operation": "download",
4397            "success": true,
4398            "exitCode": "success",
4399            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4400            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4401            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4402        }"#;
4403        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4404        assert!(result.success);
4405        assert_eq!(
4406            result.exit_code,
4407            kaniop_backup_core::result::ExitCode::Success
4408        );
4409        assert_eq!(result.operation, "download");
4410        assert_eq!(
4411            result.backup_id.as_deref(),
4412            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4413        );
4414        assert!(result.manifest_key.is_some());
4415        assert!(result.payload_sha256.is_some());
4416    }
4417
4418    #[test]
4419    fn download_result_document_rejects_wrong_operation() {
4420        let result_json = r#"{
4421            "apiVersion": "backup.kaniop.rs/v1alpha1",
4422            "kind": "ResultDocument",
4423            "operation": "upload",
4424            "success": true,
4425            "exitCode": "success",
4426            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4427            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4428            "payloadSha256": "abc123"
4429        }"#;
4430        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4431        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4432    }
4433
4434    #[test]
4435    fn download_result_document_rejects_failure_success_field() {
4436        let result_json = r#"{
4437            "apiVersion": "backup.kaniop.rs/v1alpha1",
4438            "kind": "ResultDocument",
4439            "operation": "download",
4440            "success": false,
4441            "exitCode": "retryable",
4442            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4443            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4444            "payloadSha256": "abc123"
4445        }"#;
4446        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4447        assert!(!result.success);
4448        assert_ne!(
4449            result.exit_code,
4450            kaniop_backup_core::result::ExitCode::Success
4451        );
4452    }
4453
4454    #[test]
4455    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4456        let status = KanidmRestoreStatus {
4457            phase: KanidmRestorePhase::PreparingSource,
4458            database_mutation_started: false,
4459            ..Default::default()
4460        };
4461        assert!(!status.database_mutation_started);
4462        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4463    }
4464
4465    #[test]
4466    fn backup_job_volume_size_default_is_10gi() {
4467        let qty = crate::controller::backup_job_volume_size();
4468        assert_eq!(qty, Quantity("10Gi".to_string()));
4469    }
4470
4471    #[test]
4472    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4473        use kaniop_backup_core::auth::build_encryption_env_vars;
4474        use kaniop_backup_core::crd::EncryptionKeyRef;
4475
4476        let key_ref = EncryptionKeyRef {
4477            name: "kek-secret".to_string(),
4478            key: "encryption-key".to_string(),
4479        };
4480        let env_vars = build_encryption_env_vars(Some(&key_ref));
4481        assert_eq!(env_vars.len(), 1);
4482        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4483        let secret_ref = env_vars[0]
4484            .value_from
4485            .as_ref()
4486            .unwrap()
4487            .secret_key_ref
4488            .as_ref()
4489            .unwrap();
4490        assert_eq!(secret_ref.name, "kek-secret");
4491        assert_eq!(secret_ref.key, "encryption-key");
4492    }
4493
4494    #[test]
4495    fn encryption_env_vars_absent_when_no_key_ref() {
4496        use kaniop_backup_core::auth::build_encryption_env_vars;
4497
4498        let env_vars = build_encryption_env_vars(None);
4499        assert!(
4500            env_vars.is_empty(),
4501            "encryption env vars must be empty when key_ref is None"
4502        );
4503    }
4504
4505    #[test]
4506    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4507        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, EncryptionKeyRef};
4508
4509        let source = KanidmRestoreSource {
4510            local: None,
4511            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4512                name: "backup-1".to_string(),
4513            }),
4514        };
4515        let mut restore = make_restore(source, None);
4516        restore.metadata.namespace = Some("test-ns".to_string());
4517        let target = super::super::crd::Kanidm::default();
4518        let encryption = RepositoryEncryption {
4519            mode: EncryptionMode::ClientSide,
4520            key_id: None,
4521            key_ref: Some(EncryptionKeyRef {
4522                name: "kek-secret".to_string(),
4523                key: "encryption-key".to_string(),
4524            }),
4525        };
4526        let doc_str = super::build_safety_upload_operation_doc(
4527            &restore,
4528            &target,
4529            "test-backup-id",
4530            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4531            "my-bucket",
4532            "prod",
4533            "https://s3.example.com",
4534            "us-east-1",
4535            false,
4536            false,
4537            None,
4538            Some(&encryption),
4539        )
4540        .unwrap();
4541        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4542        assert_eq!(parsed["encryptionMode"], "clientSide");
4543        assert!(parsed["encryptionKeyId"].is_null());
4544    }
4545
4546    #[test]
4547    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4548        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4549
4550        let source = KanidmRestoreSource {
4551            local: None,
4552            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4553                name: "backup-1".to_string(),
4554            }),
4555        };
4556        let _restore = make_restore(source, None);
4557        let target = super::super::crd::Kanidm::default();
4558        let encryption = RepositoryEncryption {
4559            mode: EncryptionMode::ProviderKms,
4560            key_id: Some("alias/kaniop-backups".to_string()),
4561            key_ref: None,
4562        };
4563        let doc_str = super::build_download_operation_doc(
4564            &target,
4565            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4566            "019c7c76-f423-7a12-8f41-2bea7588a303",
4567            "source-uid-456",
4568            "real-bucket",
4569            "real-prefix",
4570            "https://real-endpoint.com",
4571            "eu-west-1",
4572            true,
4573            false,
4574            None,
4575            Some(&encryption),
4576        );
4577        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4578        assert_eq!(parsed["encryptionMode"], "providerKms");
4579        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4580    }
4581
4582    #[test]
4583    fn post_mutation_failure_records_job_names() {
4584        let existing = KanidmRestoreStatus {
4585            phase: KanidmRestorePhase::RestoringPrimary,
4586            database_mutation_started: true,
4587            ..Default::default()
4588        };
4589
4590        let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4591        assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4592        assert_eq!(
4593            restore_failed.restore_job_name.as_deref(),
4594            Some("my-restore-job")
4595        );
4596        assert!(restore_failed.verify_job_name.is_none());
4597        assert!(restore_failed.database_mutation_started);
4598        assert_eq!(
4599            restore_failed.message.as_deref(),
4600            Some("database restore job failed")
4601        );
4602
4603        let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4604        assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4605        assert_eq!(
4606            verify_failed.verify_job_name.as_deref(),
4607            Some("my-verify-job")
4608        );
4609        assert!(verify_failed.restore_job_name.is_none());
4610        assert!(verify_failed.database_mutation_started);
4611        assert_eq!(
4612            verify_failed.message.as_deref(),
4613            Some("database verification failed")
4614        );
4615    }
4616
4617    #[test]
4618    fn restore_and_verify_job_names_are_deterministic() {
4619        let source = KanidmRestoreSource {
4620            local: Some(super::KanidmRestoreLocalSource {
4621                file_name: "backup.json".to_string(),
4622            }),
4623            backup_ref: None,
4624        };
4625        let restore = make_restore(source, None);
4626        assert_eq!(restore_job_name(&restore), "test-restore-restore");
4627        assert_eq!(verify_job_name(&restore), "test-restore-verify");
4628    }
4629}