Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78    feature = "schemars",
79    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82    category = "kaniop",
83    group = "kaniop.rs",
84    version = "v1beta1",
85    kind = "KanidmRestore",
86    plural = "kanidmrestores",
87    singular = "kanidmrestore",
88    shortname = "idmrestore",
89    namespaced,
90    status = "KanidmRestoreStatus",
91    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97    pub target_ref: KanidmRestoreTargetRef,
98    pub source: KanidmRestoreSource,
99    pub restore_image: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108    pub name: String,
109    pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115    feature = "schemars",
116    schemars(extend("x-kubernetes-validations" = [
117        {
118            "message": "exactly one of local or backupRef must be set",
119            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120        }
121    ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125    /// Local file source. Mutually exclusive with `backupRef`.
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub local: Option<KanidmRestoreLocalSource>,
128    /// Remote cataloged backup reference. Mutually exclusive with `local`.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137    /// Basename of a backup below /data. Paths and traversal are rejected.
138    pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
146    pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub repository_ref: Option<SafetyBackupRepositoryRef>,
156    /// Skip the mandatory safety backup. Requires break-glass annotations.
157    #[serde(default)]
158    pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165    pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171    #[default]
172    Pending,
173    Validating,
174    Quiescing,
175    SafetyBackup,
176    PreparingSource,
177    RestoringPrimary,
178    Verifying,
179    RebuildingReplicas,
180    Resuming,
181    Completed,
182    Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_generation: Option<i64>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_target_uid: Option<String>,
193    #[serde(default)]
194    pub phase: KanidmRestorePhase,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub restore_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub verify_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub safety_backup_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub source_prep_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_ref: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_expected_backup_id: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_manifest_key: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_payload_sha256: Option<String>,
211    #[serde(default)]
212    pub replicas_cleared: bool,
213    #[serde(default)]
214    pub certificates_cleared: bool,
215    #[serde(default)]
216    pub database_mutation_started: bool,
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub original_replicas: Vec<ReplicaCountEntry>,
219    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220    pub phase_timestamps: std::collections::BTreeMap<String, String>,
221    #[serde(default, skip_serializing_if = "Vec::is_empty")]
222    #[cfg_attr(
223        feature = "schemars",
224        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225    )]
226    pub conditions: Vec<Condition>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235    pub group: String,
236    pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241    attempts: Counter<u64>,
242    outcomes: Counter<u64>,
243    duration_seconds: Histogram<f64>,
244    start_timestamp_seconds: Gauge<i64>,
245    break_glass_total: Counter<u64>,
246    disaster_recovery_total: Counter<u64>,
247    safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251    fn new() -> Self {
252        let meter = global::meter("kaniop");
253        Self {
254            attempts: meter
255                .u64_counter("restore_attempts")
256                .with_description("Number of Kanidm restore attempts started")
257                .build(),
258            outcomes: meter
259                .u64_counter("restore_outcomes")
260                .with_description("Number of terminal Kanidm restore outcomes")
261                .build(),
262            duration_seconds: meter
263                .f64_histogram("restore_duration_seconds")
264                .with_description("Kanidm restore duration from object creation to terminal phase")
265                .with_unit("s")
266                .build(),
267            start_timestamp_seconds: meter
268                .i64_gauge("restore_start_timestamp_seconds")
269                .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270                .with_unit("s")
271                .build(),
272            break_glass_total: meter
273                .u64_counter("restore_break_glass")
274                .with_description("Number of break-glass safety backup overrides")
275                .build(),
276            disaster_recovery_total: meter
277                .u64_counter("restore_disaster_recovery")
278                .with_description("Number of approved disaster-recovery source identity overrides")
279                .build(),
280            safety_backup_duration_seconds: meter
281                .f64_histogram("restore_safety_backup_duration_seconds")
282                .with_description("Duration of pre-restore safety backup creation")
283                .with_unit("s")
284                .build(),
285        }
286    }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290    let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291    let active = restore.metadata.deletion_timestamp.is_none()
292        && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293    let timestamp = if active {
294        restore
295            .metadata
296            .creation_timestamp
297            .as_ref()
298            .map(|created| created.0.as_second())
299            .unwrap_or_else(|| Timestamp::now().as_second())
300    } else {
301        0
302    };
303    let attributes = [
304        KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305        KeyValue::new("restore", restore.name_any()),
306    ];
307    metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312    client: Client,
313    recorder: Recorder,
314    metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319    match phase {
320        KanidmRestorePhase::Pending => {
321            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322            Ok(Action::requeue(REQUEUE))
323        }
324        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325            Ok((target, disaster_recovery_source)) => {
326                mark_restoring(&restore, &target, &ctx).await?;
327                if let Some(source) = disaster_recovery_source.as_ref() {
328                    record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329                }
330                let mut status = restore.status.clone().unwrap_or_default();
331                if let Some(source) = disaster_recovery_source.as_ref() {
332                    let target_uid = target.uid().unwrap_or_default();
333                    let condition = restore_condition(
334                        &status.conditions,
335                        CONDITION_DISASTER_RECOVERY,
336                        CONDITION_TRUE,
337                        "ApprovedSourceIdentityMismatch",
338                        &format!(
339                            "Approved disaster recovery from source {}/{} into target {}/{}",
340                            source.name,
341                            source.uid,
342                            target.name_any(),
343                            target_uid
344                        ),
345                        restore.metadata.generation,
346                    );
347                    status
348                        .conditions
349                        .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350                    status.conditions.push(condition);
351                }
352                status.observed_target_uid = target.uid();
353                status.original_replicas = capture_original_replicas(&target);
354                status.phase = KanidmRestorePhase::Quiescing;
355                status.message = None;
356                patch_status(&restore, &ctx, status).await?;
357                Ok(Action::requeue(REQUEUE))
358            }
359            Err(error) => {
360                set_phase(
361                    &restore,
362                    &ctx,
363                    KanidmRestorePhase::Failed,
364                    Some(error.to_string()),
365                )
366                .await?;
367                Ok(Action::requeue(Duration::from_secs(300)))
368            }
369        },
370        KanidmRestorePhase::Quiescing => {
371            let target = get_target(&restore, &ctx).await?;
372            scale_all(&target, &ctx, 0).await?;
373            if target_pods_stopped(&target, &ctx).await?
374                && target_volumes_detached(&target, &ctx).await?
375            {
376                let next_phase = if requires_safety_backup(&restore) {
377                    KanidmRestorePhase::SafetyBackup
378                } else {
379                    record_break_glass(&restore, &ctx).await?;
380                    KanidmRestorePhase::PreparingSource
381                };
382                if next_phase == KanidmRestorePhase::PreparingSource {
383                    let refreshed = Api::<KanidmRestore>::namespaced(
384                        ctx.client.clone(),
385                        restore.namespace().as_deref().unwrap_or_default(),
386                    )
387                    .get(&restore.name_any())
388                    .await
389                    .map_err(|error| {
390                        Error::kube_error(
391                            "get",
392                            "KanidmRestore",
393                            restore.namespace().as_deref().unwrap_or_default(),
394                            restore.name_any(),
395                            error,
396                        )
397                    })?;
398                    set_phase(&refreshed, &ctx, next_phase, None).await?;
399                } else {
400                    set_phase(&restore, &ctx, next_phase, None).await?;
401                }
402            }
403            Ok(Action::requeue(REQUEUE))
404        }
405        KanidmRestorePhase::SafetyBackup => {
406            let target = get_target(&restore, &ctx).await?;
407            let name = safety_backup_job_name(&restore);
408            let expected_backup_id = compute_safety_backup_id(&restore)?;
409            let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410            ensure_safety_backup_job(
411                &restore,
412                &target,
413                &ctx,
414                &name,
415                &expected_backup_id,
416                &expected_manifest_key,
417            )
418            .await?;
419            let mut status = restore.status.clone().unwrap_or_default();
420            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421            match job_state(&restore, &ctx, &name).await? {
422                JobState::Complete => {
423                    let result =
424                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425                    match result {
426                        Ok(verified) => {
427                            status.safety_backup_job_name = Some(name);
428                            status.safety_backup_ref =
429                                Some(format!("safety-{}", restore.name_any()));
430                            status.safety_backup_manifest_key = Some(verified.manifest_key);
431                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432                            record_safety_backup_duration(&restore, &ctx).await;
433                            status.phase = KanidmRestorePhase::PreparingSource;
434                            status.message = None;
435                            patch_status(&restore, &ctx, status).await?;
436                        }
437                        Err(error) => {
438                            resume_before_mutation(&restore, &ctx).await?;
439                            set_phase(
440                                &restore,
441                                &ctx,
442                                KanidmRestorePhase::Failed,
443                                Some(format!("safety backup result verification failed: {error}")),
444                            )
445                            .await?;
446                        }
447                    }
448                }
449                JobState::Failed => {
450                    resume_before_mutation(&restore, &ctx).await?;
451                    set_phase(
452                        &restore,
453                        &ctx,
454                        KanidmRestorePhase::Failed,
455                        Some(
456                            "safety backup job failed; target restored to original state"
457                                .to_string(),
458                        ),
459                    )
460                    .await?;
461                }
462                JobState::Running => {
463                    patch_status(&restore, &ctx, status).await?;
464                }
465            }
466            Ok(Action::requeue(REQUEUE))
467        }
468        KanidmRestorePhase::PreparingSource => {
469            if is_remote_source(&restore) {
470                let target = get_target(&restore, &ctx).await?;
471                let name = source_prep_job_name(&restore);
472                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473                match job_state(&restore, &ctx, &name).await? {
474                    JobState::Complete => {
475                        let backup_id = restore
476                            .spec
477                            .source
478                            .backup_ref
479                            .as_ref()
480                            .map(|b| b.name.clone())
481                            .unwrap_or_default();
482                        let backup = {
483                            let ns = restore.namespace().unwrap();
484                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485                                .get(&backup_id)
486                                .await
487                                .map_err(|e| {
488                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489                                })?
490                        };
491                        let expected_backup_id = &backup.spec.backup_id;
492                        let result =
493                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494                                .await;
495                        match result {
496                            Ok(verified) => {
497                                if let Some(expected_sha256) = backup
498                                    .status
499                                    .as_ref()
500                                    .and_then(|s| s.payload_sha256.as_deref())
501                                {
502                                    if verified.payload_sha256 != expected_sha256 {
503                                        resume_before_mutation(&restore, &ctx).await?;
504                                        set_phase(
505                                            &restore,
506                                            &ctx,
507                                            KanidmRestorePhase::Failed,
508                                            Some(format!(
509                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510                                                verified.payload_sha256
511                                            )),
512                                        )
513                                        .await?;
514                                        return Ok(Action::requeue(REQUEUE));
515                                    }
516                                }
517                                let mut status = restore.status.clone().unwrap_or_default();
518                                status.source_prep_job_name = Some(name);
519                                status.phase = KanidmRestorePhase::RestoringPrimary;
520                                status.message = None;
521                                patch_status(&restore, &ctx, status).await?;
522                            }
523                            Err(error) => {
524                                resume_before_mutation(&restore, &ctx).await?;
525                                set_phase(
526                                    &restore,
527                                    &ctx,
528                                    KanidmRestorePhase::Failed,
529                                    Some(format!(
530                                        "source preparation result verification failed: {error}"
531                                    )),
532                                )
533                                .await?;
534                            }
535                        }
536                    }
537                    JobState::Failed => {
538                        resume_before_mutation(&restore, &ctx).await?;
539                        set_phase(
540                            &restore,
541                            &ctx,
542                            KanidmRestorePhase::Failed,
543                            Some(
544                                "source preparation job failed; target restored to original state"
545                                    .to_string(),
546                            ),
547                        )
548                        .await?;
549                    }
550                    JobState::Running => {}
551                }
552            } else {
553                let target = get_target(&restore, &ctx).await?;
554                let name = source_check_job_name(&restore);
555                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556                match job_state(&restore, &ctx, &name).await? {
557                    JobState::Complete => {
558                        let mut status = restore.status.clone().unwrap_or_default();
559                        status.source_prep_job_name = Some(name);
560                        status.phase = KanidmRestorePhase::RestoringPrimary;
561                        status.message = None;
562                        patch_status(&restore, &ctx, status).await?;
563                    }
564                    JobState::Failed => {
565                        resume_before_mutation(&restore, &ctx).await?;
566                        set_phase(
567                            &restore,
568                            &ctx,
569                            KanidmRestorePhase::Failed,
570                            Some(
571                                "local backup file check failed; target restored to original state"
572                                    .to_string(),
573                            ),
574                        )
575                        .await?;
576                    }
577                    JobState::Running => {}
578                }
579            }
580            Ok(Action::requeue(REQUEUE))
581        }
582        KanidmRestorePhase::RestoringPrimary => {
583            let target = get_target(&restore, &ctx).await?;
584            let status = restore.status.as_ref().cloned().unwrap_or_default();
585            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586                return Err(Error::MissingData(
587                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588                ));
589            }
590            if !status.database_mutation_started {
591                let mut status = status;
592                status.database_mutation_started = true;
593                status.message = Some(
594                    "database mutation boundary persisted; starting primary restore".to_string(),
595                );
596                patch_status(&restore, &ctx, status).await?;
597                return Ok(Action::requeue(REQUEUE));
598            }
599            let name = restore_job_name(&restore);
600            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601            match job_state(&restore, &ctx, &name).await? {
602                JobState::Complete => {
603                    let mut status = restore.status.clone().unwrap_or_default();
604                    status.restore_job_name = Some(name);
605                    status.phase = KanidmRestorePhase::Verifying;
606                    status.message = None;
607                    patch_status(&restore, &ctx, status).await?;
608                }
609                JobState::Failed => {
610                    let status = restore.status.clone().unwrap_or_default();
611                    let failed_status = restore_primary_failed_status(&status, &name);
612                    patch_status(&restore, &ctx, failed_status).await?;
613                }
614                JobState::Running => {}
615            }
616            Ok(Action::requeue(REQUEUE))
617        }
618        KanidmRestorePhase::Verifying => {
619            let target = get_target(&restore, &ctx).await?;
620            let name = verify_job_name(&restore);
621            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622            match job_state(&restore, &ctx, &name).await? {
623                JobState::Complete => {
624                    let mut status = restore.status.clone().unwrap_or_default();
625                    status.verify_job_name = Some(name);
626                    status.phase = KanidmRestorePhase::RebuildingReplicas;
627                    status.message = None;
628                    patch_status(&restore, &ctx, status).await?;
629                }
630                JobState::Failed => {
631                    let status = restore.status.clone().unwrap_or_default();
632                    let failed_status = verifying_failed_status(&status, &name);
633                    patch_status(&restore, &ctx, failed_status).await?;
634                }
635                JobState::Running => {}
636            }
637            Ok(Action::requeue(REQUEUE))
638        }
639        KanidmRestorePhase::RebuildingReplicas => {
640            let target = get_target(&restore, &ctx).await?;
641            let mut status = restore.status.clone().unwrap_or_default();
642            if !status.replicas_cleared {
643                if delete_secondary_pvcs(&target, &ctx).await? {
644                    status.replicas_cleared = true;
645                    status.message = Some("secondary database state cleared".to_string());
646                    patch_status(&restore, &ctx, status).await?;
647                }
648                return Ok(Action::requeue(REQUEUE));
649            }
650
651            scale_primary(&target, &ctx, 1).await?;
652            if !primary_ready(&target, &ctx).await? {
653                return Ok(Action::requeue(REQUEUE));
654            }
655            if !status.certificates_cleared {
656                delete_replica_cert_secrets(&target, &ctx).await?;
657                delete_admin_secret(&target, &ctx).await?;
658                status.certificates_cleared = true;
659                status.message = Some(
660                    "replica certificates and admin secret cleared for regeneration".to_string(),
661                );
662                patch_status(&restore, &ctx, status).await?;
663                return Ok(Action::requeue(REQUEUE));
664            }
665            status.phase = KanidmRestorePhase::Resuming;
666            status.message = None;
667            patch_status(&restore, &ctx, status).await?;
668            Ok(Action::requeue(REQUEUE))
669        }
670        KanidmRestorePhase::Resuming => {
671            let target = get_target(&restore, &ctx).await?;
672            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673            clear_restoring(&restore, &target, &ctx).await?;
674            Ok(Action::requeue(Duration::from_secs(3600)))
675        }
676        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677            Ok(Action::requeue(Duration::from_secs(3600)))
678        }
679    }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684    name: String,
685    uid: String,
686}
687
688async fn validate(
689    restore: &KanidmRestore,
690    ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692    let target = get_target(restore, ctx).await?;
693    let actual_uid = target
694        .uid()
695        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696    if actual_uid != restore.spec.target_ref.uid {
697        return Err(Error::MissingData(format!(
698            "target UID mismatch: expected {}, got {}",
699            restore.spec.target_ref.uid, actual_uid
700        )));
701    }
702    validate_source(restore)?;
703    let disaster_recovery_source = if is_remote_source(restore) {
704        validate_backup_ref(restore, &target, ctx).await?
705    } else {
706        None
707    };
708    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709    {
710        return Err(Error::MissingData(format!(
711            "restoreImage must be the target's pinned Kanidm image (target image is {})",
712            target.spec.image
713        )));
714    }
715    let storage = target
716        .spec
717        .storage
718        .as_ref()
719        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720    if storage.empty_dir.is_some()
721        || storage.ephemeral.is_some()
722        || storage.volume_claim_template.is_none()
723    {
724        return Err(Error::MissingData(
725            "restore requires PVC-backed Kanidm storage".to_string(),
726        ));
727    }
728    let primaries = target
729        .spec
730        .replica_groups
731        .iter()
732        .filter(|rg| rg.primary_node)
733        .count();
734    if primaries != 1 {
735        return Err(Error::MissingData(
736            "backup/restore requires exactly one primary replica group".to_string(),
737        ));
738    }
739    let ns = restore.namespace().unwrap();
740    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741        .list(&ListParams::default())
742        .await
743        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744    if restores.items.iter().any(|other| {
745        other.name_any() != restore.name_any()
746            && other.spec.target_ref.name == restore.spec.target_ref.name
747            && !matches!(
748                other.status.as_ref().map(|s| s.phase),
749                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750            )
751    }) {
752        return Err(Error::MissingData(
753            "another active restore targets this Kanidm".to_string(),
754        ));
755    }
756    validate_safety_backup_config(restore)?;
757    Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761    !name.is_empty()
762        && name != "."
763        && name != ".."
764        && !name.contains('/')
765        && !name.contains('\\')
766        && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770    image == "kanidm/server:latest"
771        || image.ends_with(":latest")
772        || (!image.contains('@')
773            && !image
774                .rsplit('/')
775                .next()
776                .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780    let ns = restore
781        .namespace()
782        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784        .get(&restore.spec.target_ref.name)
785        .await
786        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790    restore: &KanidmRestore,
791    target: &Kanidm,
792    ctx: &RestoreContext,
793) -> Result<()> {
794    let ns = target.namespace().unwrap();
795    let uid = restore
796        .uid()
797        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799        .patch(
800            &target.name_any(),
801            &PatchParams::default(),
802            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803        )
804        .await
805        .map(|_| ())
806        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810    restore: &KanidmRestore,
811    target: &Kanidm,
812    ctx: &RestoreContext,
813) -> Result<()> {
814    let ns = target.namespace().unwrap();
815    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816        return Ok(());
817    }
818    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819        .patch(
820            &target.name_any(),
821            &PatchParams::default(),
822            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823        )
824        .await
825        .map(|_| ())
826        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830    restore: &KanidmRestore,
831    ctx: &RestoreContext,
832    mut status: KanidmRestoreStatus,
833) -> Result<()> {
834    let ns = restore.namespace().unwrap();
835    let previous_phase = restore.status.as_ref().map(|current| current.phase);
836    let next_phase = status.phase;
837    status.observed_generation = restore.metadata.generation;
838    update_restore_conditions(&mut status, restore.metadata.generation);
839
840    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841        .patch_status(
842            &restore.name_any(),
843            &PatchParams::default(),
844            &Patch::Merge(json!({"status": &status})),
845        )
846        .await
847        .map_err(|e| {
848            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849        })?;
850
851    if previous_phase != Some(next_phase) {
852        record_restore_transition(
853            restore,
854            ctx,
855            previous_phase,
856            next_phase,
857            status.message.as_deref(),
858        )
859        .await;
860    }
861    Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865    let previous = status.conditions.clone();
866    let phase = status.phase;
867    let phase_reason = format!("{phase:?}");
868    let message = status
869        .message
870        .clone()
871        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872    let terminal_success = phase == KanidmRestorePhase::Completed;
873    let terminal_failure = phase == KanidmRestorePhase::Failed;
874    let progressing = !terminal_success && !terminal_failure;
875
876    let sticky_conditions: Vec<Condition> = previous
877        .iter()
878        .filter(|condition| {
879            condition.type_ == CONDITION_BREAK_GLASS
880                || condition.type_ == CONDITION_DISASTER_RECOVERY
881        })
882        .cloned()
883        .collect();
884    status.conditions = vec![
885        restore_condition(
886            &previous,
887            CONDITION_PROGRESSING,
888            if progressing {
889                CONDITION_TRUE
890            } else {
891                CONDITION_FALSE
892            },
893            &phase_reason,
894            &message,
895            generation,
896        ),
897        restore_condition(
898            &previous,
899            CONDITION_READY,
900            if terminal_success {
901                CONDITION_TRUE
902            } else {
903                CONDITION_FALSE
904            },
905            if terminal_success {
906                "RestoreCompleted"
907            } else {
908                "RestoreNotCompleted"
909            },
910            if terminal_success {
911                "Kanidm restore completed successfully."
912            } else {
913                "Kanidm restore has not completed successfully."
914            },
915            generation,
916        ),
917        restore_condition(
918            &previous,
919            CONDITION_FAILED,
920            if terminal_failure {
921                CONDITION_TRUE
922            } else {
923                CONDITION_FALSE
924            },
925            if terminal_failure {
926                "RestoreFailed"
927            } else {
928                "NoRestoreFailure"
929            },
930            if terminal_failure {
931                status
932                    .message
933                    .as_deref()
934                    .unwrap_or("Kanidm restore failed.")
935            } else {
936                "No terminal restore failure has been recorded."
937            },
938            generation,
939        ),
940    ];
941    status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945    previous: &[Condition],
946    condition_type: &str,
947    condition_status: &str,
948    reason: &str,
949    message: &str,
950    generation: Option<i64>,
951) -> Condition {
952    let last_transition_time = previous
953        .iter()
954        .find(|condition| {
955            condition.type_ == condition_type
956                && condition.status == condition_status
957                && condition.reason == reason
958        })
959        .map(|condition| condition.last_transition_time.clone())
960        .unwrap_or_else(|| Time(Timestamp::now()));
961    Condition {
962        type_: condition_type.to_string(),
963        status: condition_status.to_string(),
964        reason: reason.to_string(),
965        message: message.to_string(),
966        last_transition_time,
967        observed_generation: generation,
968    }
969}
970
971async fn record_restore_transition(
972    restore: &KanidmRestore,
973    ctx: &RestoreContext,
974    previous_phase: Option<KanidmRestorePhase>,
975    phase: KanidmRestorePhase,
976    message: Option<&str>,
977) {
978    if phase == KanidmRestorePhase::Validating {
979        ctx.metrics.attempts.add(1, &[]);
980    }
981
982    let result = match phase {
983        KanidmRestorePhase::Completed => Some("success"),
984        KanidmRestorePhase::Failed => Some("failure"),
985        _ => None,
986    };
987    if let Some(result) = result {
988        let attributes = [
989            KeyValue::new("result", result),
990            KeyValue::new("phase", format!("{phase:?}")),
991        ];
992        ctx.metrics.outcomes.add(1, &attributes);
993        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995            ctx.metrics.duration_seconds.record(elapsed, &attributes);
996        }
997    }
998
999    let reason = if phase == KanidmRestorePhase::Failed {
1000        "RestoreFailed"
1001    } else {
1002        "RestorePhaseChanged"
1003    };
1004    let note = message.map(str::to_string).or_else(|| {
1005        Some(format!(
1006            "Kanidm restore phase changed from {} to {phase:?}.",
1007            previous_phase
1008                .map(|previous| format!("{previous:?}"))
1009                .unwrap_or_else(|| "None".to_string())
1010        ))
1011    });
1012    if let Err(error) = ctx
1013        .recorder
1014        .publish(
1015            &Event {
1016                type_: if phase == KanidmRestorePhase::Failed {
1017                    EventType::Warning
1018                } else {
1019                    EventType::Normal
1020                },
1021                reason: reason.to_string(),
1022                note,
1023                action: "Restore".to_string(),
1024                secondary: None,
1025            },
1026            &restore.object_ref(&()),
1027        )
1028        .await
1029    {
1030        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031    }
1032}
1033
1034async fn set_phase(
1035    restore: &KanidmRestore,
1036    ctx: &RestoreContext,
1037    phase: KanidmRestorePhase,
1038    message: Option<String>,
1039) -> Result<()> {
1040    let mut status = restore.status.clone().unwrap_or_default();
1041    status.phase = phase;
1042    status.message = message;
1043    patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047    restore: &KanidmRestore,
1048    ctx: &RestoreContext,
1049    message: &str,
1050) -> Result<()> {
1051    set_phase(
1052        restore,
1053        ctx,
1054        KanidmRestorePhase::Failed,
1055        Some(message.to_string()),
1056    )
1057    .await
1058}
1059
1060fn restore_primary_failed_status(
1061    existing: &KanidmRestoreStatus,
1062    job_name: &str,
1063) -> KanidmRestoreStatus {
1064    let mut status = existing.clone();
1065    status.restore_job_name = Some(job_name.to_string());
1066    status.phase = KanidmRestorePhase::Failed;
1067    status.message = Some("database restore job failed".to_string());
1068    status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072    let mut status = existing.clone();
1073    status.verify_job_name = Some(job_name.to_string());
1074    status.phase = KanidmRestorePhase::Failed;
1075    status.message = Some("database verification failed".to_string());
1076    status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080    target
1081        .spec
1082        .replica_groups
1083        .iter()
1084        .find(|rg| rg.primary_node)
1085        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089    let ns = target.namespace().unwrap();
1090    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091    for rg in &target.spec.replica_groups {
1092        let name = target.statefulset_name(&rg.name);
1093        api.patch(
1094            &name,
1095            &PatchParams::default(),
1096            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097        )
1098        .await
1099        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100    }
1101    Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105    let ns = target.namespace().unwrap();
1106    let rg = primary_group(target)?;
1107    let name = target.statefulset_name(&rg.name);
1108    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109        .patch(
1110            &name,
1111            &PatchParams::default(),
1112            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113        )
1114        .await
1115        .map(|_| ())
1116        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120    let ns = target.namespace().unwrap();
1121    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122    for rg in &target.spec.replica_groups {
1123        let name = target.statefulset_name(&rg.name);
1124        api.patch(
1125            &name,
1126            &PatchParams::default(),
1127            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128        )
1129        .await
1130        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131    }
1132    Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136    let ns = target.namespace().unwrap();
1137    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139        .await
1140        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141    Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145    let ns = target.namespace().unwrap();
1146    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147    let mut pv_names = BTreeSet::new();
1148    let primary_name = primary_pvc_name(target)?;
1149    let primary_pvc = pvc_api
1150        .get(&primary_name)
1151        .await
1152        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153    let primary_pv = primary_pvc
1154        .spec
1155        .as_ref()
1156        .and_then(|spec| spec.volume_name.clone())
1157        .ok_or_else(|| {
1158            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159        })?;
1160    pv_names.insert(primary_pv);
1161
1162    for rg in &target.spec.replica_groups {
1163        let sts = target.statefulset_name(&rg.name);
1164        for ordinal in 0..rg.replicas {
1165            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166            if name == primary_name {
1167                continue;
1168            }
1169            if let Some(pvc) = pvc_api
1170                .get_opt(&name)
1171                .await
1172                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174            {
1175                pv_names.insert(pv_name);
1176            }
1177        }
1178    }
1179
1180    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181        .list(&ListParams::default())
1182        .await
1183        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184    Ok(!attachments.items.iter().any(|attachment| {
1185        attachment
1186            .spec
1187            .source
1188            .persistent_volume_name
1189            .as_ref()
1190            .is_some_and(|pv_name| pv_names.contains(pv_name))
1191    }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195    let ns = target.namespace().unwrap();
1196    let name = target.statefulset_name(&primary_group(target)?.name);
1197    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198        .get(&name)
1199        .await
1200        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201    Ok(sts
1202        .status
1203        .as_ref()
1204        .and_then(|s| s.ready_replicas)
1205        .unwrap_or(0)
1206        >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210    let rg = primary_group(target)?;
1211    Ok(format!(
1212        "{DATA_VOLUME}-{}-0",
1213        target.statefulset_name(&rg.name)
1214    ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218    let ns = target.namespace().unwrap();
1219    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220    let mut all_absent = true;
1221    for rg in &target.spec.replica_groups {
1222        let sts = target.statefulset_name(&rg.name);
1223        for ordinal in 0..rg.replicas {
1224            if rg.primary_node && ordinal == 0 {
1225                continue;
1226            }
1227            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228            match api
1229                .get_opt(&name)
1230                .await
1231                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232            {
1233                None => {}
1234                Some(pvc) => {
1235                    all_absent = false;
1236                    if pvc.metadata.deletion_timestamp.is_none() {
1237                        match api.delete(&name, &DeleteParams::default()).await {
1238                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1240                            Err(error) => {
1241                                return Err(Error::kube_error(
1242                                    "delete",
1243                                    "PersistentVolumeClaim",
1244                                    &ns,
1245                                    &name,
1246                                    error,
1247                                ));
1248                            }
1249                        }
1250                    }
1251                }
1252            }
1253        }
1254    }
1255    Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259    let ns = target.namespace().unwrap();
1260    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261    let label_selector = format!(
1262        "{}={},{}={}",
1263        CLUSTER_LABEL,
1264        target.name_any(),
1265        SECRET_TYPE_LABEL,
1266        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267    );
1268    let secrets = api
1269        .list(&ListParams::default().labels(&label_selector))
1270        .await
1271        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272    let mut all_absent = secrets.items.is_empty();
1273    for secret in secrets.items {
1274        let name = secret.name_any();
1275        if secret.metadata.deletion_timestamp.is_none() {
1276            match api.delete(&name, &DeleteParams::default()).await {
1277                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278                Err(kube::Error::Api(status)) if status.code == 404 => {}
1279                Err(error) => {
1280                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281                }
1282            }
1283            all_absent = false;
1284        }
1285    }
1286    Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290    let ns = target.namespace().unwrap();
1291    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292    let name = format!("{}-admin-passwords", target.name_any());
1293    match api
1294        .get_opt(&name)
1295        .await
1296        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297    {
1298        None => Ok(true),
1299        Some(secret) => {
1300            if secret.metadata.deletion_timestamp.is_some() {
1301                return Ok(false);
1302            }
1303            match api.delete(&name, &DeleteParams::default()).await {
1304                Ok(_) => {
1305                    debug!(secret = %name, "deleting stale admin secret");
1306                    Ok(false)
1307                }
1308                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310            }
1311        }
1312    }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316    format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319    format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323    let origin = target
1324        .spec
1325        .origin
1326        .clone()
1327        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328    vec![
1329        EnvVar {
1330            name: "KANIDM_DB_PATH".to_string(),
1331            value: Some("/data/kanidm.db".to_string()),
1332            ..Default::default()
1333        },
1334        EnvVar {
1335            name: "KANIDM_DOMAIN".to_string(),
1336            value: Some(target.spec.domain.clone()),
1337            ..Default::default()
1338        },
1339        EnvVar {
1340            name: "KANIDM_ORIGIN".to_string(),
1341            value: Some(origin),
1342            ..Default::default()
1343        },
1344        EnvVar {
1345            name: "KANIDM_TLS_CHAIN".to_string(),
1346            value: Some(format!("{TLS_PATH}/tls.crt")),
1347            ..Default::default()
1348        },
1349        EnvVar {
1350            name: "KANIDM_TLS_KEY".to_string(),
1351            value: Some(format!("{TLS_PATH}/tls.key")),
1352            ..Default::default()
1353        },
1354    ]
1355}
1356
1357async fn ensure_database_job(
1358    restore: &KanidmRestore,
1359    target: &Kanidm,
1360    ctx: &RestoreContext,
1361    name: &str,
1362    verify: bool,
1363) -> Result<()> {
1364    let ns = restore.namespace().unwrap();
1365    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366    if api
1367        .get_opt(name)
1368        .await
1369        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370        .is_some()
1371    {
1372        return Ok(());
1373    }
1374    let mut command = vec![
1375        "kanidmd".to_string(),
1376        "database".to_string(),
1377        if verify { "backup" } else { "restore" }.to_string(),
1378    ];
1379    if verify {
1380        command.push("/verify/verification.json.gz".to_string());
1381    } else {
1382        if is_remote_source(restore) {
1383            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384        } else {
1385            let file_name = restore
1386                .spec
1387                .source
1388                .local
1389                .as_ref()
1390                .map(|l| l.file_name.clone())
1391                .unwrap_or_default();
1392            command.push(format!("{BACKUP_PATH}/{file_name}"));
1393        }
1394    }
1395    let job = Job {
1396        metadata: ObjectMeta {
1397            name: Some(name.to_string()),
1398            namespace: Some(ns.clone()),
1399            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400            ..Default::default()
1401        },
1402        spec: Some(JobSpec {
1403            backoff_limit: Some(0),
1404            template: PodTemplateSpec {
1405                metadata: Some(ObjectMeta::default()),
1406                spec: Some(PodSpec {
1407                    automount_service_account_token: Some(false),
1408                    restart_policy: Some("Never".to_string()),
1409                    security_context: Some(source_job_pod_security_context(target)),
1410                    containers: vec![Container {
1411                        name: if verify { "verify" } else { "restore" }.to_string(),
1412                        image: Some(restore.spec.restore_image.clone()),
1413                        command: Some(command),
1414                        env: Some(build_kanidm_db_env_vars(target)),
1415                        security_context: Some(kanidm_job_security_context()),
1416                        resources: Some(job_resource_requirements()),
1417                        volume_mounts: Some(vec![
1418                            VolumeMount {
1419                                name: DATA_VOLUME.to_string(),
1420                                mount_path: DATA_PATH.to_string(),
1421                                ..Default::default()
1422                            },
1423                            VolumeMount {
1424                                name: TLS_VOLUME.to_string(),
1425                                mount_path: TLS_PATH.to_string(),
1426                                read_only: Some(true),
1427                                ..Default::default()
1428                            },
1429                            VolumeMount {
1430                                name: "verification-output".to_string(),
1431                                mount_path: "/verify".to_string(),
1432                                ..Default::default()
1433                            },
1434                        ]),
1435                        ..Default::default()
1436                    }],
1437                    volumes: Some(vec![
1438                        Volume {
1439                            name: DATA_VOLUME.to_string(),
1440                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441                                claim_name: primary_pvc_name(target)?,
1442                                read_only: Some(false),
1443                            }),
1444                            ..Default::default()
1445                        },
1446                        Volume {
1447                            name: TLS_VOLUME.to_string(),
1448                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449                                secret_name: Some(target.effective_tls_secret_name()),
1450                                ..Default::default()
1451                            }),
1452                            ..Default::default()
1453                        },
1454                        Volume {
1455                            name: "verification-output".to_string(),
1456                            empty_dir: Some(Default::default()),
1457                            ..Default::default()
1458                        },
1459                    ]),
1460                    ..Default::default()
1461                }),
1462            },
1463            ..Default::default()
1464        }),
1465        ..Default::default()
1466    };
1467    api.create(&PostParams::default(), &job)
1468        .await
1469        .map(|_| ())
1470        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474    Running,
1475    Complete,
1476    Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480    let ns = restore.namespace().unwrap();
1481    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482        .get(name)
1483        .await
1484        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486        return Ok(JobState::Failed);
1487    }
1488    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489        return Ok(JobState::Complete);
1490    }
1491    Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495    restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499    let safety = restore.spec.safety_backup.as_ref();
1500    let skip = safety.is_some_and(|s| s.skip);
1501    !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505    let has_local = restore.spec.source.local.is_some();
1506    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507    if has_local && has_backup_ref {
1508        return Err(Error::MissingData(
1509            "source.local and source.backupRef are mutually exclusive".to_string(),
1510        ));
1511    }
1512    if !has_local && !has_backup_ref {
1513        return Err(Error::MissingData(
1514            "source must specify either local or backupRef".to_string(),
1515        ));
1516    }
1517    if let Some(local) = &restore.spec.source.local {
1518        if !safe_basename(&local.file_name) {
1519            return Err(Error::MissingData(
1520                "restore source fileName must be a safe basename".to_string(),
1521            ));
1522        }
1523    }
1524    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525        if backup_ref.name.is_empty() {
1526            return Err(Error::MissingData(
1527                "source.backupRef.name must not be empty".to_string(),
1528            ));
1529        }
1530    }
1531    Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535    restore
1536        .metadata
1537        .annotations
1538        .as_ref()
1539        .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540        .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544    let annotations = restore.metadata.annotations.as_ref();
1545    let reason = annotations
1546        .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547        .map(String::as_str)
1548        .filter(|value| !value.trim().is_empty());
1549    let approved_by = annotations
1550        .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551        .map(String::as_str)
1552        .filter(|value| !value.trim().is_empty());
1553
1554    if reason.is_none() || approved_by.is_none() {
1555        return Err(Error::MissingData(format!(
1556            "disaster-recovery restore requires non-empty {} and {} annotations",
1557            BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558        )));
1559    }
1560    Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564    restore: &KanidmRestore,
1565    target: &Kanidm,
1566    ctx: &RestoreContext,
1567    source: &DisasterRecoverySource,
1568) {
1569    ctx.metrics.disaster_recovery_total.add(1, &[]);
1570    let reason = restore
1571        .metadata
1572        .annotations
1573        .as_ref()
1574        .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575        .cloned()
1576        .unwrap_or_default();
1577    let approved_by = restore
1578        .metadata
1579        .annotations
1580        .as_ref()
1581        .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582        .cloned()
1583        .unwrap_or_default();
1584    let target_uid = target.uid().unwrap_or_default();
1585    let note = format!(
1586        "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587        source.name,
1588        source.uid,
1589        target.name_any(),
1590        target_uid,
1591        reason,
1592        approved_by
1593    );
1594    if let Err(error) = ctx
1595        .recorder
1596        .publish(
1597            &Event {
1598                type_: EventType::Warning,
1599                reason: "DisasterRecoveryOverride".to_string(),
1600                note: Some(note.clone()),
1601                action: "Restore".to_string(),
1602                secondary: None,
1603            },
1604            &restore.object_ref(&()),
1605        )
1606        .await
1607    {
1608        warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609    }
1610    warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614    let safety = restore.spec.safety_backup.as_ref();
1615    let skip = safety.is_some_and(|s| s.skip);
1616    if skip {
1617        let annotations = restore.metadata.annotations.as_ref();
1618        let reason = annotations
1619            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620            .map(|s| s.trim())
1621            .unwrap_or("");
1622        let approver = annotations
1623            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624            .map(|s| s.trim())
1625            .unwrap_or("");
1626        if reason.is_empty() {
1627            return Err(Error::MissingData(format!(
1628                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629            )));
1630        }
1631        if approver.is_empty() {
1632            return Err(Error::MissingData(format!(
1633                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634            )));
1635        }
1636    }
1637    if !skip {
1638        let has_repo = safety
1639            .and_then(|s| s.repository_ref.as_ref())
1640            .is_some_and(|r| !r.name.is_empty());
1641        if !has_repo {
1642            return Err(Error::MissingData(
1643                "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644                    .to_string(),
1645            ));
1646        }
1647    }
1648    Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652    target
1653        .spec
1654        .replica_groups
1655        .iter()
1656        .map(|rg| ReplicaCountEntry {
1657            group: rg.name.clone(),
1658            replicas: rg.replicas,
1659        })
1660        .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664    if let Ok(target) = get_target(restore, ctx).await {
1665        let owns_maintenance =
1666            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667        if owns_maintenance {
1668            scale_desired(&target, ctx).await?;
1669            clear_restoring(restore, &target, ctx).await?;
1670        }
1671    }
1672    Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676    ctx.metrics.break_glass_total.add(1, &[]);
1677    let reason = restore
1678        .metadata
1679        .annotations
1680        .as_ref()
1681        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682        .cloned()
1683        .unwrap_or_default();
1684    let approver = restore
1685        .metadata
1686        .annotations
1687        .as_ref()
1688        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689        .cloned()
1690        .unwrap_or_default();
1691    let mut status = restore.status.clone().unwrap_or_default();
1692    let break_glass_condition = restore_condition(
1693        &status.conditions,
1694        CONDITION_BREAK_GLASS,
1695        CONDITION_TRUE,
1696        "BreakGlassOverride",
1697        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698        restore.metadata.generation,
1699    );
1700    status
1701        .conditions
1702        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703    status.conditions.push(break_glass_condition);
1704    patch_status(restore, ctx, status).await?;
1705    if let Err(error) = ctx
1706        .recorder
1707        .publish(
1708            &Event {
1709                type_: EventType::Warning,
1710                reason: "SafetyBackupSkipped".to_string(),
1711                note: Some(format!(
1712                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713                )),
1714                action: "Restore".to_string(),
1715                secondary: None,
1716            },
1717            &restore.object_ref(&()),
1718        )
1719        .await
1720    {
1721        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722    }
1723    warn!(
1724        restore = %restore.name_any(),
1725        reason = %reason,
1726        approver = %approver,
1727        "break-glass safety backup override activated"
1728    );
1729    Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733    let status = restore.status.as_ref().cloned().unwrap_or_default();
1734    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735        if let Ok(start_ts) = start.parse::<Timestamp>() {
1736            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737            ctx.metrics
1738                .safety_backup_duration_seconds
1739                .record(elapsed, &[]);
1740        }
1741    }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745    format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749    format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753    format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757    restore: &KanidmRestore,
1758    target: &Kanidm,
1759    ctx: &RestoreContext,
1760    name: &str,
1761) -> Result<()> {
1762    let ns = restore.namespace().unwrap();
1763    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764    if api
1765        .get_opt(name)
1766        .await
1767        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768        .is_some()
1769    {
1770        return Ok(());
1771    }
1772    let file_name = restore
1773        .spec
1774        .source
1775        .local
1776        .as_ref()
1777        .map(|l| l.file_name.clone())
1778        .unwrap_or_default();
1779    let operation_doc = serde_json::json!({
1780        "apiVersion": "backup.kaniop.rs/v1alpha1",
1781        "kind": "OperationDocument",
1782        "operation": "check",
1783        "path": format!("{BACKUP_PATH}/{file_name}"),
1784        "resultPath": "/run/kaniop-result/result.json",
1785        "format": "kanidmJsonGzip",
1786    })
1787    .to_string();
1788    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790    let data_mover_image = data_mover_image();
1791    let job = Job {
1792        metadata: ObjectMeta {
1793            name: Some(name.to_string()),
1794            namespace: Some(ns.clone()),
1795            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796            ..Default::default()
1797        },
1798        spec: Some(JobSpec {
1799            backoff_limit: Some(0),
1800            template: PodTemplateSpec {
1801                metadata: Some(ObjectMeta::default()),
1802                spec: Some(PodSpec {
1803                    automount_service_account_token: Some(false),
1804                    restart_policy: Some("Never".to_string()),
1805                    security_context: Some(source_job_pod_security_context(target)),
1806                    containers: vec![Container {
1807                        name: "source-check".to_string(),
1808                        image: Some(data_mover_image),
1809                        command: Some(vec![
1810                            "/bin/kaniop-data-mover".to_string(),
1811                            "check".to_string(),
1812                            "--operation-doc".to_string(),
1813                            "/run/kaniop/operation.json".to_string(),
1814                        ]),
1815                        security_context: Some(hardened_security_context()),
1816                        resources: Some(job_resource_requirements()),
1817                        env: Some(vec![EnvVar {
1818                            name: "RUST_LOG".to_string(),
1819                            value: Some("info".to_string()),
1820                            ..Default::default()
1821                        }]),
1822                        volume_mounts: Some(vec![
1823                            VolumeMount {
1824                                name: DATA_VOLUME.to_string(),
1825                                mount_path: DATA_PATH.to_string(),
1826                                read_only: Some(true),
1827                                ..Default::default()
1828                            },
1829                            VolumeMount {
1830                                name: "operation".to_string(),
1831                                mount_path: "/run/kaniop".to_string(),
1832                                read_only: Some(true),
1833                                ..Default::default()
1834                            },
1835                            VolumeMount {
1836                                name: "result".to_string(),
1837                                mount_path: "/run/kaniop-result".to_string(),
1838                                ..Default::default()
1839                            },
1840                        ]),
1841                        ..Default::default()
1842                    }],
1843                    volumes: Some(vec![
1844                        Volume {
1845                            name: DATA_VOLUME.to_string(),
1846                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847                                claim_name: primary_pvc_name(target)?,
1848                                read_only: Some(true),
1849                            }),
1850                            ..Default::default()
1851                        },
1852                        Volume {
1853                            name: "operation".to_string(),
1854                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855                                name: operation_cm_name,
1856                                ..Default::default()
1857                            }),
1858                            ..Default::default()
1859                        },
1860                        Volume {
1861                            name: "result".to_string(),
1862                            empty_dir: Some(Default::default()),
1863                            ..Default::default()
1864                        },
1865                    ]),
1866                    ..Default::default()
1867                }),
1868            },
1869            ..Default::default()
1870        }),
1871        ..Default::default()
1872    };
1873    api.create(&PostParams::default(), &job)
1874        .await
1875        .map(|_| ())
1876        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882    let ns = restore.namespace().unwrap_or_default();
1883    let uid = restore.uid().ok_or_else(|| {
1884        Error::MissingData(
1885            "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886                .to_string(),
1887        )
1888    })?;
1889    let namespace_uuid =
1890        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892    Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895/// Compute the safety-backup manifest key.
1896///
1897/// This key is hand-rolled with `format!` instead of going through
1898/// [`kaniop_backup_core::paths::RepositoryPath`], so it is the site most exposed to keyspace drift:
1899/// it must stay byte-identical to `RepositoryPath::manifest_key`, and its `tenants/{ns}` segment is
1900/// the Kubernetes namespace NAME (`restore.namespace()`), not the Namespace object UID, matching the
1901/// `namespaceUid` wire field the upload operation emits (misnomer tracked in issue #1084).
1902fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1903    let ns = restore.namespace().unwrap_or_default();
1904    let kanidm_uid = &restore.spec.target_ref.uid;
1905    let backup_id = compute_safety_backup_id(restore)?;
1906    Ok(format!(
1907        "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1908    ))
1909}
1910
1911struct VerifiedSafetyBackup {
1912    manifest_key: String,
1913    payload_sha256: String,
1914}
1915
1916async fn read_safety_backup_result(
1917    restore: &KanidmRestore,
1918    ctx: &RestoreContext,
1919    job_name: &str,
1920    expected_backup_id: &str,
1921) -> Result<VerifiedSafetyBackup> {
1922    let ns = restore.namespace().unwrap();
1923    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924    let pod_list = pods
1925        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926        .await
1927        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928    let pod = pod_list.items.first().ok_or_else(|| {
1929        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1930    })?;
1931    let pod_name = pod.name_any();
1932    let container_status = pod
1933        .status
1934        .as_ref()
1935        .and_then(|s| s.container_statuses.as_ref())
1936        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1937        .ok_or_else(|| {
1938            Error::MissingData(format!(
1939                "safety-upload container status not found in pod {pod_name}"
1940            ))
1941        })?;
1942    let termination_message = container_status
1943        .state
1944        .as_ref()
1945        .and_then(|state| state.terminated.as_ref())
1946        .and_then(|t| t.message.as_ref())
1947        .ok_or_else(|| {
1948            Error::MissingData(format!(
1949                "termination message not found in pod {pod_name}; result transport failed"
1950            ))
1951        })?;
1952    if termination_message.is_empty() {
1953        return Err(Error::MissingData(format!(
1954            "termination message is empty in pod {pod_name}; result document was not written"
1955        )));
1956    }
1957    let result_doc = parse_result_document(termination_message).map_err(|e| {
1958        Error::ParseError(format!(
1959            "failed to parse result document from pod {pod_name}: {e}"
1960        ))
1961    })?;
1962    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963        return Err(Error::ParseError(format!(
1964            "unsupported result document version: {}",
1965            result_doc.api_version
1966        )));
1967    }
1968    if result_doc.kind != "ResultDocument" {
1969        return Err(Error::ParseError(format!(
1970            "invalid result document kind: {}",
1971            result_doc.kind
1972        )));
1973    }
1974    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1975        return Err(Error::ParseError(format!(
1976            "result document operation mismatch: expected '{}', got '{}'",
1977            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1978        )));
1979    }
1980    if !result_doc.success {
1981        return Err(Error::ParseError(format!(
1982            "safety backup reported failure with exit code {:?}",
1983            result_doc.exit_code
1984        )));
1985    }
1986    if result_doc.exit_code != ExitCode::Success {
1987        return Err(Error::ParseError(format!(
1988            "safety backup exit code is not Success: {:?}",
1989            result_doc.exit_code
1990        )));
1991    }
1992    let doc_backup_id = result_doc
1993        .backup_id
1994        .as_deref()
1995        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996    if doc_backup_id != expected_backup_id {
1997        return Err(Error::ParseError(format!(
1998            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999        )));
2000    }
2001    let manifest_key = result_doc
2002        .manifest_key
2003        .clone()
2004        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005    if manifest_key.is_empty() {
2006        return Err(Error::ParseError(
2007            "result document manifestKey is empty".to_string(),
2008        ));
2009    }
2010    let payload_sha256 = result_doc
2011        .payload_sha256
2012        .clone()
2013        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014    if payload_sha256.is_empty() {
2015        return Err(Error::ParseError(
2016            "result document payloadSha256 is empty".to_string(),
2017        ));
2018    }
2019    info!(
2020        restore = %restore.name_any(),
2021        backup_id = %expected_backup_id,
2022        manifest_key = %manifest_key,
2023        "safety backup result verified"
2024    );
2025    Ok(VerifiedSafetyBackup {
2026        manifest_key,
2027        payload_sha256,
2028    })
2029}
2030
2031struct VerifiedSourcePrepResult {
2032    #[allow(dead_code)]
2033    manifest_key: String,
2034    payload_sha256: String,
2035}
2036
2037async fn read_source_prep_result(
2038    restore: &KanidmRestore,
2039    ctx: &RestoreContext,
2040    job_name: &str,
2041    expected_backup_id: &str,
2042) -> Result<VerifiedSourcePrepResult> {
2043    let ns = restore.namespace().unwrap();
2044    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2045    let pod_list = pods
2046        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2047        .await
2048        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2049    let pod = pod_list.items.first().ok_or_else(|| {
2050        Error::MissingData(format!("no pod found for source prep job {job_name}"))
2051    })?;
2052    let pod_name = pod.name_any();
2053    let container_status = pod
2054        .status
2055        .as_ref()
2056        .and_then(|s| s.container_statuses.as_ref())
2057        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2058        .ok_or_else(|| {
2059            Error::MissingData(format!(
2060                "source-prep container status not found in pod {pod_name}"
2061            ))
2062        })?;
2063    let termination_message = container_status
2064        .state
2065        .as_ref()
2066        .and_then(|state| state.terminated.as_ref())
2067        .and_then(|t| t.message.as_ref())
2068        .ok_or_else(|| {
2069            Error::MissingData(format!(
2070                "termination message not found in pod {pod_name}; result transport failed"
2071            ))
2072        })?;
2073    if termination_message.is_empty() {
2074        return Err(Error::MissingData(format!(
2075            "termination message is empty in pod {pod_name}; result document was not written"
2076        )));
2077    }
2078    let result_doc = parse_result_document(termination_message).map_err(|e| {
2079        Error::ParseError(format!(
2080            "failed to parse result document from pod {pod_name}: {e}"
2081        ))
2082    })?;
2083    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2084        return Err(Error::ParseError(format!(
2085            "unsupported result document version: {}",
2086            result_doc.api_version
2087        )));
2088    }
2089    if result_doc.kind != "ResultDocument" {
2090        return Err(Error::ParseError(format!(
2091            "invalid result document kind: {}",
2092            result_doc.kind
2093        )));
2094    }
2095    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2096        return Err(Error::ParseError(format!(
2097            "result document operation mismatch: expected '{}', got '{}'",
2098            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2099        )));
2100    }
2101    if !result_doc.success {
2102        return Err(Error::ParseError(format!(
2103            "source prep reported failure with exit code {:?}",
2104            result_doc.exit_code
2105        )));
2106    }
2107    if result_doc.exit_code != ExitCode::Success {
2108        return Err(Error::ParseError(format!(
2109            "source prep exit code is not Success: {:?}",
2110            result_doc.exit_code
2111        )));
2112    }
2113    let doc_backup_id = result_doc
2114        .backup_id
2115        .as_deref()
2116        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2117    if doc_backup_id != expected_backup_id {
2118        return Err(Error::ParseError(format!(
2119            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2120        )));
2121    }
2122    let manifest_key = result_doc
2123        .manifest_key
2124        .clone()
2125        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2126    if manifest_key.is_empty() {
2127        return Err(Error::ParseError(
2128            "result document manifestKey is empty".to_string(),
2129        ));
2130    }
2131    let payload_sha256 = result_doc
2132        .payload_sha256
2133        .clone()
2134        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2135    if payload_sha256.is_empty() {
2136        return Err(Error::ParseError(
2137            "result document payloadSha256 is empty".to_string(),
2138        ));
2139    }
2140    info!(
2141        restore = %restore.name_any(),
2142        backup_id = %expected_backup_id,
2143        manifest_key = %manifest_key,
2144        payload_sha256 = %payload_sha256,
2145        "source prep result verified"
2146    );
2147    Ok(VerifiedSourcePrepResult {
2148        manifest_key,
2149        payload_sha256,
2150    })
2151}
2152
2153fn source_job_pod_security_context(
2154    target: &Kanidm,
2155) -> k8s_openapi::api::core::v1::PodSecurityContext {
2156    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2157    security_context.seccomp_profile = Some(SeccompProfile {
2158        type_: "RuntimeDefault".to_string(),
2159        ..Default::default()
2160    });
2161    security_context
2162}
2163
2164fn hardened_security_context() -> SecurityContext {
2165    SecurityContext {
2166        allow_privilege_escalation: Some(false),
2167        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2168            drop: Some(vec!["ALL".to_string()]),
2169            ..Default::default()
2170        }),
2171        read_only_root_filesystem: Some(true),
2172        run_as_non_root: Some(true),
2173        seccomp_profile: Some(SeccompProfile {
2174            type_: "RuntimeDefault".to_string(),
2175            ..Default::default()
2176        }),
2177        ..Default::default()
2178    }
2179}
2180
2181fn kanidm_job_security_context() -> SecurityContext {
2182    SecurityContext {
2183        allow_privilege_escalation: Some(false),
2184        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2185            drop: Some(vec!["ALL".to_string()]),
2186            ..Default::default()
2187        }),
2188        seccomp_profile: Some(SeccompProfile {
2189            type_: "RuntimeDefault".to_string(),
2190            ..Default::default()
2191        }),
2192        ..Default::default()
2193    }
2194}
2195
2196fn job_resource_requirements() -> ResourceRequirements {
2197    ResourceRequirements {
2198        requests: Some(std::collections::BTreeMap::from([
2199            ("cpu".to_string(), Quantity("100m".to_string())),
2200            ("memory".to_string(), Quantity("128Mi".to_string())),
2201        ])),
2202        limits: Some(std::collections::BTreeMap::from([
2203            ("cpu".to_string(), Quantity("2".to_string())),
2204            ("memory".to_string(), Quantity("2Gi".to_string())),
2205        ])),
2206        ..Default::default()
2207    }
2208}
2209
2210async fn ensure_safety_backup_job(
2211    restore: &KanidmRestore,
2212    target: &Kanidm,
2213    ctx: &RestoreContext,
2214    name: &str,
2215    backup_id: &str,
2216    manifest_key: &str,
2217) -> Result<()> {
2218    let ns = restore.namespace().unwrap();
2219    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2220    if api
2221        .get_opt(name)
2222        .await
2223        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2224        .is_some()
2225    {
2226        return Ok(());
2227    }
2228    let safety = restore
2229        .spec
2230        .safety_backup
2231        .as_ref()
2232        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2233    let repo_ref = safety
2234        .repository_ref
2235        .as_ref()
2236        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2237    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2238        .get(&repo_ref.name)
2239        .await
2240        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2241    let _kanidm_uid = &restore.spec.target_ref.uid;
2242    let endpoint = &repo.spec.s3.endpoint;
2243    let region = &repo.spec.s3.region;
2244    let operation_doc = build_safety_upload_operation_doc(
2245        restore,
2246        target,
2247        backup_id,
2248        manifest_key,
2249        &repo.spec.s3.bucket,
2250        &repo.spec.s3.prefix,
2251        endpoint,
2252        region,
2253        repo.spec.s3.force_path_style,
2254        repo.spec.s3.insecure,
2255        repo.spec
2256            .s3
2257            .ca_bundle_ref
2258            .as_ref()
2259            .map(|_| ca_bundle_path())
2260            .as_deref(),
2261        repo.spec.encryption.as_ref(),
2262    )?;
2263    let operation_cm_name = format!("{name}-op");
2264    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2265    let auth_method = &repo.spec.authentication.writer;
2266    let data_mover_image = data_mover_image();
2267    let init_command = vec![
2268        "kanidmd".to_string(),
2269        "database".to_string(),
2270        "backup".to_string(),
2271        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2272    ];
2273    let upload_script = format!(
2274        r#"set -eu
2275/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2276RESULT_FILE="/run/kaniop-result/result.json"
2277TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2278if [ ! -f "$RESULT_FILE" ]; then
2279  echo "ERROR: result document not found at $RESULT_FILE"
2280  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2281  exit 1
2282fi
2283if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2284  echo "ERROR: safety backup upload reported failure"
2285  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2286  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2287  exit 1
2288fi
2289dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2290echo "safety backup upload completed successfully"
2291"#
2292    );
2293    let job = Job {
2294        metadata: ObjectMeta {
2295            name: Some(name.to_string()),
2296            namespace: Some(ns.clone()),
2297            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2298            ..Default::default()
2299        },
2300        spec: Some(JobSpec {
2301            backoff_limit: Some(0),
2302            template: PodTemplateSpec {
2303                metadata: Some(ObjectMeta::default()),
2304                spec: Some(PodSpec {
2305                    automount_service_account_token: Some(false),
2306                    restart_policy: Some("Never".to_string()),
2307                    security_context: Some(source_job_pod_security_context(target)),
2308                    init_containers: Some(vec![Container {
2309                        name: "safety-backup".to_string(),
2310                        image: Some(restore.spec.restore_image.clone()),
2311                        command: Some(init_command),
2312                        security_context: Some(SecurityContext {
2313                            allow_privilege_escalation: Some(false),
2314                            capabilities: Some(Capabilities {
2315                                drop: Some(vec!["ALL".to_string()]),
2316                                ..Default::default()
2317                            }),
2318                            read_only_root_filesystem: Some(true),
2319                            seccomp_profile: Some(SeccompProfile {
2320                                type_: "RuntimeDefault".to_string(),
2321                                ..Default::default()
2322                            }),
2323                            ..Default::default()
2324                        }),
2325                        resources: Some(job_resource_requirements()),
2326                        env: Some(build_kanidm_db_env_vars(target)),
2327                        volume_mounts: Some(vec![
2328                            VolumeMount {
2329                                name: DATA_VOLUME.to_string(),
2330                                mount_path: DATA_PATH.to_string(),
2331                                read_only: Some(false),
2332                                ..Default::default()
2333                            },
2334                            VolumeMount {
2335                                name: TLS_VOLUME.to_string(),
2336                                mount_path: TLS_PATH.to_string(),
2337                                read_only: Some(true),
2338                                ..Default::default()
2339                            },
2340                            VolumeMount {
2341                                name: SHARED_VOLUME.to_string(),
2342                                mount_path: SHARED_VOL_PATH.to_string(),
2343                                ..Default::default()
2344                            },
2345                        ]),
2346                        ..Default::default()
2347                    }]),
2348                    containers: vec![Container {
2349                        name: "safety-upload".to_string(),
2350                        image: Some(data_mover_image),
2351                        command: Some(vec!["/bin/sh".to_string()]),
2352                        args: Some(vec!["-c".to_string(), upload_script]),
2353                        security_context: Some(hardened_security_context()),
2354                        resources: Some(job_resource_requirements()),
2355                        env: {
2356                            let mut env_vars =
2357                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2358                            env_vars.push(EnvVar {
2359                                name: "RUST_LOG".to_string(),
2360                                value: Some("info".to_string()),
2361                                ..Default::default()
2362                            });
2363                            if repo.spec.s3.ca_bundle_ref.is_some() {
2364                                env_vars.push(ca_bundle_env_var());
2365                            }
2366                            env_vars.extend(build_encryption_env_vars(
2367                                repo.spec
2368                                    .encryption
2369                                    .as_ref()
2370                                    .and_then(|e| e.key_ref.as_ref()),
2371                            ));
2372                            Some(env_vars)
2373                        },
2374                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2375                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2376                        volume_mounts: {
2377                            let mut mounts = vec![
2378                                VolumeMount {
2379                                    name: SHARED_VOLUME.to_string(),
2380                                    mount_path: SHARED_VOL_PATH.to_string(),
2381                                    read_only: Some(true),
2382                                    ..Default::default()
2383                                },
2384                                VolumeMount {
2385                                    name: "operation".to_string(),
2386                                    mount_path: "/run/kaniop".to_string(),
2387                                    read_only: Some(true),
2388                                    ..Default::default()
2389                                },
2390                                VolumeMount {
2391                                    name: "result".to_string(),
2392                                    mount_path: "/run/kaniop-result".to_string(),
2393                                    ..Default::default()
2394                                },
2395                            ];
2396                            mounts.extend(build_auth_volume_mounts(auth_method));
2397                            if repo.spec.s3.ca_bundle_ref.is_some() {
2398                                mounts.push(build_ca_bundle_volume_mount());
2399                            }
2400                            Some(mounts)
2401                        },
2402                        ..Default::default()
2403                    }],
2404                    volumes: {
2405                        let mut vols = vec![
2406                            Volume {
2407                                name: DATA_VOLUME.to_string(),
2408                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2409                                    claim_name: primary_pvc_name(target)?,
2410                                    read_only: Some(false),
2411                                }),
2412                                ..Default::default()
2413                            },
2414                            Volume {
2415                                name: TLS_VOLUME.to_string(),
2416                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2417                                    secret_name: Some(target.effective_tls_secret_name()),
2418                                    ..Default::default()
2419                                }),
2420                                ..Default::default()
2421                            },
2422                            Volume {
2423                                name: SHARED_VOLUME.to_string(),
2424                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2425                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2426                                    ..Default::default()
2427                                }),
2428                                ..Default::default()
2429                            },
2430                            Volume {
2431                                name: "operation".to_string(),
2432                                config_map: Some(ConfigMapVolumeSource {
2433                                    name: operation_cm_name,
2434                                    ..Default::default()
2435                                }),
2436                                ..Default::default()
2437                            },
2438                            Volume {
2439                                name: "result".to_string(),
2440                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2441                                    size_limit: Some(Quantity("16Mi".to_string())),
2442                                    ..Default::default()
2443                                }),
2444                                ..Default::default()
2445                            },
2446                        ];
2447                        vols.extend(build_auth_volumes(auth_method));
2448                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2449                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2450                        }
2451                        Some(vols)
2452                    },
2453                    ..Default::default()
2454                }),
2455            },
2456            ..Default::default()
2457        }),
2458        ..Default::default()
2459    };
2460    api.create(&PostParams::default(), &job)
2461        .await
2462        .map(|_| ())
2463        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2464}
2465
2466async fn ensure_source_prep_job(
2467    restore: &KanidmRestore,
2468    target: &Kanidm,
2469    ctx: &RestoreContext,
2470    name: &str,
2471) -> Result<()> {
2472    let ns = restore.namespace().unwrap();
2473    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2474    if api
2475        .get_opt(name)
2476        .await
2477        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2478        .is_some()
2479    {
2480        return Ok(());
2481    }
2482    let backup_name = restore
2483        .spec
2484        .source
2485        .backup_ref
2486        .as_ref()
2487        .map(|b| b.name.clone())
2488        .unwrap_or_default();
2489    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2490        .get(&backup_name)
2491        .await
2492        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2493    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2494        .get(&backup.spec.repository_ref.name)
2495        .await
2496        .map_err(|e| {
2497            Error::kube_error(
2498                "get",
2499                "KanidmBackupRepository",
2500                &ns,
2501                &backup.spec.repository_ref.name,
2502                e,
2503            )
2504        })?;
2505    let endpoint = &repo.spec.s3.endpoint;
2506    let region = &repo.spec.s3.region;
2507    let operation_doc = build_download_operation_doc(
2508        target,
2509        &backup.spec.manifest_key,
2510        &backup.spec.backup_id,
2511        &backup.spec.kanidm_ref.uid,
2512        &repo.spec.s3.bucket,
2513        &repo.spec.s3.prefix,
2514        endpoint,
2515        region,
2516        repo.spec.s3.force_path_style,
2517        repo.spec.s3.insecure,
2518        repo.spec
2519            .s3
2520            .ca_bundle_ref
2521            .as_ref()
2522            .map(|_| ca_bundle_path())
2523            .as_deref(),
2524        repo.spec.encryption.as_ref(),
2525    );
2526    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2527    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2528    let auth_method = &repo.spec.authentication.reader;
2529    let data_mover_image = data_mover_image();
2530    let download_script = format!(
2531        r#"set -eu
2532/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2533RESULT_FILE="/run/kaniop-result/result.json"
2534TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2535if [ ! -f "$RESULT_FILE" ]; then
2536  echo "ERROR: result document not found at $RESULT_FILE"
2537  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2538  exit 1
2539fi
2540if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2541  echo "ERROR: source preparation download reported failure"
2542  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2543  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2544  exit 1
2545fi
2546dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2547echo "source preparation download completed successfully"
2548"#
2549    );
2550    let job = Job {
2551        metadata: ObjectMeta {
2552            name: Some(name.to_string()),
2553            namespace: Some(ns.clone()),
2554            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2555            ..Default::default()
2556        },
2557        spec: Some(JobSpec {
2558            backoff_limit: Some(0),
2559            template: PodTemplateSpec {
2560                metadata: Some(ObjectMeta::default()),
2561                spec: Some(PodSpec {
2562                    automount_service_account_token: Some(false),
2563                    restart_policy: Some("Never".to_string()),
2564                    security_context: Some(source_job_pod_security_context(target)),
2565                    containers: vec![Container {
2566                        name: "source-prep".to_string(),
2567                        image: Some(data_mover_image),
2568                        command: Some(vec!["/bin/sh".to_string()]),
2569                        args: Some(vec!["-c".to_string(), download_script]),
2570                        security_context: Some(hardened_security_context()),
2571                        resources: Some(job_resource_requirements()),
2572                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2573                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2574                        env: {
2575                            let mut env_vars = build_auth_env_vars(
2576                                auth_method,
2577                                &backup.spec.repository_ref.name,
2578                                AuthRole::Reader,
2579                            );
2580                            env_vars.push(EnvVar {
2581                                name: "RUST_LOG".to_string(),
2582                                value: Some("info".to_string()),
2583                                ..Default::default()
2584                            });
2585                            if repo.spec.s3.ca_bundle_ref.is_some() {
2586                                env_vars.push(ca_bundle_env_var());
2587                            }
2588                            env_vars.extend(build_encryption_env_vars(
2589                                repo.spec
2590                                    .encryption
2591                                    .as_ref()
2592                                    .and_then(|e| e.key_ref.as_ref()),
2593                            ));
2594                            Some(env_vars)
2595                        },
2596                        volume_mounts: {
2597                            let mut mounts = vec![
2598                                VolumeMount {
2599                                    name: DATA_VOLUME.to_string(),
2600                                    mount_path: DATA_PATH.to_string(),
2601                                    ..Default::default()
2602                                },
2603                                VolumeMount {
2604                                    name: "operation".to_string(),
2605                                    mount_path: "/run/kaniop".to_string(),
2606                                    read_only: Some(true),
2607                                    ..Default::default()
2608                                },
2609                                VolumeMount {
2610                                    name: "result".to_string(),
2611                                    mount_path: "/run/kaniop-result".to_string(),
2612                                    ..Default::default()
2613                                },
2614                            ];
2615                            mounts.extend(build_auth_volume_mounts(auth_method));
2616                            if repo.spec.s3.ca_bundle_ref.is_some() {
2617                                mounts.push(build_ca_bundle_volume_mount());
2618                            }
2619                            Some(mounts)
2620                        },
2621                        ..Default::default()
2622                    }],
2623                    volumes: {
2624                        let mut vols = vec![
2625                            Volume {
2626                                name: DATA_VOLUME.to_string(),
2627                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2628                                    claim_name: primary_pvc_name(target)?,
2629                                    read_only: Some(false),
2630                                }),
2631                                ..Default::default()
2632                            },
2633                            Volume {
2634                                name: "operation".to_string(),
2635                                config_map: Some(ConfigMapVolumeSource {
2636                                    name: operation_cm_name,
2637                                    ..Default::default()
2638                                }),
2639                                ..Default::default()
2640                            },
2641                            Volume {
2642                                name: "result".to_string(),
2643                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2644                                    size_limit: Some(Quantity("16Mi".to_string())),
2645                                    ..Default::default()
2646                                }),
2647                                ..Default::default()
2648                            },
2649                        ];
2650                        vols.extend(build_auth_volumes(auth_method));
2651                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2652                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2653                        }
2654                        Some(vols)
2655                    },
2656                    ..Default::default()
2657                }),
2658            },
2659            ..Default::default()
2660        }),
2661        ..Default::default()
2662    };
2663    api.create(&PostParams::default(), &job)
2664        .await
2665        .map(|_| ())
2666        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2667}
2668
2669#[allow(clippy::too_many_arguments)]
2670fn build_safety_upload_operation_doc(
2671    restore: &KanidmRestore,
2672    target: &Kanidm,
2673    backup_id: &str,
2674    manifest_key: &str,
2675    bucket: &str,
2676    prefix: &str,
2677    endpoint: &str,
2678    region: &str,
2679    force_path_style: bool,
2680    insecure: bool,
2681    ca_bundle_path: Option<&str>,
2682    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2683) -> Result<String> {
2684    let enc_mode =
2685        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2686    let enc_key_id = encryption
2687        .and_then(|e| e.key_id.as_ref())
2688        .map(|s| serde_json::Value::String(s.clone()));
2689    let op = UploadOperation {
2690        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2691        bucket: bucket.to_string(),
2692        prefix: prefix.to_string(),
2693        endpoint: endpoint.to_string(),
2694        region: region.to_string(),
2695        force_path_style,
2696        insecure,
2697        ca_bundle_path: ca_bundle_path.map(str::to_string),
2698        backup_id: backup_id.to_string(),
2699        // `UploadOperation.namespace_uid` carries the Kubernetes namespace NAME (`restore.namespace()`),
2700        // not the Namespace object UID: it selects the `tenants/<x>` keyspace segment and matches the
2701        // `namespaceUid` wire field name, which is frozen for compatibility (issue #1084).
2702        namespace_uid: restore.namespace().unwrap_or_default(),
2703        kanidm_uid: restore.spec.target_ref.uid.clone(),
2704        kanidm_name: restore.spec.target_ref.name.clone(),
2705        domain: target.spec.domain.clone(),
2706        kanidm_version: target
2707            .status
2708            .as_ref()
2709            .and_then(|s| s.version.as_ref())
2710            .map(|v| v.image_tag.clone())
2711            .unwrap_or_default(),
2712        image_digest: None,
2713        consistency: "kanidm-offline".to_string(),
2714        reason: "restore-safety".to_string(),
2715        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2716        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2717        result_path: "/run/kaniop-result/result.json".to_string(),
2718        max_concurrent_parts: 4,
2719        max_retries: 3,
2720    };
2721    let _ = manifest_key;
2722    let doc = OperationDocument {
2723        api_version: OPERATION_DOC_VERSION.to_string(),
2724        kind: "OperationDocument".to_string(),
2725        spec: OperationSpec::Upload(op),
2726    };
2727    serde_json::to_string(&doc)
2728        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2729}
2730
2731async fn validate_backup_ref(
2732    restore: &KanidmRestore,
2733    target: &Kanidm,
2734    ctx: &RestoreContext,
2735) -> Result<Option<DisasterRecoverySource>> {
2736    let backup_ref = restore
2737        .spec
2738        .source
2739        .backup_ref
2740        .as_ref()
2741        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2742    let ns = restore.namespace().unwrap();
2743    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2744        .get(&backup_ref.name)
2745        .await
2746        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2747    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2748        return Err(Error::MissingData(format!(
2749            "KanidmBackup '{}' is not Ready (phase: {:?})",
2750            backup_ref.name,
2751            backup.status.as_ref().map(|s| &s.phase)
2752        )));
2753    }
2754    let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2755        || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2756    let disaster_recovery_source = if source_identity_mismatch {
2757        if !disaster_recovery_requested(restore) {
2758            return Err(Error::MissingData(format!(
2759                "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2760                backup.spec.kanidm_ref.name,
2761                backup.spec.kanidm_ref.uid,
2762                restore.spec.target_ref.name,
2763                restore.spec.target_ref.uid,
2764                DISASTER_RECOVERY_ANNOTATION,
2765                BREAK_GLASS_REASON_ANNOTATION,
2766                BREAK_GLASS_APPROVED_BY_ANNOTATION,
2767            )));
2768        }
2769        validate_disaster_recovery_approval(restore)?;
2770        Some(DisasterRecoverySource {
2771            name: backup.spec.kanidm_ref.name.clone(),
2772            uid: backup.spec.kanidm_ref.uid.clone(),
2773        })
2774    } else {
2775        None
2776    };
2777    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2778        .get(&backup.spec.repository_ref.name)
2779        .await
2780        .map_err(|e| {
2781            Error::kube_error(
2782                "get",
2783                "KanidmBackupRepository",
2784                &ns,
2785                &backup.spec.repository_ref.name,
2786                e,
2787            )
2788        })?;
2789    if !has_accepted_condition(
2790        &repo
2791            .status
2792            .as_ref()
2793            .map(|s| &s.conditions)
2794            .cloned()
2795            .unwrap_or_default(),
2796    ) {
2797        return Err(Error::MissingData(format!(
2798            "KanidmBackupRepository '{}' configuration has not been accepted",
2799            backup.spec.repository_ref.name
2800        )));
2801    }
2802    validate_backup_compatibility(restore, target, &backup)?;
2803    Ok(disaster_recovery_source)
2804}
2805
2806fn has_accepted_condition(conditions: &[Condition]) -> bool {
2807    conditions
2808        .iter()
2809        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2810}
2811
2812fn validate_backup_compatibility(
2813    restore: &KanidmRestore,
2814    target: &Kanidm,
2815    backup: &KanidmBackup,
2816) -> Result<()> {
2817    let status = backup.status.as_ref();
2818    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2819        let target_version = target
2820            .status
2821            .as_ref()
2822            .and_then(|s| s.version.as_ref())
2823            .map(|v| v.image_tag.as_str());
2824        if let Some(tv) = target_version {
2825            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2826                return Err(Error::MissingData(format!(
2827                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2828                )));
2829            }
2830        }
2831    }
2832    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2833        if !backup_digest.is_empty()
2834            && restore.spec.restore_image.contains('@')
2835            && !restore.spec.restore_image.contains(backup_digest)
2836        {
2837            return Err(Error::MissingData(format!(
2838                "restore image digest does not match backup image digest '{backup_digest}'"
2839            )));
2840        }
2841    }
2842    Ok(())
2843}
2844
2845#[allow(clippy::too_many_arguments)]
2846fn build_download_operation_doc(
2847    target: &Kanidm,
2848    manifest_key: &str,
2849    expected_backup_id: &str,
2850    expected_source_uid: &str,
2851    bucket: &str,
2852    prefix: &str,
2853    endpoint: &str,
2854    region: &str,
2855    force_path_style: bool,
2856    insecure: bool,
2857    ca_bundle_path: Option<&str>,
2858    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2859) -> String {
2860    let enc_mode =
2861        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2862    let enc_key_id = encryption
2863        .and_then(|e| e.key_id.as_ref())
2864        .map(|s| serde_json::Value::String(s.clone()));
2865    serde_json::json!({
2866        "apiVersion": "backup.kaniop.rs/v1alpha1",
2867        "kind": "OperationDocument",
2868        "operation": "download",
2869        "manifestKey": manifest_key,
2870        "bucket": bucket,
2871        "prefix": prefix,
2872        "endpoint": endpoint,
2873        "region": region,
2874        "forcePathStyle": force_path_style,
2875        "insecure": insecure,
2876        "caBundlePath": ca_bundle_path,
2877        "expectedBackupId": expected_backup_id,
2878        "expectedKanidmUid": expected_source_uid,
2879        "expectedDomain": target.spec.domain,
2880        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2881        "resultPath": "/run/kaniop-result/result.json",
2882        "maxRetries": 3,
2883        "encryptionMode": enc_mode,
2884        "encryptionKeyId": enc_key_id,
2885    })
2886    .to_string()
2887}
2888
2889async fn ensure_operation_configmap(
2890    restore: &KanidmRestore,
2891    name: &str,
2892    data: &str,
2893    ns: &str,
2894    ctx: &RestoreContext,
2895) -> Result<()> {
2896    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2897    if api
2898        .get_opt(name)
2899        .await
2900        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2901        .is_some()
2902    {
2903        return Ok(());
2904    }
2905    let cm = ConfigMap {
2906        metadata: ObjectMeta {
2907            name: Some(name.to_string()),
2908            namespace: Some(ns.to_string()),
2909            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2910            ..Default::default()
2911        },
2912        data: Some(std::collections::BTreeMap::from([(
2913            "operation.json".to_string(),
2914            data.to_string(),
2915        )])),
2916        ..Default::default()
2917    };
2918    api.create(&PostParams::default(), &cm)
2919        .await
2920        .map(|_| ())
2921        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926    use super::{
2927        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2928        CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2929        DISASTER_RECOVERY_ANNOTATION,
2930        KanidmRestore, KanidmRestorePhase,
2931        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2932        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2933        disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2934        is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2935        restore_job_name, safe_basename, validate_disaster_recovery_approval,
2936        validate_safety_backup_config, validate_source, verify_job_name,
2937    };
2938    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2939    use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2940    use k8s_openapi::jiff::Timestamp;
2941    use kube::api::ObjectMeta as ApiObjectMeta;
2942    use std::collections::BTreeMap;
2943
2944    fn make_restore(
2945        source: KanidmRestoreSource,
2946        safety: Option<SafetyBackupConfig>,
2947    ) -> KanidmRestore {
2948        KanidmRestore {
2949            metadata: ApiObjectMeta {
2950                name: Some("test-restore".to_string()),
2951                namespace: Some("default".to_string()),
2952                uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2953                ..Default::default()
2954            },
2955            spec: KanidmRestoreSpec {
2956                target_ref: KanidmRestoreTargetRef {
2957                    name: "corp-idm".to_string(),
2958                    uid: "test-uid-123".to_string(),
2959                },
2960                source,
2961                restore_image: "kanidm/server@sha256:abc".to_string(),
2962                safety_backup: safety,
2963            },
2964            status: None,
2965        }
2966    }
2967
2968    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2969        super::super::crd::Kanidm {
2970            status: Some(super::super::crd::KanidmStatus {
2971                available_replicas: 1,
2972                replicas: 1,
2973                unavailable_replicas: 0,
2974                updated_replicas: 1,
2975                replica_statuses: vec![],
2976                replica_column: "1/1".to_string(),
2977                secret_name: None,
2978                version: Some(super::super::crd::KanidmVersionStatus {
2979                    image_tag: image_tag.to_string(),
2980                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2981                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2982                }),
2983                domain_appearance_image: None,
2984                mail_sender: None,
2985                conditions: None,
2986            }),
2987            ..Default::default()
2988        }
2989    }
2990
2991    #[test]
2992    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2993        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2994    }
2995
2996    #[test]
2997    fn completed_restore_sets_ready_condition() {
2998        let mut status = KanidmRestoreStatus {
2999            phase: KanidmRestorePhase::Completed,
3000            ..Default::default()
3001        };
3002        super::update_restore_conditions(&mut status, Some(7));
3003        assert!(status.conditions.iter().any(|condition| {
3004            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3005        }));
3006        assert!(!status.conditions.iter().any(|condition| {
3007            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3008        }));
3009    }
3010
3011    #[test]
3012    fn disaster_recovery_condition_survives_phase_updates() {
3013        let dr_condition = Condition {
3014            type_: CONDITION_DISASTER_RECOVERY.to_string(),
3015            status: CONDITION_TRUE.to_string(),
3016            reason: "ApprovedSourceIdentityMismatch".to_string(),
3017            message: "approved DR".to_string(),
3018            last_transition_time: Time(Timestamp::now()),
3019            observed_generation: Some(7),
3020        };
3021        let mut status = KanidmRestoreStatus {
3022            phase: KanidmRestorePhase::Quiescing,
3023            conditions: vec![dr_condition.clone()],
3024            ..Default::default()
3025        };
3026
3027        super::update_restore_conditions(&mut status, Some(8));
3028
3029        assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3030    }
3031
3032    #[test]
3033    fn failed_restore_sets_failed_condition() {
3034        let mut status = KanidmRestoreStatus {
3035            phase: KanidmRestorePhase::Failed,
3036            message: Some("verification failed".to_string()),
3037            ..Default::default()
3038        };
3039        super::update_restore_conditions(&mut status, Some(8));
3040        assert!(status.conditions.iter().any(|condition| {
3041            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3042        }));
3043    }
3044
3045    #[test]
3046    fn restore_filename_is_confined_to_backup_directory() {
3047        assert!(safe_basename("backup.json.gz"));
3048        assert!(!safe_basename("../backup.json.gz"));
3049        assert!(!safe_basename("a/b"));
3050        assert!(!safe_basename(""));
3051    }
3052
3053    #[test]
3054    fn restore_image_rejects_latest_and_unpinned_names() {
3055        assert!(mutable_image("kanidm/server:latest"));
3056        assert!(mutable_image("kanidm/server"));
3057        assert!(!mutable_image("kanidm/server:1.10.0"));
3058        assert!(!mutable_image("kanidm/server:1.11.2@sha256:serverdigest"));
3059        assert!(!mutable_image("kanidm/server@sha256:abc"));
3060        assert!(mutable_image("registry.example.com:5000/kanidm/server"));
3061    }
3062
3063    #[test]
3064    fn source_validation_rejects_both_local_and_backup_ref() {
3065        let source = KanidmRestoreSource {
3066            local: Some(super::KanidmRestoreLocalSource {
3067                file_name: "backup.json".to_string(),
3068            }),
3069            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3070                name: "some-backup".to_string(),
3071            }),
3072        };
3073        let restore = make_restore(source, None);
3074        assert!(validate_source(&restore).is_err());
3075    }
3076
3077    #[test]
3078    fn source_validation_rejects_empty_source() {
3079        let source = KanidmRestoreSource {
3080            local: None,
3081            backup_ref: None,
3082        };
3083        let restore = make_restore(source, None);
3084        assert!(validate_source(&restore).is_err());
3085    }
3086
3087    #[test]
3088    fn source_validation_accepts_local_source() {
3089        let source = KanidmRestoreSource {
3090            local: Some(super::KanidmRestoreLocalSource {
3091                file_name: "backup.json".to_string(),
3092            }),
3093            backup_ref: None,
3094        };
3095        let restore = make_restore(source, None);
3096        assert!(validate_source(&restore).is_ok());
3097    }
3098
3099    #[test]
3100    fn source_validation_accepts_backup_ref_source() {
3101        let source = KanidmRestoreSource {
3102            local: None,
3103            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3104                name: "corp-idm-019c7c76".to_string(),
3105            }),
3106        };
3107        let restore = make_restore(source, None);
3108        assert!(validate_source(&restore).is_ok());
3109    }
3110
3111    #[test]
3112    fn source_validation_rejects_empty_backup_ref_name() {
3113        let source = KanidmRestoreSource {
3114            local: None,
3115            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3116                name: "".to_string(),
3117            }),
3118        };
3119        let restore = make_restore(source, None);
3120        assert!(validate_source(&restore).is_err());
3121    }
3122
3123    #[test]
3124    fn is_remote_source_true_for_backup_ref() {
3125        let source = KanidmRestoreSource {
3126            local: None,
3127            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3128                name: "backup-1".to_string(),
3129            }),
3130        };
3131        let restore = make_restore(source, None);
3132        assert!(is_remote_source(&restore));
3133    }
3134
3135    #[test]
3136    fn is_remote_source_false_for_local() {
3137        let source = KanidmRestoreSource {
3138            local: Some(super::KanidmRestoreLocalSource {
3139                file_name: "backup.json".to_string(),
3140            }),
3141            backup_ref: None,
3142        };
3143        let restore = make_restore(source, None);
3144        assert!(!is_remote_source(&restore));
3145    }
3146
3147    #[test]
3148    fn requires_safety_backup_default_true() {
3149        let source = KanidmRestoreSource {
3150            local: None,
3151            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3152                name: "backup-1".to_string(),
3153            }),
3154        };
3155        let restore = make_restore(source, None);
3156        assert!(requires_safety_backup(&restore));
3157    }
3158
3159    #[test]
3160    fn requires_safety_backup_false_when_skip() {
3161        let source = KanidmRestoreSource {
3162            local: None,
3163            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3164                name: "backup-1".to_string(),
3165            }),
3166        };
3167        let safety = SafetyBackupConfig {
3168            repository_ref: Some(SafetyBackupRepositoryRef {
3169                name: "offsite".to_string(),
3170            }),
3171            skip: true,
3172        };
3173        let restore = make_restore(source, Some(safety));
3174        assert!(!requires_safety_backup(&restore));
3175    }
3176
3177    #[test]
3178    fn break_glass_validation_rejects_skip_without_reason() {
3179        let source = KanidmRestoreSource {
3180            local: None,
3181            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3182                name: "backup-1".to_string(),
3183            }),
3184        };
3185        let safety = SafetyBackupConfig {
3186            repository_ref: Some(SafetyBackupRepositoryRef {
3187                name: "offsite".to_string(),
3188            }),
3189            skip: true,
3190        };
3191        let mut restore = make_restore(source, Some(safety));
3192        restore.metadata.annotations = Some(BTreeMap::from([(
3193            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3194            "commander@example.com".to_string(),
3195        )]));
3196        assert!(validate_safety_backup_config(&restore).is_err());
3197    }
3198
3199    #[test]
3200    fn break_glass_validation_rejects_skip_without_approver() {
3201        let source = KanidmRestoreSource {
3202            local: None,
3203            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3204                name: "backup-1".to_string(),
3205            }),
3206        };
3207        let safety = SafetyBackupConfig {
3208            repository_ref: Some(SafetyBackupRepositoryRef {
3209                name: "offsite".to_string(),
3210            }),
3211            skip: true,
3212        };
3213        let mut restore = make_restore(source, Some(safety));
3214        restore.metadata.annotations = Some(BTreeMap::from([(
3215            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3216            "PVC is unreadable".to_string(),
3217        )]));
3218        assert!(validate_safety_backup_config(&restore).is_err());
3219    }
3220
3221    #[test]
3222    fn break_glass_validation_accepts_valid_annotations() {
3223        let source = KanidmRestoreSource {
3224            local: None,
3225            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3226                name: "backup-1".to_string(),
3227            }),
3228        };
3229        let safety = SafetyBackupConfig {
3230            repository_ref: Some(SafetyBackupRepositoryRef {
3231                name: "offsite".to_string(),
3232            }),
3233            skip: true,
3234        };
3235        let mut restore = make_restore(source, Some(safety));
3236        restore.metadata.annotations = Some(BTreeMap::from([
3237            (
3238                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3239                "PVC is unreadable".to_string(),
3240            ),
3241            (
3242                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3243                "commander@example.com".to_string(),
3244            ),
3245        ]));
3246        assert!(validate_safety_backup_config(&restore).is_ok());
3247    }
3248
3249    #[test]
3250    fn disaster_recovery_requires_explicit_approval_annotations() {
3251        let source = KanidmRestoreSource {
3252            local: None,
3253            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3254                name: "backup-1".to_string(),
3255            }),
3256        };
3257        let mut restore = make_restore(source, None);
3258        restore.metadata.annotations = Some(BTreeMap::from([(
3259            DISASTER_RECOVERY_ANNOTATION.to_string(),
3260            "true".to_string(),
3261        )]));
3262        assert!(disaster_recovery_requested(&restore));
3263        assert!(validate_disaster_recovery_approval(&restore).is_err());
3264    }
3265
3266    #[test]
3267    fn disaster_recovery_accepts_reason_and_approver() {
3268        let source = KanidmRestoreSource {
3269            local: None,
3270            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3271                name: "backup-1".to_string(),
3272            }),
3273        };
3274        let mut restore = make_restore(source, None);
3275        restore.metadata.annotations = Some(BTreeMap::from([
3276            (
3277                DISASTER_RECOVERY_ANNOTATION.to_string(),
3278                "true".to_string(),
3279            ),
3280            (
3281                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3282                "original cluster lost".to_string(),
3283            ),
3284            (
3285                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3286                "incident-commander".to_string(),
3287            ),
3288        ]));
3289        assert!(disaster_recovery_requested(&restore));
3290        assert!(validate_disaster_recovery_approval(&restore).is_ok());
3291    }
3292
3293    #[test]
3294    fn remote_source_requires_repository_ref_when_not_skipping() {
3295        let source = KanidmRestoreSource {
3296            local: None,
3297            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3298                name: "backup-1".to_string(),
3299            }),
3300        };
3301        let safety = SafetyBackupConfig {
3302            repository_ref: None,
3303            skip: false,
3304        };
3305        let restore = make_restore(source, Some(safety));
3306        assert!(validate_safety_backup_config(&restore).is_err());
3307    }
3308
3309    #[test]
3310    fn remote_source_accepts_repository_ref() {
3311        let source = KanidmRestoreSource {
3312            local: None,
3313            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3314                name: "backup-1".to_string(),
3315            }),
3316        };
3317        let safety = SafetyBackupConfig {
3318            repository_ref: Some(SafetyBackupRepositoryRef {
3319                name: "offsite".to_string(),
3320            }),
3321            skip: false,
3322        };
3323        let restore = make_restore(source, Some(safety));
3324        assert!(validate_safety_backup_config(&restore).is_ok());
3325    }
3326
3327    #[test]
3328    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3329        let phases = vec![
3330            KanidmRestorePhase::Pending,
3331            KanidmRestorePhase::Validating,
3332            KanidmRestorePhase::Quiescing,
3333            KanidmRestorePhase::SafetyBackup,
3334            KanidmRestorePhase::PreparingSource,
3335            KanidmRestorePhase::RestoringPrimary,
3336            KanidmRestorePhase::Verifying,
3337            KanidmRestorePhase::RebuildingReplicas,
3338            KanidmRestorePhase::Resuming,
3339            KanidmRestorePhase::Completed,
3340        ];
3341        assert_eq!(phases.len(), 10);
3342        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3343        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3344    }
3345
3346    #[test]
3347    fn status_default_has_empty_phase_timestamps() {
3348        let status = KanidmRestoreStatus::default();
3349        assert!(status.phase_timestamps.is_empty());
3350    }
3351
3352    #[test]
3353    fn status_default_has_empty_original_replicas() {
3354        let status = KanidmRestoreStatus::default();
3355        assert!(status.original_replicas.is_empty());
3356    }
3357
3358    #[test]
3359    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3360        let mut target = super::super::crd::Kanidm::default();
3361        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3362            run_as_non_root: Some(false),
3363            run_as_user: Some(389),
3364            run_as_group: Some(389),
3365            fs_group: Some(389),
3366            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3367            supplemental_groups: Some(vec![390, 391]),
3368            ..Default::default()
3369        });
3370
3371        let ctx = super::source_job_pod_security_context(&target);
3372
3373        assert_eq!(ctx.run_as_non_root, Some(false));
3374        assert_eq!(ctx.run_as_user, Some(389));
3375        assert_eq!(ctx.run_as_group, Some(389));
3376        assert_eq!(ctx.fs_group, Some(389));
3377        assert_eq!(
3378            ctx.fs_group_change_policy.as_deref(),
3379            Some("OnRootMismatch")
3380        );
3381        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3382        assert_eq!(
3383            ctx.seccomp_profile
3384                .as_ref()
3385                .map(|profile| profile.type_.as_str()),
3386            Some("RuntimeDefault")
3387        );
3388    }
3389
3390    #[test]
3391    fn hardened_security_context_drops_all_capabilities() {
3392        let ctx = hardened_security_context();
3393        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3394        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3395        assert_eq!(ctx.run_as_non_root, Some(true));
3396        let caps = ctx.capabilities.unwrap();
3397        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3398        assert_eq!(
3399            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3400            Some("RuntimeDefault")
3401        );
3402    }
3403
3404    #[test]
3405    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3406        let ctx = kanidm_job_security_context();
3407        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3408        assert!(ctx.run_as_non_root.is_none());
3409        assert!(ctx.read_only_root_filesystem.is_none());
3410        let caps = ctx.capabilities.unwrap();
3411        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3412        assert_eq!(
3413            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3414            Some("RuntimeDefault")
3415        );
3416    }
3417
3418    #[test]
3419    fn safety_backup_job_name_is_deterministic() {
3420        let source = KanidmRestoreSource {
3421            local: Some(super::KanidmRestoreLocalSource {
3422                file_name: "backup.json".to_string(),
3423            }),
3424            backup_ref: None,
3425        };
3426        let restore = make_restore(source, None);
3427        let name1 = super::safety_backup_job_name(&restore);
3428        let name2 = super::safety_backup_job_name(&restore);
3429        assert_eq!(name1, name2);
3430        assert_eq!(name1, "test-restore-safety-backup");
3431    }
3432
3433    #[test]
3434    fn source_prep_job_name_is_deterministic() {
3435        let source = KanidmRestoreSource {
3436            local: None,
3437            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3438                name: "backup-1".to_string(),
3439            }),
3440        };
3441        let restore = make_restore(source, None);
3442        let name = super::source_prep_job_name(&restore);
3443        assert_eq!(name, "test-restore-source-prep");
3444    }
3445
3446    #[test]
3447    fn break_glass_condition_is_recorded_in_status() {
3448        let mut status = KanidmRestoreStatus::default();
3449        let condition = super::restore_condition(
3450            &status.conditions,
3451            CONDITION_BREAK_GLASS,
3452            CONDITION_TRUE,
3453            "BreakGlassOverride",
3454            "Safety backup skipped",
3455            Some(1),
3456        );
3457        status.conditions.push(condition);
3458        assert!(status.conditions.iter().any(|c| {
3459            c.type_ == CONDITION_BREAK_GLASS
3460                && c.status == CONDITION_TRUE
3461                && c.reason == "BreakGlassOverride"
3462        }));
3463    }
3464
3465    #[test]
3466    fn replica_count_entry_serialization() {
3467        let entry = ReplicaCountEntry {
3468            group: "primary".to_string(),
3469            replicas: 3,
3470        };
3471        let json = serde_json::to_string(&entry).unwrap();
3472        assert!(json.contains("\"group\":\"primary\""));
3473        assert!(json.contains("\"replicas\":3"));
3474    }
3475
3476    #[test]
3477    fn local_source_requires_repository_ref_when_not_skipping() {
3478        let source = KanidmRestoreSource {
3479            local: Some(super::KanidmRestoreLocalSource {
3480                file_name: "backup.json".to_string(),
3481            }),
3482            backup_ref: None,
3483        };
3484        let safety = SafetyBackupConfig {
3485            repository_ref: None,
3486            skip: false,
3487        };
3488        let restore = make_restore(source, Some(safety));
3489        assert!(validate_safety_backup_config(&restore).is_err());
3490    }
3491
3492    #[test]
3493    fn local_source_accepts_repository_ref_when_not_skipping() {
3494        let source = KanidmRestoreSource {
3495            local: Some(super::KanidmRestoreLocalSource {
3496                file_name: "backup.json".to_string(),
3497            }),
3498            backup_ref: None,
3499        };
3500        let safety = SafetyBackupConfig {
3501            repository_ref: Some(SafetyBackupRepositoryRef {
3502                name: "offsite".to_string(),
3503            }),
3504            skip: false,
3505        };
3506        let restore = make_restore(source, Some(safety));
3507        assert!(validate_safety_backup_config(&restore).is_ok());
3508    }
3509
3510    #[test]
3511    fn skip_without_break_glass_annotations_is_rejected() {
3512        let source = KanidmRestoreSource {
3513            local: Some(super::KanidmRestoreLocalSource {
3514                file_name: "backup.json".to_string(),
3515            }),
3516            backup_ref: None,
3517        };
3518        let safety = SafetyBackupConfig {
3519            repository_ref: None,
3520            skip: true,
3521        };
3522        let restore = make_restore(source, Some(safety));
3523        assert!(validate_safety_backup_config(&restore).is_err());
3524    }
3525
3526    #[test]
3527    fn has_accepted_condition_true() {
3528        let conditions = vec![Condition {
3529            type_: "Ready".to_string(),
3530            status: "True".to_string(),
3531            reason: "Accepted".to_string(),
3532            message: String::new(),
3533            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3534                k8s_openapi::jiff::Timestamp::now(),
3535            ),
3536            observed_generation: None,
3537        }];
3538        assert!(has_accepted_condition(&conditions));
3539    }
3540
3541    #[test]
3542    fn has_accepted_condition_false_when_not_accepted_reason() {
3543        let conditions = vec![Condition {
3544            type_: "Ready".to_string(),
3545            status: "True".to_string(),
3546            reason: "SomeOtherReason".to_string(),
3547            message: String::new(),
3548            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3549                k8s_openapi::jiff::Timestamp::now(),
3550            ),
3551            observed_generation: None,
3552        }];
3553        assert!(!has_accepted_condition(&conditions));
3554    }
3555
3556    #[test]
3557    fn has_accepted_condition_false_when_status_false() {
3558        let conditions = vec![Condition {
3559            type_: "Ready".to_string(),
3560            status: "False".to_string(),
3561            reason: "Accepted".to_string(),
3562            message: String::new(),
3563            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3564                k8s_openapi::jiff::Timestamp::now(),
3565            ),
3566            observed_generation: None,
3567        }];
3568        assert!(!has_accepted_condition(&conditions));
3569    }
3570
3571    #[test]
3572    fn has_accepted_condition_empty() {
3573        assert!(!has_accepted_condition(&[]));
3574    }
3575
3576    #[test]
3577    fn safety_backup_upload_operation_doc_is_valid() {
3578        let source = KanidmRestoreSource {
3579            local: None,
3580            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3581                name: "backup-1".to_string(),
3582            }),
3583        };
3584        let mut restore = make_restore(source, None);
3585        restore.metadata.namespace = Some("test-ns".to_string());
3586        let target = super::super::crd::Kanidm::default();
3587        let doc_str = super::build_safety_upload_operation_doc(
3588            &restore,
3589            &target,
3590            "test-backup-id",
3591            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3592            "my-bucket",
3593            "prod",
3594            "https://s3.example.com",
3595            "us-east-1",
3596            false,
3597            false,
3598            None,
3599            None,
3600        )
3601        .unwrap();
3602        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3603        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3604        assert_eq!(parsed["kind"], "OperationDocument");
3605        assert_eq!(parsed["operation"], "upload");
3606        assert_eq!(parsed["bucket"], "my-bucket");
3607        assert_eq!(parsed["prefix"], "prod");
3608        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3609        assert_eq!(parsed["region"], "us-east-1");
3610        assert_eq!(parsed["backupId"], "test-backup-id");
3611        assert_eq!(parsed["consistency"], "kanidm-offline");
3612        assert_eq!(parsed["reason"], "restore-safety");
3613        assert_eq!(
3614            parsed["payloadPath"],
3615            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3616        );
3617        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3618    }
3619
3620    #[test]
3621    fn download_operation_doc_uses_real_repository_data() {
3622        let source = KanidmRestoreSource {
3623            local: None,
3624            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3625                name: "backup-1".to_string(),
3626            }),
3627        };
3628        let _restore = make_restore(source, None);
3629        let target = super::super::crd::Kanidm::default();
3630        let doc_str = super::build_download_operation_doc(
3631            &target,
3632            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3633            "019c7c76-f423-7a12-8f41-2bea7588a303",
3634            "source-uid-456",
3635            "real-bucket",
3636            "real-prefix",
3637            "https://real-endpoint.com",
3638            "eu-west-1",
3639            true,
3640            false,
3641            None,
3642            None,
3643        );
3644        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3645        assert_eq!(parsed["bucket"], "real-bucket");
3646        assert_eq!(parsed["prefix"], "real-prefix");
3647        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3648        assert_eq!(parsed["region"], "eu-west-1");
3649        assert_eq!(parsed["forcePathStyle"], true);
3650        assert_eq!(
3651            parsed["manifestKey"],
3652            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3653        );
3654        assert_eq!(
3655            parsed["expectedBackupId"],
3656            "019c7c76-f423-7a12-8f41-2bea7588a303"
3657        );
3658        assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3659    }
3660
3661    #[test]
3662    fn safety_backup_ref_is_set_after_verification() {
3663        let status = KanidmRestoreStatus {
3664            phase: KanidmRestorePhase::SafetyBackup,
3665            safety_backup_job_name: Some("test-safety-backup".to_string()),
3666            safety_backup_ref: Some("safety-test-restore".to_string()),
3667            safety_backup_expected_backup_id: Some(
3668                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3669            ),
3670            safety_backup_manifest_key: Some(
3671                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3672            ),
3673            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3674            ..Default::default()
3675        };
3676        assert!(status.safety_backup_ref.is_some());
3677        assert!(
3678            status
3679                .safety_backup_ref
3680                .as_ref()
3681                .unwrap()
3682                .starts_with("safety-")
3683        );
3684        assert!(status.safety_backup_expected_backup_id.is_some());
3685        assert!(status.safety_backup_manifest_key.is_some());
3686        assert!(status.safety_backup_payload_sha256.is_some());
3687    }
3688
3689    #[test]
3690    fn database_mutation_not_started_without_safety_backup_ref() {
3691        let status = KanidmRestoreStatus {
3692            phase: KanidmRestorePhase::PreparingSource,
3693            safety_backup_ref: None,
3694            database_mutation_started: false,
3695            ..Default::default()
3696        };
3697        assert!(!status.database_mutation_started);
3698        assert!(status.safety_backup_ref.is_none());
3699    }
3700
3701    #[test]
3702    fn pre_boundary_failure_does_not_set_mutation_started() {
3703        let status = KanidmRestoreStatus {
3704            phase: KanidmRestorePhase::Failed,
3705            message: Some(
3706                "safety backup job failed; target restored to original state".to_string(),
3707            ),
3708            database_mutation_started: false,
3709            ..Default::default()
3710        };
3711        assert!(!status.database_mutation_started);
3712    }
3713
3714    #[test]
3715    fn shared_volume_constants_are_distinct() {
3716        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3717        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3718    }
3719
3720    #[test]
3721    fn validate_backup_compatibility_matching_versions_passes() {
3722        let source = KanidmRestoreSource {
3723            local: None,
3724            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3725                name: "backup-1".to_string(),
3726            }),
3727        };
3728        let restore = make_restore(source, None);
3729        let target = kanidm_with_version("1.10.4");
3730        let backup = kaniop_backup_core::crd::KanidmBackup {
3731            metadata: kube::api::ObjectMeta::default(),
3732            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3733                backup_id: "id".to_string(),
3734                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3735                    name: "k".to_string(),
3736                    uid: "uid".to_string(),
3737                },
3738                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3739                    name: "repo".to_string(),
3740                },
3741                manifest_key: "key".to_string(),
3742            },
3743            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3744                kanidm_version: Some("1.10.4".to_string()),
3745                ..Default::default()
3746            }),
3747        };
3748        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3749    }
3750
3751    #[test]
3752    fn validate_backup_compatibility_version_mismatch_fails() {
3753        let source = KanidmRestoreSource {
3754            local: None,
3755            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3756                name: "backup-1".to_string(),
3757            }),
3758        };
3759        let restore = make_restore(source, None);
3760        let target = kanidm_with_version("1.11.0");
3761        let backup = kaniop_backup_core::crd::KanidmBackup {
3762            metadata: kube::api::ObjectMeta::default(),
3763            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3764                backup_id: "id".to_string(),
3765                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3766                    name: "k".to_string(),
3767                    uid: "uid".to_string(),
3768                },
3769                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3770                    name: "repo".to_string(),
3771                },
3772                manifest_key: "key".to_string(),
3773            },
3774            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3775                kanidm_version: Some("1.10.4".to_string()),
3776                ..Default::default()
3777            }),
3778        };
3779        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3780    }
3781
3782    #[test]
3783    fn validate_backup_compatibility_no_backup_version_passes() {
3784        let source = KanidmRestoreSource {
3785            local: None,
3786            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3787                name: "backup-1".to_string(),
3788            }),
3789        };
3790        let restore = make_restore(source, None);
3791        let target = super::super::crd::Kanidm::default();
3792        let backup = kaniop_backup_core::crd::KanidmBackup {
3793            metadata: kube::api::ObjectMeta::default(),
3794            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3795                backup_id: "id".to_string(),
3796                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3797                    name: "k".to_string(),
3798                    uid: "uid".to_string(),
3799                },
3800                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3801                    name: "repo".to_string(),
3802                },
3803                manifest_key: "key".to_string(),
3804            },
3805            status: None,
3806        };
3807        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3808    }
3809
3810    #[test]
3811    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3812        let source = KanidmRestoreSource {
3813            local: None,
3814            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3815                name: "backup-1".to_string(),
3816            }),
3817        };
3818        let mut restore = make_restore(source, None);
3819        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3820        let target = super::super::crd::Kanidm::default();
3821        let backup = kaniop_backup_core::crd::KanidmBackup {
3822            metadata: kube::api::ObjectMeta::default(),
3823            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3824                backup_id: "id".to_string(),
3825                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3826                    name: "k".to_string(),
3827                    uid: "uid".to_string(),
3828                },
3829                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3830                    name: "repo".to_string(),
3831                },
3832                manifest_key: "key".to_string(),
3833            },
3834            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3835                image_digest: Some("sha256:originaldigest".to_string()),
3836                ..Default::default()
3837            }),
3838        };
3839        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3840    }
3841
3842    #[test]
3843    fn validate_backup_compatibility_image_digest_match_passes() {
3844        let source = KanidmRestoreSource {
3845            local: None,
3846            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3847                name: "backup-1".to_string(),
3848            }),
3849        };
3850        let mut restore = make_restore(source, None);
3851        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3852        let target = super::super::crd::Kanidm::default();
3853        let backup = kaniop_backup_core::crd::KanidmBackup {
3854            metadata: kube::api::ObjectMeta::default(),
3855            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3856                backup_id: "id".to_string(),
3857                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3858                    name: "k".to_string(),
3859                    uid: "uid".to_string(),
3860                },
3861                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3862                    name: "repo".to_string(),
3863                },
3864                manifest_key: "key".to_string(),
3865            },
3866            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3867                image_digest: Some("sha256:abc123".to_string()),
3868                ..Default::default()
3869            }),
3870        };
3871        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3872    }
3873
3874    #[test]
3875    fn safety_backup_id_is_deterministic() {
3876        let source = KanidmRestoreSource {
3877            local: None,
3878            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3879                name: "backup-1".to_string(),
3880            }),
3881        };
3882        let restore = make_restore(source, None);
3883        let id1 = super::compute_safety_backup_id(&restore).unwrap();
3884        let id2 = super::compute_safety_backup_id(&restore).unwrap();
3885        assert_eq!(id1, id2);
3886        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3887    }
3888
3889    #[test]
3890    fn safety_manifest_key_is_deterministic() {
3891        let source = KanidmRestoreSource {
3892            local: None,
3893            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3894                name: "backup-1".to_string(),
3895            }),
3896        };
3897        let restore = make_restore(source, None);
3898        let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3899        let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3900        assert_eq!(key1, key2);
3901        assert!(key1.contains("manifest.json"));
3902        assert!(key1.starts_with("v1/tenants/"));
3903        // The tenant segment is the Kubernetes namespace NAME from `make_restore`, never the
3904        // Namespace object UID or the restore UID.
3905        assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3906        assert!(
3907            !key1.contains("a1b2c3d4"),
3908            "safety manifest key must not key the tenant segment by object UID: {key1}"
3909        );
3910    }
3911
3912    /// `compute_safety_manifest_key` hand-rolls the keyspace with `format!` instead of going through
3913    /// `kaniop_backup_core::paths::RepositoryPath`, so it is pinned here byte-for-byte against the
3914    /// deployed layout: `v1/tenants/<namespace-name>/clusters/<kanidm-uid>/backups/<backup-id>/`
3915    /// `manifest.json`. The `<backup-id>` is the UUIDv5 chain `DNS -> namespace name -> restore UID`
3916    /// and is load-bearing for transport re-upload idempotency, so both the segment and the derived
3917    /// ID are asserted as literals. The tenant segment carries the namespace NAME even though the
3918    /// `namespaceUid` wire field it mirrors is named like a UID (misnomer tracked in issue #1084).
3919    #[test]
3920    fn safety_manifest_key_matches_deployed_keyspace_layout() {
3921        let source = KanidmRestoreSource {
3922            local: None,
3923            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3924                name: "backup-1".to_string(),
3925            }),
3926        };
3927        let restore = make_restore(source, None);
3928        let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3929        assert_eq!(
3930            super::compute_safety_backup_id(&restore).unwrap(),
3931            expected_backup_id
3932        );
3933        assert_eq!(
3934            super::compute_safety_manifest_key(&restore).unwrap(),
3935            format!(
3936                "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
3937            )
3938        );
3939    }
3940
3941    #[test]
3942    fn safety_backup_id_survives_restart() {
3943        let source = KanidmRestoreSource {
3944            local: None,
3945            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3946                name: "backup-1".to_string(),
3947            }),
3948        };
3949        let mut restore = make_restore(source, None);
3950        let original_id = super::compute_safety_backup_id(&restore).unwrap();
3951        // Pinning the derived ID pins its UUIDv5 inputs: the namespace NAME and the restore UID.
3952        // Feeding the Namespace object UID here would change every safety backup ID and orphan an
3953        // in-flight safety backup (issue #1084).
3954        assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
3955        restore.status = Some(KanidmRestoreStatus {
3956            phase: KanidmRestorePhase::SafetyBackup,
3957            safety_backup_expected_backup_id: Some(original_id.clone()),
3958            ..Default::default()
3959        });
3960        let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3961        assert_eq!(original_id, recomputed_id);
3962    }
3963
3964    #[test]
3965    fn safety_backup_id_derived_from_uid_not_name() {
3966        let source = KanidmRestoreSource {
3967            local: None,
3968            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3969                name: "backup-1".to_string(),
3970            }),
3971        };
3972        let restore1 = make_restore(source.clone(), None);
3973        let mut restore2 = make_restore(source, None);
3974        restore2.metadata.name = Some("different-name-same-uid".to_string());
3975        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3976        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3977        assert_eq!(id1, id2);
3978    }
3979
3980    #[test]
3981    fn safety_backup_id_differs_for_different_uids() {
3982        let source = KanidmRestoreSource {
3983            local: None,
3984            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3985                name: "backup-1".to_string(),
3986            }),
3987        };
3988        let restore1 = make_restore(source.clone(), None);
3989        let mut restore2 = make_restore(source, None);
3990        restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3991        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3992        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3993        assert_ne!(id1, id2);
3994    }
3995
3996    #[test]
3997    fn safety_backup_id_errors_without_uid() {
3998        let source = KanidmRestoreSource {
3999            local: None,
4000            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4001                name: "backup-1".to_string(),
4002            }),
4003        };
4004        let mut restore = make_restore(source, None);
4005        restore.metadata.uid = None;
4006        assert!(super::compute_safety_backup_id(&restore).is_err());
4007    }
4008
4009    #[test]
4010    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4011        let source = KanidmRestoreSource {
4012            local: None,
4013            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4014                name: "backup-1".to_string(),
4015            }),
4016        };
4017        let restore = make_restore(source, None);
4018        assert!(super::requires_safety_backup(&restore));
4019        let status_without_ref = KanidmRestoreStatus {
4020            phase: KanidmRestorePhase::RestoringPrimary,
4021            safety_backup_ref: None,
4022            database_mutation_started: false,
4023            ..Default::default()
4024        };
4025        assert!(status_without_ref.safety_backup_ref.is_none());
4026        assert!(!status_without_ref.database_mutation_started);
4027    }
4028
4029    #[test]
4030    fn mutation_boundary_allows_progression_when_safety_ref_set() {
4031        let status_with_ref = KanidmRestoreStatus {
4032            phase: KanidmRestorePhase::RestoringPrimary,
4033            safety_backup_ref: Some("safety-test-restore".to_string()),
4034            safety_backup_manifest_key: Some(
4035                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4036            ),
4037            safety_backup_payload_sha256: Some("abc123".to_string()),
4038            database_mutation_started: false,
4039            ..Default::default()
4040        };
4041        assert!(status_with_ref.safety_backup_ref.is_some());
4042        assert!(!status_with_ref.database_mutation_started);
4043    }
4044
4045    #[test]
4046    fn result_document_verification_rejects_missing_backup_id() {
4047        let result_json = r#"{
4048            "apiVersion": "backup.kaniop.rs/v1alpha1",
4049            "kind": "ResultDocument",
4050            "operation": "upload",
4051            "success": true,
4052            "exitCode": "success",
4053            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4054            "payloadSha256": "abc123"
4055        }"#;
4056        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4057        assert!(result.backup_id.is_none());
4058    }
4059
4060    #[test]
4061    fn result_document_verification_rejects_missing_manifest_key() {
4062        let result_json = r#"{
4063            "apiVersion": "backup.kaniop.rs/v1alpha1",
4064            "kind": "ResultDocument",
4065            "operation": "upload",
4066            "success": true,
4067            "exitCode": "success",
4068            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4069            "payloadSha256": "abc123"
4070        }"#;
4071        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4072        assert!(result.manifest_key.is_none());
4073    }
4074
4075    #[test]
4076    fn result_document_verification_rejects_missing_payload_sha256() {
4077        let result_json = r#"{
4078            "apiVersion": "backup.kaniop.rs/v1alpha1",
4079            "kind": "ResultDocument",
4080            "operation": "upload",
4081            "success": true,
4082            "exitCode": "success",
4083            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4084            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4085        }"#;
4086        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4087        assert!(result.payload_sha256.is_none());
4088    }
4089
4090    #[test]
4091    fn result_document_verification_rejects_failure_exit_code() {
4092        let result_json = r#"{
4093            "apiVersion": "backup.kaniop.rs/v1alpha1",
4094            "kind": "ResultDocument",
4095            "operation": "upload",
4096            "success": false,
4097            "exitCode": "integrity",
4098            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4099            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4100            "payloadSha256": "abc123"
4101        }"#;
4102        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4103        assert!(!result.success);
4104        assert_eq!(
4105            result.exit_code,
4106            kaniop_backup_core::result::ExitCode::Integrity
4107        );
4108    }
4109
4110    #[test]
4111    fn result_document_verification_rejects_wrong_operation() {
4112        let result_json = r#"{
4113            "apiVersion": "backup.kaniop.rs/v1alpha1",
4114            "kind": "ResultDocument",
4115            "operation": "download",
4116            "success": true,
4117            "exitCode": "success",
4118            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4119            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4120            "payloadSha256": "abc123"
4121        }"#;
4122        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4123        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4124    }
4125
4126    #[test]
4127    fn result_document_verification_accepts_valid_document() {
4128        let result_json = r#"{
4129            "apiVersion": "backup.kaniop.rs/v1alpha1",
4130            "kind": "ResultDocument",
4131            "operation": "upload",
4132            "success": true,
4133            "exitCode": "success",
4134            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4135            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4136            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4137        }"#;
4138        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4139        assert!(result.success);
4140        assert_eq!(
4141            result.exit_code,
4142            kaniop_backup_core::result::ExitCode::Success
4143        );
4144        assert_eq!(result.operation, "upload");
4145        assert!(result.backup_id.is_some());
4146        assert!(result.manifest_key.is_some());
4147        assert!(result.payload_sha256.is_some());
4148    }
4149
4150    #[test]
4151    fn termination_message_path_constant_is_valid() {
4152        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4153        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4154    }
4155
4156    #[test]
4157    fn status_tracks_safety_backup_evidence() {
4158        let status = KanidmRestoreStatus {
4159            phase: KanidmRestorePhase::PreparingSource,
4160            safety_backup_ref: Some("safety-test-restore".to_string()),
4161            safety_backup_expected_backup_id: Some(
4162                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4163            ),
4164            safety_backup_manifest_key: Some(
4165                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4166            ),
4167            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4168            ..Default::default()
4169        };
4170        assert!(status.safety_backup_ref.is_some());
4171        assert!(status.safety_backup_expected_backup_id.is_some());
4172        assert!(status.safety_backup_manifest_key.is_some());
4173        assert!(status.safety_backup_payload_sha256.is_some());
4174    }
4175
4176    #[test]
4177    fn fail_closed_semantics_preserved_on_result_parse_failure() {
4178        let invalid_json = "not valid json";
4179        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4180        assert!(result.is_err());
4181    }
4182
4183    #[test]
4184    fn fail_closed_semantics_preserved_on_oversized_result() {
4185        let oversized_json = format!(
4186            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4187            kaniop_backup_core::result::RESULT_DOC_VERSION,
4188            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4189        );
4190        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4191        assert!(result.is_err());
4192    }
4193
4194    #[test]
4195    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4196        let source = KanidmRestoreSource {
4197            local: None,
4198            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4199                name: "backup-1".to_string(),
4200            }),
4201        };
4202        let mut restore = make_restore(source, None);
4203        restore.metadata.namespace = Some("test-ns".to_string());
4204        let target = super::super::crd::Kanidm::default();
4205        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4206        let doc_str = super::build_safety_upload_operation_doc(
4207            &restore,
4208            &target,
4209            "test-backup-id",
4210            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4211            "my-bucket",
4212            "prod",
4213            "https://s3.example.com",
4214            "us-east-1",
4215            false,
4216            false,
4217            Some(&ca_path),
4218            None,
4219        )
4220        .unwrap();
4221        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4222        assert_eq!(parsed["caBundlePath"], ca_path);
4223    }
4224
4225    #[test]
4226    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4227        let source = KanidmRestoreSource {
4228            local: None,
4229            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4230                name: "backup-1".to_string(),
4231            }),
4232        };
4233        let mut restore = make_restore(source, None);
4234        restore.metadata.namespace = Some("test-ns".to_string());
4235        let target = super::super::crd::Kanidm::default();
4236        let doc_str = super::build_safety_upload_operation_doc(
4237            &restore,
4238            &target,
4239            "test-backup-id",
4240            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4241            "my-bucket",
4242            "prod",
4243            "https://s3.example.com",
4244            "us-east-1",
4245            false,
4246            false,
4247            None,
4248            None,
4249        )
4250        .unwrap();
4251        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4252        assert!(parsed["caBundlePath"].is_null());
4253    }
4254
4255    #[test]
4256    fn download_operation_doc_includes_ca_bundle_path_when_set() {
4257        let source = KanidmRestoreSource {
4258            local: None,
4259            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4260                name: "backup-1".to_string(),
4261            }),
4262        };
4263        let _restore = make_restore(source, None);
4264        let target = super::super::crd::Kanidm::default();
4265        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4266        let doc_str = super::build_download_operation_doc(
4267            &target,
4268            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4269            "019c7c76-f423-7a12-8f41-2bea7588a303",
4270            "source-uid-456",
4271            "real-bucket",
4272            "real-prefix",
4273            "https://real-endpoint.com",
4274            "eu-west-1",
4275            true,
4276            false,
4277            Some(&ca_path),
4278            None,
4279        );
4280        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4281        assert_eq!(parsed["caBundlePath"], ca_path);
4282    }
4283
4284    #[test]
4285    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4286        let source = KanidmRestoreSource {
4287            local: None,
4288            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4289                name: "backup-1".to_string(),
4290            }),
4291        };
4292        let _restore = make_restore(source, None);
4293        let target = super::super::crd::Kanidm::default();
4294        let doc_str = super::build_download_operation_doc(
4295            &target,
4296            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4297            "019c7c76-f423-7a12-8f41-2bea7588a303",
4298            "source-uid-456",
4299            "real-bucket",
4300            "real-prefix",
4301            "https://real-endpoint.com",
4302            "eu-west-1",
4303            true,
4304            false,
4305            None,
4306            None,
4307        );
4308        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4309        assert!(parsed["caBundlePath"].is_null());
4310    }
4311
4312    #[test]
4313    fn source_prep_result_operation_is_download() {
4314        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4315        assert_ne!(
4316            super::SOURCE_PREP_RESULT_OPERATION,
4317            super::SAFETY_BACKUP_RESULT_OPERATION
4318        );
4319    }
4320
4321    #[test]
4322    fn download_result_document_verification_accepts_valid_document() {
4323        let result_json = r#"{
4324            "apiVersion": "backup.kaniop.rs/v1alpha1",
4325            "kind": "ResultDocument",
4326            "operation": "download",
4327            "success": true,
4328            "exitCode": "success",
4329            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4330            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4331            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4332        }"#;
4333        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4334        assert!(result.success);
4335        assert_eq!(
4336            result.exit_code,
4337            kaniop_backup_core::result::ExitCode::Success
4338        );
4339        assert_eq!(result.operation, "download");
4340        assert_eq!(
4341            result.backup_id.as_deref(),
4342            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4343        );
4344        assert!(result.manifest_key.is_some());
4345        assert!(result.payload_sha256.is_some());
4346    }
4347
4348    #[test]
4349    fn download_result_document_rejects_wrong_operation() {
4350        let result_json = r#"{
4351            "apiVersion": "backup.kaniop.rs/v1alpha1",
4352            "kind": "ResultDocument",
4353            "operation": "upload",
4354            "success": true,
4355            "exitCode": "success",
4356            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4357            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4358            "payloadSha256": "abc123"
4359        }"#;
4360        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4361        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4362    }
4363
4364    #[test]
4365    fn download_result_document_rejects_failure_success_field() {
4366        let result_json = r#"{
4367            "apiVersion": "backup.kaniop.rs/v1alpha1",
4368            "kind": "ResultDocument",
4369            "operation": "download",
4370            "success": false,
4371            "exitCode": "retryable",
4372            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4373            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4374            "payloadSha256": "abc123"
4375        }"#;
4376        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4377        assert!(!result.success);
4378        assert_ne!(
4379            result.exit_code,
4380            kaniop_backup_core::result::ExitCode::Success
4381        );
4382    }
4383
4384    #[test]
4385    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4386        let status = KanidmRestoreStatus {
4387            phase: KanidmRestorePhase::PreparingSource,
4388            database_mutation_started: false,
4389            ..Default::default()
4390        };
4391        assert!(!status.database_mutation_started);
4392        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4393    }
4394
4395    #[test]
4396    fn backup_job_volume_size_default_is_10gi() {
4397        let qty = crate::controller::backup_job_volume_size();
4398        assert_eq!(qty, Quantity("10Gi".to_string()));
4399    }
4400
4401    #[test]
4402    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4403        use kaniop_backup_core::auth::build_encryption_env_vars;
4404        use kaniop_backup_core::crd::SecretRef;
4405
4406        let key_ref = SecretRef {
4407            name: "kek-secret".to_string(),
4408        };
4409        let env_vars = build_encryption_env_vars(Some(&key_ref));
4410        assert_eq!(env_vars.len(), 1);
4411        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4412        let secret_ref = env_vars[0]
4413            .value_from
4414            .as_ref()
4415            .unwrap()
4416            .secret_key_ref
4417            .as_ref()
4418            .unwrap();
4419        assert_eq!(secret_ref.name, "kek-secret");
4420        assert_eq!(secret_ref.key, "encryption-key");
4421    }
4422
4423    #[test]
4424    fn encryption_env_vars_absent_when_no_key_ref() {
4425        use kaniop_backup_core::auth::build_encryption_env_vars;
4426
4427        let env_vars = build_encryption_env_vars(None);
4428        assert!(
4429            env_vars.is_empty(),
4430            "encryption env vars must be empty when key_ref is None"
4431        );
4432    }
4433
4434    #[test]
4435    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4436        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4437
4438        let source = KanidmRestoreSource {
4439            local: None,
4440            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4441                name: "backup-1".to_string(),
4442            }),
4443        };
4444        let mut restore = make_restore(source, None);
4445        restore.metadata.namespace = Some("test-ns".to_string());
4446        let target = super::super::crd::Kanidm::default();
4447        let encryption = RepositoryEncryption {
4448            mode: EncryptionMode::ClientSide,
4449            key_id: None,
4450            key_ref: Some(SecretRef {
4451                name: "kek-secret".to_string(),
4452            }),
4453        };
4454        let doc_str = super::build_safety_upload_operation_doc(
4455            &restore,
4456            &target,
4457            "test-backup-id",
4458            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4459            "my-bucket",
4460            "prod",
4461            "https://s3.example.com",
4462            "us-east-1",
4463            false,
4464            false,
4465            None,
4466            Some(&encryption),
4467        )
4468        .unwrap();
4469        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4470        assert_eq!(parsed["encryptionMode"], "clientSide");
4471        assert!(parsed["encryptionKeyId"].is_null());
4472    }
4473
4474    #[test]
4475    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4476        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4477
4478        let source = KanidmRestoreSource {
4479            local: None,
4480            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4481                name: "backup-1".to_string(),
4482            }),
4483        };
4484        let _restore = make_restore(source, None);
4485        let target = super::super::crd::Kanidm::default();
4486        let encryption = RepositoryEncryption {
4487            mode: EncryptionMode::ProviderKms,
4488            key_id: Some("alias/kaniop-backups".to_string()),
4489            key_ref: None,
4490        };
4491        let doc_str = super::build_download_operation_doc(
4492            &target,
4493            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4494            "019c7c76-f423-7a12-8f41-2bea7588a303",
4495            "source-uid-456",
4496            "real-bucket",
4497            "real-prefix",
4498            "https://real-endpoint.com",
4499            "eu-west-1",
4500            true,
4501            false,
4502            None,
4503            Some(&encryption),
4504        );
4505        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4506        assert_eq!(parsed["encryptionMode"], "providerKms");
4507        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4508    }
4509
4510    #[test]
4511    fn post_mutation_failure_records_job_names() {
4512        let existing = KanidmRestoreStatus {
4513            phase: KanidmRestorePhase::RestoringPrimary,
4514            database_mutation_started: true,
4515            ..Default::default()
4516        };
4517
4518        let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4519        assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4520        assert_eq!(
4521            restore_failed.restore_job_name.as_deref(),
4522            Some("my-restore-job")
4523        );
4524        assert!(restore_failed.verify_job_name.is_none());
4525        assert!(restore_failed.database_mutation_started);
4526        assert_eq!(
4527            restore_failed.message.as_deref(),
4528            Some("database restore job failed")
4529        );
4530
4531        let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4532        assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4533        assert_eq!(
4534            verify_failed.verify_job_name.as_deref(),
4535            Some("my-verify-job")
4536        );
4537        assert!(verify_failed.restore_job_name.is_none());
4538        assert!(verify_failed.database_mutation_started);
4539        assert_eq!(
4540            verify_failed.message.as_deref(),
4541            Some("database verification failed")
4542        );
4543    }
4544
4545    #[test]
4546    fn restore_and_verify_job_names_are_deterministic() {
4547        let source = KanidmRestoreSource {
4548            local: Some(super::KanidmRestoreLocalSource {
4549                file_name: "backup.json".to_string(),
4550            }),
4551            backup_ref: None,
4552        };
4553        let restore = make_restore(source, None);
4554        assert_eq!(restore_job_name(&restore), "test-restore-restore");
4555        assert_eq!(verify_job_name(&restore), "test-restore-verify");
4556    }
4557}