Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78    feature = "schemars",
79    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82    category = "kaniop",
83    group = "kaniop.rs",
84    version = "v1beta1",
85    kind = "KanidmRestore",
86    plural = "kanidmrestores",
87    singular = "kanidmrestore",
88    shortname = "idmrestore",
89    namespaced,
90    status = "KanidmRestoreStatus",
91    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97    pub target_ref: KanidmRestoreTargetRef,
98    pub source: KanidmRestoreSource,
99    pub restore_image: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108    pub name: String,
109    pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115    feature = "schemars",
116    schemars(extend("x-kubernetes-validations" = [
117        {
118            "message": "exactly one of local or backupRef must be set",
119            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120        }
121    ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125    /// Local file source. Mutually exclusive with `backupRef`.
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub local: Option<KanidmRestoreLocalSource>,
128    /// Remote cataloged backup reference. Mutually exclusive with `local`.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137    /// Basename of a backup below /data. Paths and traversal are rejected.
138    pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
146    pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub repository_ref: Option<SafetyBackupRepositoryRef>,
156    /// Skip the mandatory safety backup. Requires break-glass annotations.
157    #[serde(default)]
158    pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165    pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171    #[default]
172    Pending,
173    Validating,
174    Quiescing,
175    SafetyBackup,
176    PreparingSource,
177    RestoringPrimary,
178    Verifying,
179    RebuildingReplicas,
180    Resuming,
181    Completed,
182    Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_generation: Option<i64>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_target_uid: Option<String>,
193    #[serde(default)]
194    pub phase: KanidmRestorePhase,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub restore_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub verify_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub safety_backup_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub source_prep_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_ref: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_expected_backup_id: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_manifest_key: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_payload_sha256: Option<String>,
211    #[serde(default)]
212    pub replicas_cleared: bool,
213    #[serde(default)]
214    pub certificates_cleared: bool,
215    #[serde(default)]
216    pub database_mutation_started: bool,
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub original_replicas: Vec<ReplicaCountEntry>,
219    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220    pub phase_timestamps: std::collections::BTreeMap<String, String>,
221    #[serde(default, skip_serializing_if = "Vec::is_empty")]
222    #[cfg_attr(
223        feature = "schemars",
224        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225    )]
226    pub conditions: Vec<Condition>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235    pub group: String,
236    pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241    attempts: Counter<u64>,
242    outcomes: Counter<u64>,
243    duration_seconds: Histogram<f64>,
244    start_timestamp_seconds: Gauge<i64>,
245    break_glass_total: Counter<u64>,
246    disaster_recovery_total: Counter<u64>,
247    safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251    fn new() -> Self {
252        let meter = global::meter("kaniop");
253        Self {
254            attempts: meter
255                .u64_counter("restore_attempts")
256                .with_description("Number of Kanidm restore attempts started")
257                .build(),
258            outcomes: meter
259                .u64_counter("restore_outcomes")
260                .with_description("Number of terminal Kanidm restore outcomes")
261                .build(),
262            duration_seconds: meter
263                .f64_histogram("restore_duration_seconds")
264                .with_description("Kanidm restore duration from object creation to terminal phase")
265                .with_unit("s")
266                .build(),
267            start_timestamp_seconds: meter
268                .i64_gauge("restore_start_timestamp_seconds")
269                .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270                .with_unit("s")
271                .build(),
272            break_glass_total: meter
273                .u64_counter("restore_break_glass")
274                .with_description("Number of break-glass safety backup overrides")
275                .build(),
276            disaster_recovery_total: meter
277                .u64_counter("restore_disaster_recovery")
278                .with_description("Number of approved disaster-recovery source identity overrides")
279                .build(),
280            safety_backup_duration_seconds: meter
281                .f64_histogram("restore_safety_backup_duration_seconds")
282                .with_description("Duration of pre-restore safety backup creation")
283                .with_unit("s")
284                .build(),
285        }
286    }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290    let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291    let active = restore.metadata.deletion_timestamp.is_none()
292        && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293    let timestamp = if active {
294        restore
295            .metadata
296            .creation_timestamp
297            .as_ref()
298            .map(|created| created.0.as_second())
299            .unwrap_or_else(|| Timestamp::now().as_second())
300    } else {
301        0
302    };
303    let attributes = [
304        KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305        KeyValue::new("restore", restore.name_any()),
306    ];
307    metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312    client: Client,
313    recorder: Recorder,
314    metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319    match phase {
320        KanidmRestorePhase::Pending => {
321            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322            Ok(Action::requeue(REQUEUE))
323        }
324        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325            Ok((target, disaster_recovery_source)) => {
326                mark_restoring(&restore, &target, &ctx).await?;
327                if let Some(source) = disaster_recovery_source.as_ref() {
328                    record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329                }
330                let mut status = restore.status.clone().unwrap_or_default();
331                if let Some(source) = disaster_recovery_source.as_ref() {
332                    let target_uid = target.uid().unwrap_or_default();
333                    let condition = restore_condition(
334                        &status.conditions,
335                        CONDITION_DISASTER_RECOVERY,
336                        CONDITION_TRUE,
337                        "ApprovedSourceIdentityMismatch",
338                        &format!(
339                            "Approved disaster recovery from source {}/{} into target {}/{}",
340                            source.name,
341                            source.uid,
342                            target.name_any(),
343                            target_uid
344                        ),
345                        restore.metadata.generation,
346                    );
347                    status
348                        .conditions
349                        .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350                    status.conditions.push(condition);
351                }
352                status.observed_target_uid = target.uid();
353                status.original_replicas = capture_original_replicas(&target);
354                status.phase = KanidmRestorePhase::Quiescing;
355                status.message = None;
356                patch_status(&restore, &ctx, status).await?;
357                Ok(Action::requeue(REQUEUE))
358            }
359            Err(error) => {
360                set_phase(
361                    &restore,
362                    &ctx,
363                    KanidmRestorePhase::Failed,
364                    Some(error.to_string()),
365                )
366                .await?;
367                Ok(Action::requeue(Duration::from_secs(300)))
368            }
369        },
370        KanidmRestorePhase::Quiescing => {
371            let target = get_target(&restore, &ctx).await?;
372            scale_all(&target, &ctx, 0).await?;
373            if target_pods_stopped(&target, &ctx).await?
374                && target_volumes_detached(&target, &ctx).await?
375            {
376                let next_phase = if requires_safety_backup(&restore) {
377                    KanidmRestorePhase::SafetyBackup
378                } else {
379                    record_break_glass(&restore, &ctx).await?;
380                    KanidmRestorePhase::PreparingSource
381                };
382                if next_phase == KanidmRestorePhase::PreparingSource {
383                    let refreshed = Api::<KanidmRestore>::namespaced(
384                        ctx.client.clone(),
385                        restore.namespace().as_deref().unwrap_or_default(),
386                    )
387                    .get(&restore.name_any())
388                    .await
389                    .map_err(|error| {
390                        Error::kube_error(
391                            "get",
392                            "KanidmRestore",
393                            restore.namespace().as_deref().unwrap_or_default(),
394                            restore.name_any(),
395                            error,
396                        )
397                    })?;
398                    set_phase(&refreshed, &ctx, next_phase, None).await?;
399                } else {
400                    set_phase(&restore, &ctx, next_phase, None).await?;
401                }
402            }
403            Ok(Action::requeue(REQUEUE))
404        }
405        KanidmRestorePhase::SafetyBackup => {
406            let target = get_target(&restore, &ctx).await?;
407            let name = safety_backup_job_name(&restore);
408            let expected_backup_id = compute_safety_backup_id(&restore)?;
409            let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410            ensure_safety_backup_job(
411                &restore,
412                &target,
413                &ctx,
414                &name,
415                &expected_backup_id,
416                &expected_manifest_key,
417            )
418            .await?;
419            let mut status = restore.status.clone().unwrap_or_default();
420            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421            match job_state(&restore, &ctx, &name).await? {
422                JobState::Complete => {
423                    let result =
424                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425                    match result {
426                        Ok(verified) => {
427                            status.safety_backup_job_name = Some(name);
428                            status.safety_backup_ref =
429                                Some(format!("safety-{}", restore.name_any()));
430                            status.safety_backup_manifest_key = Some(verified.manifest_key);
431                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432                            record_safety_backup_duration(&restore, &ctx).await;
433                            status.phase = KanidmRestorePhase::PreparingSource;
434                            status.message = None;
435                            patch_status(&restore, &ctx, status).await?;
436                        }
437                        Err(error) => {
438                            resume_before_mutation(&restore, &ctx).await?;
439                            set_phase(
440                                &restore,
441                                &ctx,
442                                KanidmRestorePhase::Failed,
443                                Some(format!("safety backup result verification failed: {error}")),
444                            )
445                            .await?;
446                        }
447                    }
448                }
449                JobState::Failed => {
450                    resume_before_mutation(&restore, &ctx).await?;
451                    set_phase(
452                        &restore,
453                        &ctx,
454                        KanidmRestorePhase::Failed,
455                        Some(
456                            "safety backup job failed; target restored to original state"
457                                .to_string(),
458                        ),
459                    )
460                    .await?;
461                }
462                JobState::Running => {
463                    patch_status(&restore, &ctx, status).await?;
464                }
465            }
466            Ok(Action::requeue(REQUEUE))
467        }
468        KanidmRestorePhase::PreparingSource => {
469            if is_remote_source(&restore) {
470                let target = get_target(&restore, &ctx).await?;
471                let name = source_prep_job_name(&restore);
472                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473                match job_state(&restore, &ctx, &name).await? {
474                    JobState::Complete => {
475                        let backup_id = restore
476                            .spec
477                            .source
478                            .backup_ref
479                            .as_ref()
480                            .map(|b| b.name.clone())
481                            .unwrap_or_default();
482                        let backup = {
483                            let ns = restore.namespace().unwrap();
484                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485                                .get(&backup_id)
486                                .await
487                                .map_err(|e| {
488                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489                                })?
490                        };
491                        let expected_backup_id = &backup.spec.backup_id;
492                        let result =
493                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494                                .await;
495                        match result {
496                            Ok(verified) => {
497                                if let Some(expected_sha256) = backup
498                                    .status
499                                    .as_ref()
500                                    .and_then(|s| s.payload_sha256.as_deref())
501                                {
502                                    if verified.payload_sha256 != expected_sha256 {
503                                        resume_before_mutation(&restore, &ctx).await?;
504                                        set_phase(
505                                            &restore,
506                                            &ctx,
507                                            KanidmRestorePhase::Failed,
508                                            Some(format!(
509                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510                                                verified.payload_sha256
511                                            )),
512                                        )
513                                        .await?;
514                                        return Ok(Action::requeue(REQUEUE));
515                                    }
516                                }
517                                let mut status = restore.status.clone().unwrap_or_default();
518                                status.source_prep_job_name = Some(name);
519                                status.phase = KanidmRestorePhase::RestoringPrimary;
520                                status.message = None;
521                                patch_status(&restore, &ctx, status).await?;
522                            }
523                            Err(error) => {
524                                resume_before_mutation(&restore, &ctx).await?;
525                                set_phase(
526                                    &restore,
527                                    &ctx,
528                                    KanidmRestorePhase::Failed,
529                                    Some(format!(
530                                        "source preparation result verification failed: {error}"
531                                    )),
532                                )
533                                .await?;
534                            }
535                        }
536                    }
537                    JobState::Failed => {
538                        resume_before_mutation(&restore, &ctx).await?;
539                        set_phase(
540                            &restore,
541                            &ctx,
542                            KanidmRestorePhase::Failed,
543                            Some(
544                                "source preparation job failed; target restored to original state"
545                                    .to_string(),
546                            ),
547                        )
548                        .await?;
549                    }
550                    JobState::Running => {}
551                }
552            } else {
553                let target = get_target(&restore, &ctx).await?;
554                let name = source_check_job_name(&restore);
555                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556                match job_state(&restore, &ctx, &name).await? {
557                    JobState::Complete => {
558                        let mut status = restore.status.clone().unwrap_or_default();
559                        status.source_prep_job_name = Some(name);
560                        status.phase = KanidmRestorePhase::RestoringPrimary;
561                        status.message = None;
562                        patch_status(&restore, &ctx, status).await?;
563                    }
564                    JobState::Failed => {
565                        resume_before_mutation(&restore, &ctx).await?;
566                        set_phase(
567                            &restore,
568                            &ctx,
569                            KanidmRestorePhase::Failed,
570                            Some(
571                                "local backup file check failed; target restored to original state"
572                                    .to_string(),
573                            ),
574                        )
575                        .await?;
576                    }
577                    JobState::Running => {}
578                }
579            }
580            Ok(Action::requeue(REQUEUE))
581        }
582        KanidmRestorePhase::RestoringPrimary => {
583            let target = get_target(&restore, &ctx).await?;
584            let status = restore.status.as_ref().cloned().unwrap_or_default();
585            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586                return Err(Error::MissingData(
587                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588                ));
589            }
590            if !status.database_mutation_started {
591                let mut status = status;
592                status.database_mutation_started = true;
593                status.message = Some(
594                    "database mutation boundary persisted; starting primary restore".to_string(),
595                );
596                patch_status(&restore, &ctx, status).await?;
597                return Ok(Action::requeue(REQUEUE));
598            }
599            let name = restore_job_name(&restore);
600            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601            match job_state(&restore, &ctx, &name).await? {
602                JobState::Complete => {
603                    let mut status = restore.status.clone().unwrap_or_default();
604                    status.restore_job_name = Some(name);
605                    status.phase = KanidmRestorePhase::Verifying;
606                    status.message = None;
607                    patch_status(&restore, &ctx, status).await?;
608                }
609                JobState::Failed => {
610                    let status = restore.status.clone().unwrap_or_default();
611                    let failed_status = restore_primary_failed_status(&status, &name);
612                    patch_status(&restore, &ctx, failed_status).await?;
613                }
614                JobState::Running => {}
615            }
616            Ok(Action::requeue(REQUEUE))
617        }
618        KanidmRestorePhase::Verifying => {
619            let target = get_target(&restore, &ctx).await?;
620            let name = verify_job_name(&restore);
621            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622            match job_state(&restore, &ctx, &name).await? {
623                JobState::Complete => {
624                    let mut status = restore.status.clone().unwrap_or_default();
625                    status.verify_job_name = Some(name);
626                    status.phase = KanidmRestorePhase::RebuildingReplicas;
627                    status.message = None;
628                    patch_status(&restore, &ctx, status).await?;
629                }
630                JobState::Failed => {
631                    let status = restore.status.clone().unwrap_or_default();
632                    let failed_status = verifying_failed_status(&status, &name);
633                    patch_status(&restore, &ctx, failed_status).await?;
634                }
635                JobState::Running => {}
636            }
637            Ok(Action::requeue(REQUEUE))
638        }
639        KanidmRestorePhase::RebuildingReplicas => {
640            let target = get_target(&restore, &ctx).await?;
641            let mut status = restore.status.clone().unwrap_or_default();
642            if !status.replicas_cleared {
643                if delete_secondary_pvcs(&target, &ctx).await? {
644                    status.replicas_cleared = true;
645                    status.message = Some("secondary database state cleared".to_string());
646                    patch_status(&restore, &ctx, status).await?;
647                }
648                return Ok(Action::requeue(REQUEUE));
649            }
650
651            scale_primary(&target, &ctx, 1).await?;
652            if !primary_ready(&target, &ctx).await? {
653                return Ok(Action::requeue(REQUEUE));
654            }
655            if !status.certificates_cleared {
656                delete_replica_cert_secrets(&target, &ctx).await?;
657                delete_admin_secret(&target, &ctx).await?;
658                status.certificates_cleared = true;
659                status.message = Some(
660                    "replica certificates and admin secret cleared for regeneration".to_string(),
661                );
662                patch_status(&restore, &ctx, status).await?;
663                return Ok(Action::requeue(REQUEUE));
664            }
665            status.phase = KanidmRestorePhase::Resuming;
666            status.message = None;
667            patch_status(&restore, &ctx, status).await?;
668            Ok(Action::requeue(REQUEUE))
669        }
670        KanidmRestorePhase::Resuming => {
671            let target = get_target(&restore, &ctx).await?;
672            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673            clear_restoring(&restore, &target, &ctx).await?;
674            Ok(Action::requeue(Duration::from_secs(3600)))
675        }
676        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677            Ok(Action::requeue(Duration::from_secs(3600)))
678        }
679    }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684    name: String,
685    uid: String,
686}
687
688async fn validate(
689    restore: &KanidmRestore,
690    ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692    let target = get_target(restore, ctx).await?;
693    let actual_uid = target
694        .uid()
695        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696    if actual_uid != restore.spec.target_ref.uid {
697        return Err(Error::MissingData(format!(
698            "target UID mismatch: expected {}, got {}",
699            restore.spec.target_ref.uid, actual_uid
700        )));
701    }
702    validate_source(restore)?;
703    let disaster_recovery_source = if is_remote_source(restore) {
704        validate_backup_ref(restore, &target, ctx).await?
705    } else {
706        None
707    };
708    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709    {
710        return Err(Error::MissingData(format!(
711            "restoreImage must be the target's pinned Kanidm image (target image is {})",
712            target.spec.image
713        )));
714    }
715    let storage = target
716        .spec
717        .storage
718        .as_ref()
719        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720    if storage.empty_dir.is_some()
721        || storage.ephemeral.is_some()
722        || storage.volume_claim_template.is_none()
723    {
724        return Err(Error::MissingData(
725            "restore requires PVC-backed Kanidm storage".to_string(),
726        ));
727    }
728    let primaries = target
729        .spec
730        .replica_groups
731        .iter()
732        .filter(|rg| rg.primary_node)
733        .count();
734    if primaries != 1 {
735        return Err(Error::MissingData(
736            "backup/restore requires exactly one primary replica group".to_string(),
737        ));
738    }
739    let ns = restore.namespace().unwrap();
740    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741        .list(&ListParams::default())
742        .await
743        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744    if restores.items.iter().any(|other| {
745        other.name_any() != restore.name_any()
746            && other.spec.target_ref.name == restore.spec.target_ref.name
747            && !matches!(
748                other.status.as_ref().map(|s| s.phase),
749                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750            )
751    }) {
752        return Err(Error::MissingData(
753            "another active restore targets this Kanidm".to_string(),
754        ));
755    }
756    validate_safety_backup_config(restore)?;
757    Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761    !name.is_empty()
762        && name != "."
763        && name != ".."
764        && !name.contains('/')
765        && !name.contains('\\')
766        && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770    image == "kanidm/server:latest"
771        || image.ends_with(":latest")
772        || (!image.contains('@')
773            && !image
774                .rsplit('/')
775                .next()
776                .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780    let ns = restore
781        .namespace()
782        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784        .get(&restore.spec.target_ref.name)
785        .await
786        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790    restore: &KanidmRestore,
791    target: &Kanidm,
792    ctx: &RestoreContext,
793) -> Result<()> {
794    let ns = target.namespace().unwrap();
795    let uid = restore
796        .uid()
797        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799        .patch(
800            &target.name_any(),
801            &PatchParams::default(),
802            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803        )
804        .await
805        .map(|_| ())
806        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810    restore: &KanidmRestore,
811    target: &Kanidm,
812    ctx: &RestoreContext,
813) -> Result<()> {
814    let ns = target.namespace().unwrap();
815    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816        return Ok(());
817    }
818    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819        .patch(
820            &target.name_any(),
821            &PatchParams::default(),
822            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823        )
824        .await
825        .map(|_| ())
826        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830    restore: &KanidmRestore,
831    ctx: &RestoreContext,
832    mut status: KanidmRestoreStatus,
833) -> Result<()> {
834    let ns = restore.namespace().unwrap();
835    let previous_phase = restore.status.as_ref().map(|current| current.phase);
836    let next_phase = status.phase;
837    status.observed_generation = restore.metadata.generation;
838    update_restore_conditions(&mut status, restore.metadata.generation);
839
840    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841        .patch_status(
842            &restore.name_any(),
843            &PatchParams::default(),
844            &Patch::Merge(json!({"status": &status})),
845        )
846        .await
847        .map_err(|e| {
848            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849        })?;
850
851    if previous_phase != Some(next_phase) {
852        record_restore_transition(
853            restore,
854            ctx,
855            previous_phase,
856            next_phase,
857            status.message.as_deref(),
858        )
859        .await;
860    }
861    Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865    let previous = status.conditions.clone();
866    let phase = status.phase;
867    let phase_reason = format!("{phase:?}");
868    let message = status
869        .message
870        .clone()
871        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872    let terminal_success = phase == KanidmRestorePhase::Completed;
873    let terminal_failure = phase == KanidmRestorePhase::Failed;
874    let progressing = !terminal_success && !terminal_failure;
875
876    let sticky_conditions: Vec<Condition> = previous
877        .iter()
878        .filter(|condition| {
879            condition.type_ == CONDITION_BREAK_GLASS
880                || condition.type_ == CONDITION_DISASTER_RECOVERY
881        })
882        .cloned()
883        .collect();
884    status.conditions = vec![
885        restore_condition(
886            &previous,
887            CONDITION_PROGRESSING,
888            if progressing {
889                CONDITION_TRUE
890            } else {
891                CONDITION_FALSE
892            },
893            &phase_reason,
894            &message,
895            generation,
896        ),
897        restore_condition(
898            &previous,
899            CONDITION_READY,
900            if terminal_success {
901                CONDITION_TRUE
902            } else {
903                CONDITION_FALSE
904            },
905            if terminal_success {
906                "RestoreCompleted"
907            } else {
908                "RestoreNotCompleted"
909            },
910            if terminal_success {
911                "Kanidm restore completed successfully."
912            } else {
913                "Kanidm restore has not completed successfully."
914            },
915            generation,
916        ),
917        restore_condition(
918            &previous,
919            CONDITION_FAILED,
920            if terminal_failure {
921                CONDITION_TRUE
922            } else {
923                CONDITION_FALSE
924            },
925            if terminal_failure {
926                "RestoreFailed"
927            } else {
928                "NoRestoreFailure"
929            },
930            if terminal_failure {
931                status
932                    .message
933                    .as_deref()
934                    .unwrap_or("Kanidm restore failed.")
935            } else {
936                "No terminal restore failure has been recorded."
937            },
938            generation,
939        ),
940    ];
941    status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945    previous: &[Condition],
946    condition_type: &str,
947    condition_status: &str,
948    reason: &str,
949    message: &str,
950    generation: Option<i64>,
951) -> Condition {
952    let last_transition_time = previous
953        .iter()
954        .find(|condition| {
955            condition.type_ == condition_type
956                && condition.status == condition_status
957                && condition.reason == reason
958        })
959        .map(|condition| condition.last_transition_time.clone())
960        .unwrap_or_else(|| Time(Timestamp::now()));
961    Condition {
962        type_: condition_type.to_string(),
963        status: condition_status.to_string(),
964        reason: reason.to_string(),
965        message: message.to_string(),
966        last_transition_time,
967        observed_generation: generation,
968    }
969}
970
971async fn record_restore_transition(
972    restore: &KanidmRestore,
973    ctx: &RestoreContext,
974    previous_phase: Option<KanidmRestorePhase>,
975    phase: KanidmRestorePhase,
976    message: Option<&str>,
977) {
978    if phase == KanidmRestorePhase::Validating {
979        ctx.metrics.attempts.add(1, &[]);
980    }
981
982    let result = match phase {
983        KanidmRestorePhase::Completed => Some("success"),
984        KanidmRestorePhase::Failed => Some("failure"),
985        _ => None,
986    };
987    if let Some(result) = result {
988        let attributes = [
989            KeyValue::new("result", result),
990            KeyValue::new("phase", format!("{phase:?}")),
991        ];
992        ctx.metrics.outcomes.add(1, &attributes);
993        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995            ctx.metrics.duration_seconds.record(elapsed, &attributes);
996        }
997    }
998
999    let reason = if phase == KanidmRestorePhase::Failed {
1000        "RestoreFailed"
1001    } else {
1002        "RestorePhaseChanged"
1003    };
1004    let note = message.map(str::to_string).or_else(|| {
1005        Some(format!(
1006            "Kanidm restore phase changed from {} to {phase:?}.",
1007            previous_phase
1008                .map(|previous| format!("{previous:?}"))
1009                .unwrap_or_else(|| "None".to_string())
1010        ))
1011    });
1012    if let Err(error) = ctx
1013        .recorder
1014        .publish(
1015            &Event {
1016                type_: if phase == KanidmRestorePhase::Failed {
1017                    EventType::Warning
1018                } else {
1019                    EventType::Normal
1020                },
1021                reason: reason.to_string(),
1022                note,
1023                action: "Restore".to_string(),
1024                secondary: None,
1025            },
1026            &restore.object_ref(&()),
1027        )
1028        .await
1029    {
1030        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031    }
1032}
1033
1034async fn set_phase(
1035    restore: &KanidmRestore,
1036    ctx: &RestoreContext,
1037    phase: KanidmRestorePhase,
1038    message: Option<String>,
1039) -> Result<()> {
1040    let mut status = restore.status.clone().unwrap_or_default();
1041    status.phase = phase;
1042    status.message = message;
1043    patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047    restore: &KanidmRestore,
1048    ctx: &RestoreContext,
1049    message: &str,
1050) -> Result<()> {
1051    set_phase(
1052        restore,
1053        ctx,
1054        KanidmRestorePhase::Failed,
1055        Some(message.to_string()),
1056    )
1057    .await
1058}
1059
1060fn restore_primary_failed_status(
1061    existing: &KanidmRestoreStatus,
1062    job_name: &str,
1063) -> KanidmRestoreStatus {
1064    let mut status = existing.clone();
1065    status.restore_job_name = Some(job_name.to_string());
1066    status.phase = KanidmRestorePhase::Failed;
1067    status.message = Some("database restore job failed".to_string());
1068    status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072    let mut status = existing.clone();
1073    status.verify_job_name = Some(job_name.to_string());
1074    status.phase = KanidmRestorePhase::Failed;
1075    status.message = Some("database verification failed".to_string());
1076    status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080    target
1081        .spec
1082        .replica_groups
1083        .iter()
1084        .find(|rg| rg.primary_node)
1085        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089    let ns = target.namespace().unwrap();
1090    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091    for rg in &target.spec.replica_groups {
1092        let name = target.statefulset_name(&rg.name);
1093        api.patch(
1094            &name,
1095            &PatchParams::default(),
1096            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097        )
1098        .await
1099        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100    }
1101    Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105    let ns = target.namespace().unwrap();
1106    let rg = primary_group(target)?;
1107    let name = target.statefulset_name(&rg.name);
1108    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109        .patch(
1110            &name,
1111            &PatchParams::default(),
1112            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113        )
1114        .await
1115        .map(|_| ())
1116        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120    let ns = target.namespace().unwrap();
1121    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122    for rg in &target.spec.replica_groups {
1123        let name = target.statefulset_name(&rg.name);
1124        api.patch(
1125            &name,
1126            &PatchParams::default(),
1127            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128        )
1129        .await
1130        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131    }
1132    Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136    let ns = target.namespace().unwrap();
1137    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139        .await
1140        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141    Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145    let ns = target.namespace().unwrap();
1146    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147    let mut pv_names = BTreeSet::new();
1148    let primary_name = primary_pvc_name(target)?;
1149    let primary_pvc = pvc_api
1150        .get(&primary_name)
1151        .await
1152        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153    let primary_pv = primary_pvc
1154        .spec
1155        .as_ref()
1156        .and_then(|spec| spec.volume_name.clone())
1157        .ok_or_else(|| {
1158            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159        })?;
1160    pv_names.insert(primary_pv);
1161
1162    for rg in &target.spec.replica_groups {
1163        let sts = target.statefulset_name(&rg.name);
1164        for ordinal in 0..rg.replicas {
1165            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166            if name == primary_name {
1167                continue;
1168            }
1169            if let Some(pvc) = pvc_api
1170                .get_opt(&name)
1171                .await
1172                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174            {
1175                pv_names.insert(pv_name);
1176            }
1177        }
1178    }
1179
1180    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181        .list(&ListParams::default())
1182        .await
1183        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184    Ok(!attachments.items.iter().any(|attachment| {
1185        attachment
1186            .spec
1187            .source
1188            .persistent_volume_name
1189            .as_ref()
1190            .is_some_and(|pv_name| pv_names.contains(pv_name))
1191    }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195    let ns = target.namespace().unwrap();
1196    let name = target.statefulset_name(&primary_group(target)?.name);
1197    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198        .get(&name)
1199        .await
1200        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201    Ok(sts
1202        .status
1203        .as_ref()
1204        .and_then(|s| s.ready_replicas)
1205        .unwrap_or(0)
1206        >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210    let rg = primary_group(target)?;
1211    Ok(format!(
1212        "{DATA_VOLUME}-{}-0",
1213        target.statefulset_name(&rg.name)
1214    ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218    let ns = target.namespace().unwrap();
1219    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220    let mut all_absent = true;
1221    for rg in &target.spec.replica_groups {
1222        let sts = target.statefulset_name(&rg.name);
1223        for ordinal in 0..rg.replicas {
1224            if rg.primary_node && ordinal == 0 {
1225                continue;
1226            }
1227            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228            match api
1229                .get_opt(&name)
1230                .await
1231                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232            {
1233                None => {}
1234                Some(pvc) => {
1235                    all_absent = false;
1236                    if pvc.metadata.deletion_timestamp.is_none() {
1237                        match api.delete(&name, &DeleteParams::default()).await {
1238                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1240                            Err(error) => {
1241                                return Err(Error::kube_error(
1242                                    "delete",
1243                                    "PersistentVolumeClaim",
1244                                    &ns,
1245                                    &name,
1246                                    error,
1247                                ));
1248                            }
1249                        }
1250                    }
1251                }
1252            }
1253        }
1254    }
1255    Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259    let ns = target.namespace().unwrap();
1260    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261    let label_selector = format!(
1262        "{}={},{}={}",
1263        CLUSTER_LABEL,
1264        target.name_any(),
1265        SECRET_TYPE_LABEL,
1266        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267    );
1268    let secrets = api
1269        .list(&ListParams::default().labels(&label_selector))
1270        .await
1271        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272    let mut all_absent = secrets.items.is_empty();
1273    for secret in secrets.items {
1274        let name = secret.name_any();
1275        if secret.metadata.deletion_timestamp.is_none() {
1276            match api.delete(&name, &DeleteParams::default()).await {
1277                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278                Err(kube::Error::Api(status)) if status.code == 404 => {}
1279                Err(error) => {
1280                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281                }
1282            }
1283            all_absent = false;
1284        }
1285    }
1286    Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290    let ns = target.namespace().unwrap();
1291    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292    let name = format!("{}-admin-passwords", target.name_any());
1293    match api
1294        .get_opt(&name)
1295        .await
1296        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297    {
1298        None => Ok(true),
1299        Some(secret) => {
1300            if secret.metadata.deletion_timestamp.is_some() {
1301                return Ok(false);
1302            }
1303            match api.delete(&name, &DeleteParams::default()).await {
1304                Ok(_) => {
1305                    debug!(secret = %name, "deleting stale admin secret");
1306                    Ok(false)
1307                }
1308                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310            }
1311        }
1312    }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316    format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319    format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323    let origin = target
1324        .spec
1325        .origin
1326        .clone()
1327        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328    vec![
1329        EnvVar {
1330            name: "KANIDM_DB_PATH".to_string(),
1331            value: Some("/data/kanidm.db".to_string()),
1332            ..Default::default()
1333        },
1334        EnvVar {
1335            name: "KANIDM_DOMAIN".to_string(),
1336            value: Some(target.spec.domain.clone()),
1337            ..Default::default()
1338        },
1339        EnvVar {
1340            name: "KANIDM_ORIGIN".to_string(),
1341            value: Some(origin),
1342            ..Default::default()
1343        },
1344        EnvVar {
1345            name: "KANIDM_TLS_CHAIN".to_string(),
1346            value: Some(format!("{TLS_PATH}/tls.crt")),
1347            ..Default::default()
1348        },
1349        EnvVar {
1350            name: "KANIDM_TLS_KEY".to_string(),
1351            value: Some(format!("{TLS_PATH}/tls.key")),
1352            ..Default::default()
1353        },
1354    ]
1355}
1356
1357async fn ensure_database_job(
1358    restore: &KanidmRestore,
1359    target: &Kanidm,
1360    ctx: &RestoreContext,
1361    name: &str,
1362    verify: bool,
1363) -> Result<()> {
1364    let ns = restore.namespace().unwrap();
1365    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366    if api
1367        .get_opt(name)
1368        .await
1369        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370        .is_some()
1371    {
1372        return Ok(());
1373    }
1374    let mut command = vec![
1375        "kanidmd".to_string(),
1376        "database".to_string(),
1377        if verify { "backup" } else { "restore" }.to_string(),
1378    ];
1379    if verify {
1380        command.push("/verify/verification.json.gz".to_string());
1381    } else {
1382        if is_remote_source(restore) {
1383            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384        } else {
1385            let file_name = restore
1386                .spec
1387                .source
1388                .local
1389                .as_ref()
1390                .map(|l| l.file_name.clone())
1391                .unwrap_or_default();
1392            command.push(format!("{BACKUP_PATH}/{file_name}"));
1393        }
1394    }
1395    let job = Job {
1396        metadata: ObjectMeta {
1397            name: Some(name.to_string()),
1398            namespace: Some(ns.clone()),
1399            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400            ..Default::default()
1401        },
1402        spec: Some(JobSpec {
1403            backoff_limit: Some(0),
1404            template: PodTemplateSpec {
1405                metadata: Some(ObjectMeta::default()),
1406                spec: Some(PodSpec {
1407                    automount_service_account_token: Some(false),
1408                    restart_policy: Some("Never".to_string()),
1409                    security_context: Some(source_job_pod_security_context(target)),
1410                    containers: vec![Container {
1411                        name: if verify { "verify" } else { "restore" }.to_string(),
1412                        image: Some(restore.spec.restore_image.clone()),
1413                        command: Some(command),
1414                        env: Some(build_kanidm_db_env_vars(target)),
1415                        security_context: Some(kanidm_job_security_context()),
1416                        resources: Some(job_resource_requirements()),
1417                        volume_mounts: Some(vec![
1418                            VolumeMount {
1419                                name: DATA_VOLUME.to_string(),
1420                                mount_path: DATA_PATH.to_string(),
1421                                ..Default::default()
1422                            },
1423                            VolumeMount {
1424                                name: TLS_VOLUME.to_string(),
1425                                mount_path: TLS_PATH.to_string(),
1426                                read_only: Some(true),
1427                                ..Default::default()
1428                            },
1429                            VolumeMount {
1430                                name: "verification-output".to_string(),
1431                                mount_path: "/verify".to_string(),
1432                                ..Default::default()
1433                            },
1434                        ]),
1435                        ..Default::default()
1436                    }],
1437                    volumes: Some(vec![
1438                        Volume {
1439                            name: DATA_VOLUME.to_string(),
1440                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441                                claim_name: primary_pvc_name(target)?,
1442                                read_only: Some(false),
1443                            }),
1444                            ..Default::default()
1445                        },
1446                        Volume {
1447                            name: TLS_VOLUME.to_string(),
1448                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449                                secret_name: Some(target.effective_tls_secret_name()),
1450                                ..Default::default()
1451                            }),
1452                            ..Default::default()
1453                        },
1454                        Volume {
1455                            name: "verification-output".to_string(),
1456                            empty_dir: Some(Default::default()),
1457                            ..Default::default()
1458                        },
1459                    ]),
1460                    ..Default::default()
1461                }),
1462            },
1463            ..Default::default()
1464        }),
1465        ..Default::default()
1466    };
1467    api.create(&PostParams::default(), &job)
1468        .await
1469        .map(|_| ())
1470        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474    Running,
1475    Complete,
1476    Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480    let ns = restore.namespace().unwrap();
1481    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482        .get(name)
1483        .await
1484        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486        return Ok(JobState::Failed);
1487    }
1488    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489        return Ok(JobState::Complete);
1490    }
1491    Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495    restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499    let safety = restore.spec.safety_backup.as_ref();
1500    let skip = safety.is_some_and(|s| s.skip);
1501    !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505    let has_local = restore.spec.source.local.is_some();
1506    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507    if has_local && has_backup_ref {
1508        return Err(Error::MissingData(
1509            "source.local and source.backupRef are mutually exclusive".to_string(),
1510        ));
1511    }
1512    if !has_local && !has_backup_ref {
1513        return Err(Error::MissingData(
1514            "source must specify either local or backupRef".to_string(),
1515        ));
1516    }
1517    if let Some(local) = &restore.spec.source.local {
1518        if !safe_basename(&local.file_name) {
1519            return Err(Error::MissingData(
1520                "restore source fileName must be a safe basename".to_string(),
1521            ));
1522        }
1523    }
1524    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525        if backup_ref.name.is_empty() {
1526            return Err(Error::MissingData(
1527                "source.backupRef.name must not be empty".to_string(),
1528            ));
1529        }
1530    }
1531    Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535    restore
1536        .metadata
1537        .annotations
1538        .as_ref()
1539        .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540        .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544    let annotations = restore.metadata.annotations.as_ref();
1545    let reason = annotations
1546        .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547        .map(String::as_str)
1548        .filter(|value| !value.trim().is_empty());
1549    let approved_by = annotations
1550        .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551        .map(String::as_str)
1552        .filter(|value| !value.trim().is_empty());
1553
1554    if reason.is_none() || approved_by.is_none() {
1555        return Err(Error::MissingData(format!(
1556            "disaster-recovery restore requires non-empty {} and {} annotations",
1557            BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558        )));
1559    }
1560    Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564    restore: &KanidmRestore,
1565    target: &Kanidm,
1566    ctx: &RestoreContext,
1567    source: &DisasterRecoverySource,
1568) {
1569    ctx.metrics.disaster_recovery_total.add(1, &[]);
1570    let reason = restore
1571        .metadata
1572        .annotations
1573        .as_ref()
1574        .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575        .cloned()
1576        .unwrap_or_default();
1577    let approved_by = restore
1578        .metadata
1579        .annotations
1580        .as_ref()
1581        .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582        .cloned()
1583        .unwrap_or_default();
1584    let target_uid = target.uid().unwrap_or_default();
1585    let note = format!(
1586        "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587        source.name,
1588        source.uid,
1589        target.name_any(),
1590        target_uid,
1591        reason,
1592        approved_by
1593    );
1594    if let Err(error) = ctx
1595        .recorder
1596        .publish(
1597            &Event {
1598                type_: EventType::Warning,
1599                reason: "DisasterRecoveryOverride".to_string(),
1600                note: Some(note.clone()),
1601                action: "Restore".to_string(),
1602                secondary: None,
1603            },
1604            &restore.object_ref(&()),
1605        )
1606        .await
1607    {
1608        warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609    }
1610    warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614    let safety = restore.spec.safety_backup.as_ref();
1615    let skip = safety.is_some_and(|s| s.skip);
1616    if skip {
1617        let annotations = restore.metadata.annotations.as_ref();
1618        let reason = annotations
1619            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620            .map(|s| s.trim())
1621            .unwrap_or("");
1622        let approver = annotations
1623            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624            .map(|s| s.trim())
1625            .unwrap_or("");
1626        if reason.is_empty() {
1627            return Err(Error::MissingData(format!(
1628                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629            )));
1630        }
1631        if approver.is_empty() {
1632            return Err(Error::MissingData(format!(
1633                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634            )));
1635        }
1636    }
1637    if !skip {
1638        let has_repo = safety
1639            .and_then(|s| s.repository_ref.as_ref())
1640            .is_some_and(|r| !r.name.is_empty());
1641        if !has_repo {
1642            return Err(Error::MissingData(
1643                "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644                    .to_string(),
1645            ));
1646        }
1647    }
1648    Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652    target
1653        .spec
1654        .replica_groups
1655        .iter()
1656        .map(|rg| ReplicaCountEntry {
1657            group: rg.name.clone(),
1658            replicas: rg.replicas,
1659        })
1660        .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664    if let Ok(target) = get_target(restore, ctx).await {
1665        let owns_maintenance =
1666            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667        if owns_maintenance {
1668            scale_desired(&target, ctx).await?;
1669            clear_restoring(restore, &target, ctx).await?;
1670        }
1671    }
1672    Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676    ctx.metrics.break_glass_total.add(1, &[]);
1677    let reason = restore
1678        .metadata
1679        .annotations
1680        .as_ref()
1681        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682        .cloned()
1683        .unwrap_or_default();
1684    let approver = restore
1685        .metadata
1686        .annotations
1687        .as_ref()
1688        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689        .cloned()
1690        .unwrap_or_default();
1691    let mut status = restore.status.clone().unwrap_or_default();
1692    let break_glass_condition = restore_condition(
1693        &status.conditions,
1694        CONDITION_BREAK_GLASS,
1695        CONDITION_TRUE,
1696        "BreakGlassOverride",
1697        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698        restore.metadata.generation,
1699    );
1700    status
1701        .conditions
1702        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703    status.conditions.push(break_glass_condition);
1704    patch_status(restore, ctx, status).await?;
1705    if let Err(error) = ctx
1706        .recorder
1707        .publish(
1708            &Event {
1709                type_: EventType::Warning,
1710                reason: "SafetyBackupSkipped".to_string(),
1711                note: Some(format!(
1712                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713                )),
1714                action: "Restore".to_string(),
1715                secondary: None,
1716            },
1717            &restore.object_ref(&()),
1718        )
1719        .await
1720    {
1721        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722    }
1723    warn!(
1724        restore = %restore.name_any(),
1725        reason = %reason,
1726        approver = %approver,
1727        "break-glass safety backup override activated"
1728    );
1729    Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733    let status = restore.status.as_ref().cloned().unwrap_or_default();
1734    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735        if let Ok(start_ts) = start.parse::<Timestamp>() {
1736            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737            ctx.metrics
1738                .safety_backup_duration_seconds
1739                .record(elapsed, &[]);
1740        }
1741    }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745    format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749    format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753    format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757    restore: &KanidmRestore,
1758    target: &Kanidm,
1759    ctx: &RestoreContext,
1760    name: &str,
1761) -> Result<()> {
1762    let ns = restore.namespace().unwrap();
1763    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764    if api
1765        .get_opt(name)
1766        .await
1767        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768        .is_some()
1769    {
1770        return Ok(());
1771    }
1772    let file_name = restore
1773        .spec
1774        .source
1775        .local
1776        .as_ref()
1777        .map(|l| l.file_name.clone())
1778        .unwrap_or_default();
1779    let operation_doc = serde_json::json!({
1780        "apiVersion": "backup.kaniop.rs/v1alpha1",
1781        "kind": "OperationDocument",
1782        "operation": "check",
1783        "path": format!("{BACKUP_PATH}/{file_name}"),
1784        "resultPath": "/run/kaniop-result/result.json",
1785        "format": "kanidmJsonGzip",
1786    })
1787    .to_string();
1788    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790    let data_mover_image = data_mover_image();
1791    let job = Job {
1792        metadata: ObjectMeta {
1793            name: Some(name.to_string()),
1794            namespace: Some(ns.clone()),
1795            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796            ..Default::default()
1797        },
1798        spec: Some(JobSpec {
1799            backoff_limit: Some(0),
1800            template: PodTemplateSpec {
1801                metadata: Some(ObjectMeta::default()),
1802                spec: Some(PodSpec {
1803                    automount_service_account_token: Some(false),
1804                    restart_policy: Some("Never".to_string()),
1805                    security_context: Some(source_job_pod_security_context(target)),
1806                    containers: vec![Container {
1807                        name: "source-check".to_string(),
1808                        image: Some(data_mover_image),
1809                        command: Some(vec![
1810                            "/bin/kaniop-data-mover".to_string(),
1811                            "check".to_string(),
1812                            "--operation-doc".to_string(),
1813                            "/run/kaniop/operation.json".to_string(),
1814                        ]),
1815                        security_context: Some(hardened_security_context()),
1816                        resources: Some(job_resource_requirements()),
1817                        env: Some(vec![EnvVar {
1818                            name: "RUST_LOG".to_string(),
1819                            value: Some("info".to_string()),
1820                            ..Default::default()
1821                        }]),
1822                        volume_mounts: Some(vec![
1823                            VolumeMount {
1824                                name: DATA_VOLUME.to_string(),
1825                                mount_path: DATA_PATH.to_string(),
1826                                read_only: Some(true),
1827                                ..Default::default()
1828                            },
1829                            VolumeMount {
1830                                name: "operation".to_string(),
1831                                mount_path: "/run/kaniop".to_string(),
1832                                read_only: Some(true),
1833                                ..Default::default()
1834                            },
1835                            VolumeMount {
1836                                name: "result".to_string(),
1837                                mount_path: "/run/kaniop-result".to_string(),
1838                                ..Default::default()
1839                            },
1840                        ]),
1841                        ..Default::default()
1842                    }],
1843                    volumes: Some(vec![
1844                        Volume {
1845                            name: DATA_VOLUME.to_string(),
1846                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847                                claim_name: primary_pvc_name(target)?,
1848                                read_only: Some(true),
1849                            }),
1850                            ..Default::default()
1851                        },
1852                        Volume {
1853                            name: "operation".to_string(),
1854                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855                                name: operation_cm_name,
1856                                ..Default::default()
1857                            }),
1858                            ..Default::default()
1859                        },
1860                        Volume {
1861                            name: "result".to_string(),
1862                            empty_dir: Some(Default::default()),
1863                            ..Default::default()
1864                        },
1865                    ]),
1866                    ..Default::default()
1867                }),
1868            },
1869            ..Default::default()
1870        }),
1871        ..Default::default()
1872    };
1873    api.create(&PostParams::default(), &job)
1874        .await
1875        .map(|_| ())
1876        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882    let ns = restore.namespace().unwrap_or_default();
1883    let uid = restore.uid().ok_or_else(|| {
1884        Error::MissingData(
1885            "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886                .to_string(),
1887        )
1888    })?;
1889    let namespace_uuid =
1890        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892    Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895/// Compute the safety-backup manifest key.
1896///
1897/// This key is hand-rolled with `format!` instead of going through
1898/// [`kaniop_backup_core::paths::RepositoryPath`], so it is the site most exposed to keyspace drift:
1899/// it must stay byte-identical to `RepositoryPath::manifest_key`, and its `tenants/{ns}` segment is
1900/// the Kubernetes namespace NAME (`restore.namespace()`), not the Namespace object UID, matching the
1901/// `namespaceUid` wire field the upload operation emits (misnomer tracked in issue #1084).
1902fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1903    let ns = restore.namespace().unwrap_or_default();
1904    let kanidm_uid = &restore.spec.target_ref.uid;
1905    let backup_id = compute_safety_backup_id(restore)?;
1906    Ok(format!(
1907        "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1908    ))
1909}
1910
1911struct VerifiedSafetyBackup {
1912    manifest_key: String,
1913    payload_sha256: String,
1914}
1915
1916async fn read_safety_backup_result(
1917    restore: &KanidmRestore,
1918    ctx: &RestoreContext,
1919    job_name: &str,
1920    expected_backup_id: &str,
1921) -> Result<VerifiedSafetyBackup> {
1922    let ns = restore.namespace().unwrap();
1923    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924    let pod_list = pods
1925        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926        .await
1927        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928    let pod = pod_list.items.first().ok_or_else(|| {
1929        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1930    })?;
1931    let pod_name = pod.name_any();
1932    let container_status = pod
1933        .status
1934        .as_ref()
1935        .and_then(|s| s.container_statuses.as_ref())
1936        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1937        .ok_or_else(|| {
1938            Error::MissingData(format!(
1939                "safety-upload container status not found in pod {pod_name}"
1940            ))
1941        })?;
1942    let termination_message = container_status
1943        .state
1944        .as_ref()
1945        .and_then(|state| state.terminated.as_ref())
1946        .and_then(|t| t.message.as_ref())
1947        .ok_or_else(|| {
1948            Error::MissingData(format!(
1949                "termination message not found in pod {pod_name}; result transport failed"
1950            ))
1951        })?;
1952    if termination_message.is_empty() {
1953        return Err(Error::MissingData(format!(
1954            "termination message is empty in pod {pod_name}; result document was not written"
1955        )));
1956    }
1957    let result_doc = parse_result_document(termination_message).map_err(|e| {
1958        Error::ParseError(format!(
1959            "failed to parse result document from pod {pod_name}: {e}"
1960        ))
1961    })?;
1962    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963        return Err(Error::ParseError(format!(
1964            "unsupported result document version: {}",
1965            result_doc.api_version
1966        )));
1967    }
1968    if result_doc.kind != "ResultDocument" {
1969        return Err(Error::ParseError(format!(
1970            "invalid result document kind: {}",
1971            result_doc.kind
1972        )));
1973    }
1974    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1975        return Err(Error::ParseError(format!(
1976            "result document operation mismatch: expected '{}', got '{}'",
1977            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1978        )));
1979    }
1980    if !result_doc.success {
1981        return Err(Error::ParseError(format!(
1982            "safety backup reported failure with exit code {:?}",
1983            result_doc.exit_code
1984        )));
1985    }
1986    if result_doc.exit_code != ExitCode::Success {
1987        return Err(Error::ParseError(format!(
1988            "safety backup exit code is not Success: {:?}",
1989            result_doc.exit_code
1990        )));
1991    }
1992    let doc_backup_id = result_doc
1993        .backup_id
1994        .as_deref()
1995        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996    if doc_backup_id != expected_backup_id {
1997        return Err(Error::ParseError(format!(
1998            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999        )));
2000    }
2001    let manifest_key = result_doc
2002        .manifest_key
2003        .clone()
2004        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005    if manifest_key.is_empty() {
2006        return Err(Error::ParseError(
2007            "result document manifestKey is empty".to_string(),
2008        ));
2009    }
2010    let payload_sha256 = result_doc
2011        .payload_sha256
2012        .clone()
2013        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014    if payload_sha256.is_empty() {
2015        return Err(Error::ParseError(
2016            "result document payloadSha256 is empty".to_string(),
2017        ));
2018    }
2019    info!(
2020        restore = %restore.name_any(),
2021        backup_id = %expected_backup_id,
2022        manifest_key = %manifest_key,
2023        "safety backup result verified"
2024    );
2025    Ok(VerifiedSafetyBackup {
2026        manifest_key,
2027        payload_sha256,
2028    })
2029}
2030
2031struct VerifiedSourcePrepResult {
2032    #[allow(dead_code)]
2033    manifest_key: String,
2034    payload_sha256: String,
2035}
2036
2037async fn read_source_prep_result(
2038    restore: &KanidmRestore,
2039    ctx: &RestoreContext,
2040    job_name: &str,
2041    expected_backup_id: &str,
2042) -> Result<VerifiedSourcePrepResult> {
2043    let ns = restore.namespace().unwrap();
2044    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2045    let pod_list = pods
2046        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2047        .await
2048        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2049    let pod = pod_list.items.first().ok_or_else(|| {
2050        Error::MissingData(format!("no pod found for source prep job {job_name}"))
2051    })?;
2052    let pod_name = pod.name_any();
2053    let container_status = pod
2054        .status
2055        .as_ref()
2056        .and_then(|s| s.container_statuses.as_ref())
2057        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2058        .ok_or_else(|| {
2059            Error::MissingData(format!(
2060                "source-prep container status not found in pod {pod_name}"
2061            ))
2062        })?;
2063    let termination_message = container_status
2064        .state
2065        .as_ref()
2066        .and_then(|state| state.terminated.as_ref())
2067        .and_then(|t| t.message.as_ref())
2068        .ok_or_else(|| {
2069            Error::MissingData(format!(
2070                "termination message not found in pod {pod_name}; result transport failed"
2071            ))
2072        })?;
2073    if termination_message.is_empty() {
2074        return Err(Error::MissingData(format!(
2075            "termination message is empty in pod {pod_name}; result document was not written"
2076        )));
2077    }
2078    let result_doc = parse_result_document(termination_message).map_err(|e| {
2079        Error::ParseError(format!(
2080            "failed to parse result document from pod {pod_name}: {e}"
2081        ))
2082    })?;
2083    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2084        return Err(Error::ParseError(format!(
2085            "unsupported result document version: {}",
2086            result_doc.api_version
2087        )));
2088    }
2089    if result_doc.kind != "ResultDocument" {
2090        return Err(Error::ParseError(format!(
2091            "invalid result document kind: {}",
2092            result_doc.kind
2093        )));
2094    }
2095    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2096        return Err(Error::ParseError(format!(
2097            "result document operation mismatch: expected '{}', got '{}'",
2098            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2099        )));
2100    }
2101    if !result_doc.success {
2102        return Err(Error::ParseError(format!(
2103            "source prep reported failure with exit code {:?}",
2104            result_doc.exit_code
2105        )));
2106    }
2107    if result_doc.exit_code != ExitCode::Success {
2108        return Err(Error::ParseError(format!(
2109            "source prep exit code is not Success: {:?}",
2110            result_doc.exit_code
2111        )));
2112    }
2113    let doc_backup_id = result_doc
2114        .backup_id
2115        .as_deref()
2116        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2117    if doc_backup_id != expected_backup_id {
2118        return Err(Error::ParseError(format!(
2119            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2120        )));
2121    }
2122    let manifest_key = result_doc
2123        .manifest_key
2124        .clone()
2125        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2126    if manifest_key.is_empty() {
2127        return Err(Error::ParseError(
2128            "result document manifestKey is empty".to_string(),
2129        ));
2130    }
2131    let payload_sha256 = result_doc
2132        .payload_sha256
2133        .clone()
2134        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2135    if payload_sha256.is_empty() {
2136        return Err(Error::ParseError(
2137            "result document payloadSha256 is empty".to_string(),
2138        ));
2139    }
2140    info!(
2141        restore = %restore.name_any(),
2142        backup_id = %expected_backup_id,
2143        manifest_key = %manifest_key,
2144        payload_sha256 = %payload_sha256,
2145        "source prep result verified"
2146    );
2147    Ok(VerifiedSourcePrepResult {
2148        manifest_key,
2149        payload_sha256,
2150    })
2151}
2152
2153fn source_job_pod_security_context(
2154    target: &Kanidm,
2155) -> k8s_openapi::api::core::v1::PodSecurityContext {
2156    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2157    security_context.seccomp_profile = Some(SeccompProfile {
2158        type_: "RuntimeDefault".to_string(),
2159        ..Default::default()
2160    });
2161    security_context
2162}
2163
2164fn hardened_security_context() -> SecurityContext {
2165    SecurityContext {
2166        allow_privilege_escalation: Some(false),
2167        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2168            drop: Some(vec!["ALL".to_string()]),
2169            ..Default::default()
2170        }),
2171        read_only_root_filesystem: Some(true),
2172        run_as_non_root: Some(true),
2173        seccomp_profile: Some(SeccompProfile {
2174            type_: "RuntimeDefault".to_string(),
2175            ..Default::default()
2176        }),
2177        ..Default::default()
2178    }
2179}
2180
2181fn kanidm_job_security_context() -> SecurityContext {
2182    SecurityContext {
2183        allow_privilege_escalation: Some(false),
2184        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2185            drop: Some(vec!["ALL".to_string()]),
2186            ..Default::default()
2187        }),
2188        seccomp_profile: Some(SeccompProfile {
2189            type_: "RuntimeDefault".to_string(),
2190            ..Default::default()
2191        }),
2192        ..Default::default()
2193    }
2194}
2195
2196fn job_resource_requirements() -> ResourceRequirements {
2197    ResourceRequirements {
2198        requests: Some(std::collections::BTreeMap::from([
2199            ("cpu".to_string(), Quantity("100m".to_string())),
2200            ("memory".to_string(), Quantity("128Mi".to_string())),
2201        ])),
2202        limits: Some(std::collections::BTreeMap::from([
2203            ("cpu".to_string(), Quantity("2".to_string())),
2204            ("memory".to_string(), Quantity("2Gi".to_string())),
2205        ])),
2206        ..Default::default()
2207    }
2208}
2209
2210async fn ensure_safety_backup_job(
2211    restore: &KanidmRestore,
2212    target: &Kanidm,
2213    ctx: &RestoreContext,
2214    name: &str,
2215    backup_id: &str,
2216    manifest_key: &str,
2217) -> Result<()> {
2218    let ns = restore.namespace().unwrap();
2219    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2220    if api
2221        .get_opt(name)
2222        .await
2223        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2224        .is_some()
2225    {
2226        return Ok(());
2227    }
2228    let safety = restore
2229        .spec
2230        .safety_backup
2231        .as_ref()
2232        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2233    let repo_ref = safety
2234        .repository_ref
2235        .as_ref()
2236        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2237    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2238        .get(&repo_ref.name)
2239        .await
2240        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2241    let _kanidm_uid = &restore.spec.target_ref.uid;
2242    let endpoint = &repo.spec.s3.endpoint;
2243    let region = &repo.spec.s3.region;
2244    let operation_doc = build_safety_upload_operation_doc(
2245        restore,
2246        target,
2247        backup_id,
2248        manifest_key,
2249        &repo.spec.s3.bucket,
2250        &repo.spec.s3.prefix,
2251        endpoint,
2252        region,
2253        repo.spec.s3.force_path_style,
2254        repo.spec.s3.insecure,
2255        repo.spec
2256            .s3
2257            .ca_bundle_ref
2258            .as_ref()
2259            .map(|_| ca_bundle_path())
2260            .as_deref(),
2261        repo.spec.encryption.as_ref(),
2262    )?;
2263    let operation_cm_name = format!("{name}-op");
2264    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2265    let auth_method = &repo.spec.authentication.writer;
2266    let data_mover_image = data_mover_image();
2267    let init_command = vec![
2268        "kanidmd".to_string(),
2269        "database".to_string(),
2270        "backup".to_string(),
2271        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2272    ];
2273    let upload_script = format!(
2274        r#"set -eu
2275/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2276RESULT_FILE="/run/kaniop-result/result.json"
2277TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2278if [ ! -f "$RESULT_FILE" ]; then
2279  echo "ERROR: result document not found at $RESULT_FILE"
2280  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2281  exit 1
2282fi
2283if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2284  echo "ERROR: safety backup upload reported failure"
2285  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2286  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2287  exit 1
2288fi
2289dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2290echo "safety backup upload completed successfully"
2291"#
2292    );
2293    let job = Job {
2294        metadata: ObjectMeta {
2295            name: Some(name.to_string()),
2296            namespace: Some(ns.clone()),
2297            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2298            ..Default::default()
2299        },
2300        spec: Some(JobSpec {
2301            backoff_limit: Some(0),
2302            template: PodTemplateSpec {
2303                metadata: Some(ObjectMeta::default()),
2304                spec: Some(PodSpec {
2305                    automount_service_account_token: Some(false),
2306                    restart_policy: Some("Never".to_string()),
2307                    security_context: Some(source_job_pod_security_context(target)),
2308                    init_containers: Some(vec![Container {
2309                        name: "safety-backup".to_string(),
2310                        image: Some(restore.spec.restore_image.clone()),
2311                        command: Some(init_command),
2312                        security_context: Some(SecurityContext {
2313                            allow_privilege_escalation: Some(false),
2314                            capabilities: Some(Capabilities {
2315                                drop: Some(vec!["ALL".to_string()]),
2316                                ..Default::default()
2317                            }),
2318                            read_only_root_filesystem: Some(true),
2319                            seccomp_profile: Some(SeccompProfile {
2320                                type_: "RuntimeDefault".to_string(),
2321                                ..Default::default()
2322                            }),
2323                            ..Default::default()
2324                        }),
2325                        resources: Some(job_resource_requirements()),
2326                        env: Some(build_kanidm_db_env_vars(target)),
2327                        volume_mounts: Some(vec![
2328                            VolumeMount {
2329                                name: DATA_VOLUME.to_string(),
2330                                mount_path: DATA_PATH.to_string(),
2331                                read_only: Some(false),
2332                                ..Default::default()
2333                            },
2334                            VolumeMount {
2335                                name: TLS_VOLUME.to_string(),
2336                                mount_path: TLS_PATH.to_string(),
2337                                read_only: Some(true),
2338                                ..Default::default()
2339                            },
2340                            VolumeMount {
2341                                name: SHARED_VOLUME.to_string(),
2342                                mount_path: SHARED_VOL_PATH.to_string(),
2343                                ..Default::default()
2344                            },
2345                        ]),
2346                        ..Default::default()
2347                    }]),
2348                    containers: vec![Container {
2349                        name: "safety-upload".to_string(),
2350                        image: Some(data_mover_image),
2351                        command: Some(vec!["/bin/sh".to_string()]),
2352                        args: Some(vec!["-c".to_string(), upload_script]),
2353                        security_context: Some(hardened_security_context()),
2354                        resources: Some(job_resource_requirements()),
2355                        env: {
2356                            let mut env_vars =
2357                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2358                            env_vars.push(EnvVar {
2359                                name: "RUST_LOG".to_string(),
2360                                value: Some("info".to_string()),
2361                                ..Default::default()
2362                            });
2363                            if repo.spec.s3.ca_bundle_ref.is_some() {
2364                                env_vars.push(ca_bundle_env_var());
2365                            }
2366                            env_vars.extend(build_encryption_env_vars(
2367                                repo.spec
2368                                    .encryption
2369                                    .as_ref()
2370                                    .and_then(|e| e.key_ref.as_ref()),
2371                            ));
2372                            Some(env_vars)
2373                        },
2374                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2375                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2376                        volume_mounts: {
2377                            let mut mounts = vec![
2378                                VolumeMount {
2379                                    name: SHARED_VOLUME.to_string(),
2380                                    mount_path: SHARED_VOL_PATH.to_string(),
2381                                    read_only: Some(true),
2382                                    ..Default::default()
2383                                },
2384                                VolumeMount {
2385                                    name: "operation".to_string(),
2386                                    mount_path: "/run/kaniop".to_string(),
2387                                    read_only: Some(true),
2388                                    ..Default::default()
2389                                },
2390                                VolumeMount {
2391                                    name: "result".to_string(),
2392                                    mount_path: "/run/kaniop-result".to_string(),
2393                                    ..Default::default()
2394                                },
2395                            ];
2396                            mounts.extend(build_auth_volume_mounts(auth_method));
2397                            if repo.spec.s3.ca_bundle_ref.is_some() {
2398                                mounts.push(build_ca_bundle_volume_mount());
2399                            }
2400                            Some(mounts)
2401                        },
2402                        ..Default::default()
2403                    }],
2404                    volumes: {
2405                        let mut vols = vec![
2406                            Volume {
2407                                name: DATA_VOLUME.to_string(),
2408                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2409                                    claim_name: primary_pvc_name(target)?,
2410                                    read_only: Some(false),
2411                                }),
2412                                ..Default::default()
2413                            },
2414                            Volume {
2415                                name: TLS_VOLUME.to_string(),
2416                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2417                                    secret_name: Some(target.effective_tls_secret_name()),
2418                                    ..Default::default()
2419                                }),
2420                                ..Default::default()
2421                            },
2422                            Volume {
2423                                name: SHARED_VOLUME.to_string(),
2424                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2425                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2426                                    ..Default::default()
2427                                }),
2428                                ..Default::default()
2429                            },
2430                            Volume {
2431                                name: "operation".to_string(),
2432                                config_map: Some(ConfigMapVolumeSource {
2433                                    name: operation_cm_name,
2434                                    ..Default::default()
2435                                }),
2436                                ..Default::default()
2437                            },
2438                            Volume {
2439                                name: "result".to_string(),
2440                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2441                                    size_limit: Some(Quantity("16Mi".to_string())),
2442                                    ..Default::default()
2443                                }),
2444                                ..Default::default()
2445                            },
2446                        ];
2447                        vols.extend(build_auth_volumes(auth_method));
2448                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2449                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2450                        }
2451                        Some(vols)
2452                    },
2453                    ..Default::default()
2454                }),
2455            },
2456            ..Default::default()
2457        }),
2458        ..Default::default()
2459    };
2460    api.create(&PostParams::default(), &job)
2461        .await
2462        .map(|_| ())
2463        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2464}
2465
2466async fn ensure_source_prep_job(
2467    restore: &KanidmRestore,
2468    target: &Kanidm,
2469    ctx: &RestoreContext,
2470    name: &str,
2471) -> Result<()> {
2472    let ns = restore.namespace().unwrap();
2473    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2474    if api
2475        .get_opt(name)
2476        .await
2477        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2478        .is_some()
2479    {
2480        return Ok(());
2481    }
2482    let backup_name = restore
2483        .spec
2484        .source
2485        .backup_ref
2486        .as_ref()
2487        .map(|b| b.name.clone())
2488        .unwrap_or_default();
2489    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2490        .get(&backup_name)
2491        .await
2492        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2493    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2494        .get(&backup.spec.repository_ref.name)
2495        .await
2496        .map_err(|e| {
2497            Error::kube_error(
2498                "get",
2499                "KanidmBackupRepository",
2500                &ns,
2501                &backup.spec.repository_ref.name,
2502                e,
2503            )
2504        })?;
2505    let endpoint = &repo.spec.s3.endpoint;
2506    let region = &repo.spec.s3.region;
2507    let operation_doc = build_download_operation_doc(
2508        target,
2509        &backup.spec.manifest_key,
2510        &backup.spec.backup_id,
2511        &backup.spec.kanidm_ref.uid,
2512        &repo.spec.s3.bucket,
2513        &repo.spec.s3.prefix,
2514        endpoint,
2515        region,
2516        repo.spec.s3.force_path_style,
2517        repo.spec.s3.insecure,
2518        repo.spec
2519            .s3
2520            .ca_bundle_ref
2521            .as_ref()
2522            .map(|_| ca_bundle_path())
2523            .as_deref(),
2524        repo.spec.encryption.as_ref(),
2525    );
2526    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2527    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2528    let auth_method = &repo.spec.authentication.reader;
2529    let data_mover_image = data_mover_image();
2530    let download_script = format!(
2531        r#"set -eu
2532/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2533RESULT_FILE="/run/kaniop-result/result.json"
2534TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2535if [ ! -f "$RESULT_FILE" ]; then
2536  echo "ERROR: result document not found at $RESULT_FILE"
2537  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2538  exit 1
2539fi
2540if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2541  echo "ERROR: source preparation download reported failure"
2542  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2543  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2544  exit 1
2545fi
2546dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2547echo "source preparation download completed successfully"
2548"#
2549    );
2550    let job = Job {
2551        metadata: ObjectMeta {
2552            name: Some(name.to_string()),
2553            namespace: Some(ns.clone()),
2554            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2555            ..Default::default()
2556        },
2557        spec: Some(JobSpec {
2558            backoff_limit: Some(0),
2559            template: PodTemplateSpec {
2560                metadata: Some(ObjectMeta::default()),
2561                spec: Some(PodSpec {
2562                    automount_service_account_token: Some(false),
2563                    restart_policy: Some("Never".to_string()),
2564                    security_context: Some(source_job_pod_security_context(target)),
2565                    containers: vec![Container {
2566                        name: "source-prep".to_string(),
2567                        image: Some(data_mover_image),
2568                        command: Some(vec!["/bin/sh".to_string()]),
2569                        args: Some(vec!["-c".to_string(), download_script]),
2570                        security_context: Some(hardened_security_context()),
2571                        resources: Some(job_resource_requirements()),
2572                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2573                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2574                        env: {
2575                            let mut env_vars = build_auth_env_vars(
2576                                auth_method,
2577                                &backup.spec.repository_ref.name,
2578                                AuthRole::Reader,
2579                            );
2580                            env_vars.push(EnvVar {
2581                                name: "RUST_LOG".to_string(),
2582                                value: Some("info".to_string()),
2583                                ..Default::default()
2584                            });
2585                            if repo.spec.s3.ca_bundle_ref.is_some() {
2586                                env_vars.push(ca_bundle_env_var());
2587                            }
2588                            env_vars.extend(build_encryption_env_vars(
2589                                repo.spec
2590                                    .encryption
2591                                    .as_ref()
2592                                    .and_then(|e| e.key_ref.as_ref()),
2593                            ));
2594                            Some(env_vars)
2595                        },
2596                        volume_mounts: {
2597                            let mut mounts = vec![
2598                                VolumeMount {
2599                                    name: DATA_VOLUME.to_string(),
2600                                    mount_path: DATA_PATH.to_string(),
2601                                    ..Default::default()
2602                                },
2603                                VolumeMount {
2604                                    name: "operation".to_string(),
2605                                    mount_path: "/run/kaniop".to_string(),
2606                                    read_only: Some(true),
2607                                    ..Default::default()
2608                                },
2609                                VolumeMount {
2610                                    name: "result".to_string(),
2611                                    mount_path: "/run/kaniop-result".to_string(),
2612                                    ..Default::default()
2613                                },
2614                            ];
2615                            mounts.extend(build_auth_volume_mounts(auth_method));
2616                            if repo.spec.s3.ca_bundle_ref.is_some() {
2617                                mounts.push(build_ca_bundle_volume_mount());
2618                            }
2619                            Some(mounts)
2620                        },
2621                        ..Default::default()
2622                    }],
2623                    volumes: {
2624                        let mut vols = vec![
2625                            Volume {
2626                                name: DATA_VOLUME.to_string(),
2627                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2628                                    claim_name: primary_pvc_name(target)?,
2629                                    read_only: Some(false),
2630                                }),
2631                                ..Default::default()
2632                            },
2633                            Volume {
2634                                name: "operation".to_string(),
2635                                config_map: Some(ConfigMapVolumeSource {
2636                                    name: operation_cm_name,
2637                                    ..Default::default()
2638                                }),
2639                                ..Default::default()
2640                            },
2641                            Volume {
2642                                name: "result".to_string(),
2643                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2644                                    size_limit: Some(Quantity("16Mi".to_string())),
2645                                    ..Default::default()
2646                                }),
2647                                ..Default::default()
2648                            },
2649                        ];
2650                        vols.extend(build_auth_volumes(auth_method));
2651                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2652                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2653                        }
2654                        Some(vols)
2655                    },
2656                    ..Default::default()
2657                }),
2658            },
2659            ..Default::default()
2660        }),
2661        ..Default::default()
2662    };
2663    api.create(&PostParams::default(), &job)
2664        .await
2665        .map(|_| ())
2666        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2667}
2668
2669#[allow(clippy::too_many_arguments)]
2670fn build_safety_upload_operation_doc(
2671    restore: &KanidmRestore,
2672    target: &Kanidm,
2673    backup_id: &str,
2674    manifest_key: &str,
2675    bucket: &str,
2676    prefix: &str,
2677    endpoint: &str,
2678    region: &str,
2679    force_path_style: bool,
2680    insecure: bool,
2681    ca_bundle_path: Option<&str>,
2682    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2683) -> Result<String> {
2684    let enc_mode =
2685        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2686    let enc_key_id = encryption
2687        .and_then(|e| e.key_id.as_ref())
2688        .map(|s| serde_json::Value::String(s.clone()));
2689    let op = UploadOperation {
2690        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2691        bucket: bucket.to_string(),
2692        prefix: prefix.to_string(),
2693        endpoint: endpoint.to_string(),
2694        region: region.to_string(),
2695        force_path_style,
2696        insecure,
2697        ca_bundle_path: ca_bundle_path.map(str::to_string),
2698        backup_id: backup_id.to_string(),
2699        // `UploadOperation.namespace_uid` carries the Kubernetes namespace NAME (`restore.namespace()`),
2700        // not the Namespace object UID: it selects the `tenants/<x>` keyspace segment and matches the
2701        // `namespaceUid` wire field name, which is frozen for compatibility (issue #1084).
2702        namespace_uid: restore.namespace().unwrap_or_default(),
2703        kanidm_uid: restore.spec.target_ref.uid.clone(),
2704        kanidm_name: restore.spec.target_ref.name.clone(),
2705        domain: target.spec.domain.clone(),
2706        kanidm_version: target
2707            .status
2708            .as_ref()
2709            .and_then(|s| s.version.as_ref())
2710            .map(|v| v.image_tag.clone())
2711            .unwrap_or_default(),
2712        image_digest: None,
2713        consistency: "kanidm-offline".to_string(),
2714        reason: "restore-safety".to_string(),
2715        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2716        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2717        result_path: "/run/kaniop-result/result.json".to_string(),
2718        max_concurrent_parts: 4,
2719        max_retries: 3,
2720    };
2721    let _ = manifest_key;
2722    let doc = OperationDocument {
2723        api_version: OPERATION_DOC_VERSION.to_string(),
2724        kind: "OperationDocument".to_string(),
2725        spec: OperationSpec::Upload(op),
2726    };
2727    serde_json::to_string(&doc)
2728        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2729}
2730
2731async fn validate_backup_ref(
2732    restore: &KanidmRestore,
2733    target: &Kanidm,
2734    ctx: &RestoreContext,
2735) -> Result<Option<DisasterRecoverySource>> {
2736    let backup_ref = restore
2737        .spec
2738        .source
2739        .backup_ref
2740        .as_ref()
2741        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2742    let ns = restore.namespace().unwrap();
2743    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2744        .get(&backup_ref.name)
2745        .await
2746        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2747    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2748        return Err(Error::MissingData(format!(
2749            "KanidmBackup '{}' is not Ready (phase: {:?})",
2750            backup_ref.name,
2751            backup.status.as_ref().map(|s| &s.phase)
2752        )));
2753    }
2754    let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2755        || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2756    let disaster_recovery_source = if source_identity_mismatch {
2757        if !disaster_recovery_requested(restore) {
2758            return Err(Error::MissingData(format!(
2759                "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2760                backup.spec.kanidm_ref.name,
2761                backup.spec.kanidm_ref.uid,
2762                restore.spec.target_ref.name,
2763                restore.spec.target_ref.uid,
2764                DISASTER_RECOVERY_ANNOTATION,
2765                BREAK_GLASS_REASON_ANNOTATION,
2766                BREAK_GLASS_APPROVED_BY_ANNOTATION,
2767            )));
2768        }
2769        validate_disaster_recovery_approval(restore)?;
2770        Some(DisasterRecoverySource {
2771            name: backup.spec.kanidm_ref.name.clone(),
2772            uid: backup.spec.kanidm_ref.uid.clone(),
2773        })
2774    } else {
2775        None
2776    };
2777    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2778        .get(&backup.spec.repository_ref.name)
2779        .await
2780        .map_err(|e| {
2781            Error::kube_error(
2782                "get",
2783                "KanidmBackupRepository",
2784                &ns,
2785                &backup.spec.repository_ref.name,
2786                e,
2787            )
2788        })?;
2789    if !has_accepted_condition(
2790        &repo
2791            .status
2792            .as_ref()
2793            .map(|s| &s.conditions)
2794            .cloned()
2795            .unwrap_or_default(),
2796    ) {
2797        return Err(Error::MissingData(format!(
2798            "KanidmBackupRepository '{}' configuration has not been accepted",
2799            backup.spec.repository_ref.name
2800        )));
2801    }
2802    validate_backup_compatibility(restore, target, &backup)?;
2803    Ok(disaster_recovery_source)
2804}
2805
2806fn has_accepted_condition(conditions: &[Condition]) -> bool {
2807    conditions
2808        .iter()
2809        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2810}
2811
2812fn validate_backup_compatibility(
2813    restore: &KanidmRestore,
2814    target: &Kanidm,
2815    backup: &KanidmBackup,
2816) -> Result<()> {
2817    let status = backup.status.as_ref();
2818    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2819        let target_version = target
2820            .status
2821            .as_ref()
2822            .and_then(|s| s.version.as_ref())
2823            .map(|v| v.image_tag.as_str());
2824        if let Some(tv) = target_version {
2825            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2826                return Err(Error::MissingData(format!(
2827                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2828                )));
2829            }
2830        }
2831    }
2832    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2833        if !backup_digest.is_empty()
2834            && restore.spec.restore_image.contains('@')
2835            && !restore.spec.restore_image.contains(backup_digest)
2836        {
2837            return Err(Error::MissingData(format!(
2838                "restore image digest does not match backup image digest '{backup_digest}'"
2839            )));
2840        }
2841    }
2842    Ok(())
2843}
2844
2845#[allow(clippy::too_many_arguments)]
2846fn build_download_operation_doc(
2847    target: &Kanidm,
2848    manifest_key: &str,
2849    expected_backup_id: &str,
2850    expected_source_uid: &str,
2851    bucket: &str,
2852    prefix: &str,
2853    endpoint: &str,
2854    region: &str,
2855    force_path_style: bool,
2856    insecure: bool,
2857    ca_bundle_path: Option<&str>,
2858    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2859) -> String {
2860    let enc_mode =
2861        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2862    let enc_key_id = encryption
2863        .and_then(|e| e.key_id.as_ref())
2864        .map(|s| serde_json::Value::String(s.clone()));
2865    serde_json::json!({
2866        "apiVersion": "backup.kaniop.rs/v1alpha1",
2867        "kind": "OperationDocument",
2868        "operation": "download",
2869        "manifestKey": manifest_key,
2870        "bucket": bucket,
2871        "prefix": prefix,
2872        "endpoint": endpoint,
2873        "region": region,
2874        "forcePathStyle": force_path_style,
2875        "insecure": insecure,
2876        "caBundlePath": ca_bundle_path,
2877        "expectedBackupId": expected_backup_id,
2878        "expectedKanidmUid": expected_source_uid,
2879        "expectedDomain": target.spec.domain,
2880        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2881        "resultPath": "/run/kaniop-result/result.json",
2882        "maxRetries": 3,
2883        "encryptionMode": enc_mode,
2884        "encryptionKeyId": enc_key_id,
2885    })
2886    .to_string()
2887}
2888
2889async fn ensure_operation_configmap(
2890    restore: &KanidmRestore,
2891    name: &str,
2892    data: &str,
2893    ns: &str,
2894    ctx: &RestoreContext,
2895) -> Result<()> {
2896    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2897    if api
2898        .get_opt(name)
2899        .await
2900        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2901        .is_some()
2902    {
2903        return Ok(());
2904    }
2905    let cm = ConfigMap {
2906        metadata: ObjectMeta {
2907            name: Some(name.to_string()),
2908            namespace: Some(ns.to_string()),
2909            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2910            ..Default::default()
2911        },
2912        data: Some(std::collections::BTreeMap::from([(
2913            "operation.json".to_string(),
2914            data.to_string(),
2915        )])),
2916        ..Default::default()
2917    };
2918    api.create(&PostParams::default(), &cm)
2919        .await
2920        .map(|_| ())
2921        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926    use super::{
2927        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2928        CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2929        DISASTER_RECOVERY_ANNOTATION,
2930        KanidmRestore, KanidmRestorePhase,
2931        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2932        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2933        disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2934        is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2935        restore_job_name, safe_basename, validate_disaster_recovery_approval,
2936        validate_safety_backup_config, validate_source, verify_job_name,
2937    };
2938    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2939    use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2940    use k8s_openapi::jiff::Timestamp;
2941    use kube::api::ObjectMeta as ApiObjectMeta;
2942    use std::collections::BTreeMap;
2943
2944    fn make_restore(
2945        source: KanidmRestoreSource,
2946        safety: Option<SafetyBackupConfig>,
2947    ) -> KanidmRestore {
2948        KanidmRestore {
2949            metadata: ApiObjectMeta {
2950                name: Some("test-restore".to_string()),
2951                namespace: Some("default".to_string()),
2952                uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2953                ..Default::default()
2954            },
2955            spec: KanidmRestoreSpec {
2956                target_ref: KanidmRestoreTargetRef {
2957                    name: "corp-idm".to_string(),
2958                    uid: "test-uid-123".to_string(),
2959                },
2960                source,
2961                restore_image: "kanidm/server@sha256:abc".to_string(),
2962                safety_backup: safety,
2963            },
2964            status: None,
2965        }
2966    }
2967
2968    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2969        super::super::crd::Kanidm {
2970            status: Some(super::super::crd::KanidmStatus {
2971                available_replicas: 1,
2972                replicas: 1,
2973                unavailable_replicas: 0,
2974                updated_replicas: 1,
2975                replica_statuses: vec![],
2976                replica_column: "1/1".to_string(),
2977                secret_name: None,
2978                version: Some(super::super::crd::KanidmVersionStatus {
2979                    image_tag: image_tag.to_string(),
2980                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2981                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2982                }),
2983                domain_appearance_image: None,
2984                mail_sender: None,
2985                conditions: None,
2986            }),
2987            ..Default::default()
2988        }
2989    }
2990
2991    #[test]
2992    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2993        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2994    }
2995
2996    #[test]
2997    fn completed_restore_sets_ready_condition() {
2998        let mut status = KanidmRestoreStatus {
2999            phase: KanidmRestorePhase::Completed,
3000            ..Default::default()
3001        };
3002        super::update_restore_conditions(&mut status, Some(7));
3003        assert!(status.conditions.iter().any(|condition| {
3004            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3005        }));
3006        assert!(!status.conditions.iter().any(|condition| {
3007            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3008        }));
3009    }
3010
3011    #[test]
3012    fn disaster_recovery_condition_survives_phase_updates() {
3013        let dr_condition = Condition {
3014            type_: CONDITION_DISASTER_RECOVERY.to_string(),
3015            status: CONDITION_TRUE.to_string(),
3016            reason: "ApprovedSourceIdentityMismatch".to_string(),
3017            message: "approved DR".to_string(),
3018            last_transition_time: Time(Timestamp::now()),
3019            observed_generation: Some(7),
3020        };
3021        let mut status = KanidmRestoreStatus {
3022            phase: KanidmRestorePhase::Quiescing,
3023            conditions: vec![dr_condition.clone()],
3024            ..Default::default()
3025        };
3026
3027        super::update_restore_conditions(&mut status, Some(8));
3028
3029        assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3030    }
3031
3032    #[test]
3033    fn failed_restore_sets_failed_condition() {
3034        let mut status = KanidmRestoreStatus {
3035            phase: KanidmRestorePhase::Failed,
3036            message: Some("verification failed".to_string()),
3037            ..Default::default()
3038        };
3039        super::update_restore_conditions(&mut status, Some(8));
3040        assert!(status.conditions.iter().any(|condition| {
3041            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3042        }));
3043    }
3044
3045    #[test]
3046    fn restore_filename_is_confined_to_backup_directory() {
3047        assert!(safe_basename("backup.json.gz"));
3048        assert!(!safe_basename("../backup.json.gz"));
3049        assert!(!safe_basename("a/b"));
3050        assert!(!safe_basename(""));
3051    }
3052
3053    #[test]
3054    fn restore_image_rejects_latest_and_unpinned_names() {
3055        assert!(mutable_image("kanidm/server:latest"));
3056        assert!(mutable_image("kanidm/server"));
3057        assert!(!mutable_image("kanidm/server:1.10.0"));
3058        assert!(!mutable_image("kanidm/server@sha256:abc"));
3059    }
3060
3061    #[test]
3062    fn source_validation_rejects_both_local_and_backup_ref() {
3063        let source = KanidmRestoreSource {
3064            local: Some(super::KanidmRestoreLocalSource {
3065                file_name: "backup.json".to_string(),
3066            }),
3067            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3068                name: "some-backup".to_string(),
3069            }),
3070        };
3071        let restore = make_restore(source, None);
3072        assert!(validate_source(&restore).is_err());
3073    }
3074
3075    #[test]
3076    fn source_validation_rejects_empty_source() {
3077        let source = KanidmRestoreSource {
3078            local: None,
3079            backup_ref: None,
3080        };
3081        let restore = make_restore(source, None);
3082        assert!(validate_source(&restore).is_err());
3083    }
3084
3085    #[test]
3086    fn source_validation_accepts_local_source() {
3087        let source = KanidmRestoreSource {
3088            local: Some(super::KanidmRestoreLocalSource {
3089                file_name: "backup.json".to_string(),
3090            }),
3091            backup_ref: None,
3092        };
3093        let restore = make_restore(source, None);
3094        assert!(validate_source(&restore).is_ok());
3095    }
3096
3097    #[test]
3098    fn source_validation_accepts_backup_ref_source() {
3099        let source = KanidmRestoreSource {
3100            local: None,
3101            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3102                name: "corp-idm-019c7c76".to_string(),
3103            }),
3104        };
3105        let restore = make_restore(source, None);
3106        assert!(validate_source(&restore).is_ok());
3107    }
3108
3109    #[test]
3110    fn source_validation_rejects_empty_backup_ref_name() {
3111        let source = KanidmRestoreSource {
3112            local: None,
3113            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3114                name: "".to_string(),
3115            }),
3116        };
3117        let restore = make_restore(source, None);
3118        assert!(validate_source(&restore).is_err());
3119    }
3120
3121    #[test]
3122    fn is_remote_source_true_for_backup_ref() {
3123        let source = KanidmRestoreSource {
3124            local: None,
3125            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3126                name: "backup-1".to_string(),
3127            }),
3128        };
3129        let restore = make_restore(source, None);
3130        assert!(is_remote_source(&restore));
3131    }
3132
3133    #[test]
3134    fn is_remote_source_false_for_local() {
3135        let source = KanidmRestoreSource {
3136            local: Some(super::KanidmRestoreLocalSource {
3137                file_name: "backup.json".to_string(),
3138            }),
3139            backup_ref: None,
3140        };
3141        let restore = make_restore(source, None);
3142        assert!(!is_remote_source(&restore));
3143    }
3144
3145    #[test]
3146    fn requires_safety_backup_default_true() {
3147        let source = KanidmRestoreSource {
3148            local: None,
3149            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3150                name: "backup-1".to_string(),
3151            }),
3152        };
3153        let restore = make_restore(source, None);
3154        assert!(requires_safety_backup(&restore));
3155    }
3156
3157    #[test]
3158    fn requires_safety_backup_false_when_skip() {
3159        let source = KanidmRestoreSource {
3160            local: None,
3161            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3162                name: "backup-1".to_string(),
3163            }),
3164        };
3165        let safety = SafetyBackupConfig {
3166            repository_ref: Some(SafetyBackupRepositoryRef {
3167                name: "offsite".to_string(),
3168            }),
3169            skip: true,
3170        };
3171        let restore = make_restore(source, Some(safety));
3172        assert!(!requires_safety_backup(&restore));
3173    }
3174
3175    #[test]
3176    fn break_glass_validation_rejects_skip_without_reason() {
3177        let source = KanidmRestoreSource {
3178            local: None,
3179            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3180                name: "backup-1".to_string(),
3181            }),
3182        };
3183        let safety = SafetyBackupConfig {
3184            repository_ref: Some(SafetyBackupRepositoryRef {
3185                name: "offsite".to_string(),
3186            }),
3187            skip: true,
3188        };
3189        let mut restore = make_restore(source, Some(safety));
3190        restore.metadata.annotations = Some(BTreeMap::from([(
3191            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3192            "commander@example.com".to_string(),
3193        )]));
3194        assert!(validate_safety_backup_config(&restore).is_err());
3195    }
3196
3197    #[test]
3198    fn break_glass_validation_rejects_skip_without_approver() {
3199        let source = KanidmRestoreSource {
3200            local: None,
3201            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3202                name: "backup-1".to_string(),
3203            }),
3204        };
3205        let safety = SafetyBackupConfig {
3206            repository_ref: Some(SafetyBackupRepositoryRef {
3207                name: "offsite".to_string(),
3208            }),
3209            skip: true,
3210        };
3211        let mut restore = make_restore(source, Some(safety));
3212        restore.metadata.annotations = Some(BTreeMap::from([(
3213            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3214            "PVC is unreadable".to_string(),
3215        )]));
3216        assert!(validate_safety_backup_config(&restore).is_err());
3217    }
3218
3219    #[test]
3220    fn break_glass_validation_accepts_valid_annotations() {
3221        let source = KanidmRestoreSource {
3222            local: None,
3223            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3224                name: "backup-1".to_string(),
3225            }),
3226        };
3227        let safety = SafetyBackupConfig {
3228            repository_ref: Some(SafetyBackupRepositoryRef {
3229                name: "offsite".to_string(),
3230            }),
3231            skip: true,
3232        };
3233        let mut restore = make_restore(source, Some(safety));
3234        restore.metadata.annotations = Some(BTreeMap::from([
3235            (
3236                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3237                "PVC is unreadable".to_string(),
3238            ),
3239            (
3240                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3241                "commander@example.com".to_string(),
3242            ),
3243        ]));
3244        assert!(validate_safety_backup_config(&restore).is_ok());
3245    }
3246
3247    #[test]
3248    fn disaster_recovery_requires_explicit_approval_annotations() {
3249        let source = KanidmRestoreSource {
3250            local: None,
3251            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3252                name: "backup-1".to_string(),
3253            }),
3254        };
3255        let mut restore = make_restore(source, None);
3256        restore.metadata.annotations = Some(BTreeMap::from([(
3257            DISASTER_RECOVERY_ANNOTATION.to_string(),
3258            "true".to_string(),
3259        )]));
3260        assert!(disaster_recovery_requested(&restore));
3261        assert!(validate_disaster_recovery_approval(&restore).is_err());
3262    }
3263
3264    #[test]
3265    fn disaster_recovery_accepts_reason_and_approver() {
3266        let source = KanidmRestoreSource {
3267            local: None,
3268            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3269                name: "backup-1".to_string(),
3270            }),
3271        };
3272        let mut restore = make_restore(source, None);
3273        restore.metadata.annotations = Some(BTreeMap::from([
3274            (
3275                DISASTER_RECOVERY_ANNOTATION.to_string(),
3276                "true".to_string(),
3277            ),
3278            (
3279                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3280                "original cluster lost".to_string(),
3281            ),
3282            (
3283                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3284                "incident-commander".to_string(),
3285            ),
3286        ]));
3287        assert!(disaster_recovery_requested(&restore));
3288        assert!(validate_disaster_recovery_approval(&restore).is_ok());
3289    }
3290
3291    #[test]
3292    fn remote_source_requires_repository_ref_when_not_skipping() {
3293        let source = KanidmRestoreSource {
3294            local: None,
3295            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3296                name: "backup-1".to_string(),
3297            }),
3298        };
3299        let safety = SafetyBackupConfig {
3300            repository_ref: None,
3301            skip: false,
3302        };
3303        let restore = make_restore(source, Some(safety));
3304        assert!(validate_safety_backup_config(&restore).is_err());
3305    }
3306
3307    #[test]
3308    fn remote_source_accepts_repository_ref() {
3309        let source = KanidmRestoreSource {
3310            local: None,
3311            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3312                name: "backup-1".to_string(),
3313            }),
3314        };
3315        let safety = SafetyBackupConfig {
3316            repository_ref: Some(SafetyBackupRepositoryRef {
3317                name: "offsite".to_string(),
3318            }),
3319            skip: false,
3320        };
3321        let restore = make_restore(source, Some(safety));
3322        assert!(validate_safety_backup_config(&restore).is_ok());
3323    }
3324
3325    #[test]
3326    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3327        let phases = vec![
3328            KanidmRestorePhase::Pending,
3329            KanidmRestorePhase::Validating,
3330            KanidmRestorePhase::Quiescing,
3331            KanidmRestorePhase::SafetyBackup,
3332            KanidmRestorePhase::PreparingSource,
3333            KanidmRestorePhase::RestoringPrimary,
3334            KanidmRestorePhase::Verifying,
3335            KanidmRestorePhase::RebuildingReplicas,
3336            KanidmRestorePhase::Resuming,
3337            KanidmRestorePhase::Completed,
3338        ];
3339        assert_eq!(phases.len(), 10);
3340        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3341        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3342    }
3343
3344    #[test]
3345    fn status_default_has_empty_phase_timestamps() {
3346        let status = KanidmRestoreStatus::default();
3347        assert!(status.phase_timestamps.is_empty());
3348    }
3349
3350    #[test]
3351    fn status_default_has_empty_original_replicas() {
3352        let status = KanidmRestoreStatus::default();
3353        assert!(status.original_replicas.is_empty());
3354    }
3355
3356    #[test]
3357    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3358        let mut target = super::super::crd::Kanidm::default();
3359        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3360            run_as_non_root: Some(false),
3361            run_as_user: Some(389),
3362            run_as_group: Some(389),
3363            fs_group: Some(389),
3364            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3365            supplemental_groups: Some(vec![390, 391]),
3366            ..Default::default()
3367        });
3368
3369        let ctx = super::source_job_pod_security_context(&target);
3370
3371        assert_eq!(ctx.run_as_non_root, Some(false));
3372        assert_eq!(ctx.run_as_user, Some(389));
3373        assert_eq!(ctx.run_as_group, Some(389));
3374        assert_eq!(ctx.fs_group, Some(389));
3375        assert_eq!(
3376            ctx.fs_group_change_policy.as_deref(),
3377            Some("OnRootMismatch")
3378        );
3379        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3380        assert_eq!(
3381            ctx.seccomp_profile
3382                .as_ref()
3383                .map(|profile| profile.type_.as_str()),
3384            Some("RuntimeDefault")
3385        );
3386    }
3387
3388    #[test]
3389    fn hardened_security_context_drops_all_capabilities() {
3390        let ctx = hardened_security_context();
3391        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3392        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3393        assert_eq!(ctx.run_as_non_root, Some(true));
3394        let caps = ctx.capabilities.unwrap();
3395        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3396        assert_eq!(
3397            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3398            Some("RuntimeDefault")
3399        );
3400    }
3401
3402    #[test]
3403    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3404        let ctx = kanidm_job_security_context();
3405        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3406        assert!(ctx.run_as_non_root.is_none());
3407        assert!(ctx.read_only_root_filesystem.is_none());
3408        let caps = ctx.capabilities.unwrap();
3409        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3410        assert_eq!(
3411            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3412            Some("RuntimeDefault")
3413        );
3414    }
3415
3416    #[test]
3417    fn safety_backup_job_name_is_deterministic() {
3418        let source = KanidmRestoreSource {
3419            local: Some(super::KanidmRestoreLocalSource {
3420                file_name: "backup.json".to_string(),
3421            }),
3422            backup_ref: None,
3423        };
3424        let restore = make_restore(source, None);
3425        let name1 = super::safety_backup_job_name(&restore);
3426        let name2 = super::safety_backup_job_name(&restore);
3427        assert_eq!(name1, name2);
3428        assert_eq!(name1, "test-restore-safety-backup");
3429    }
3430
3431    #[test]
3432    fn source_prep_job_name_is_deterministic() {
3433        let source = KanidmRestoreSource {
3434            local: None,
3435            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3436                name: "backup-1".to_string(),
3437            }),
3438        };
3439        let restore = make_restore(source, None);
3440        let name = super::source_prep_job_name(&restore);
3441        assert_eq!(name, "test-restore-source-prep");
3442    }
3443
3444    #[test]
3445    fn break_glass_condition_is_recorded_in_status() {
3446        let mut status = KanidmRestoreStatus::default();
3447        let condition = super::restore_condition(
3448            &status.conditions,
3449            CONDITION_BREAK_GLASS,
3450            CONDITION_TRUE,
3451            "BreakGlassOverride",
3452            "Safety backup skipped",
3453            Some(1),
3454        );
3455        status.conditions.push(condition);
3456        assert!(status.conditions.iter().any(|c| {
3457            c.type_ == CONDITION_BREAK_GLASS
3458                && c.status == CONDITION_TRUE
3459                && c.reason == "BreakGlassOverride"
3460        }));
3461    }
3462
3463    #[test]
3464    fn replica_count_entry_serialization() {
3465        let entry = ReplicaCountEntry {
3466            group: "primary".to_string(),
3467            replicas: 3,
3468        };
3469        let json = serde_json::to_string(&entry).unwrap();
3470        assert!(json.contains("\"group\":\"primary\""));
3471        assert!(json.contains("\"replicas\":3"));
3472    }
3473
3474    #[test]
3475    fn local_source_requires_repository_ref_when_not_skipping() {
3476        let source = KanidmRestoreSource {
3477            local: Some(super::KanidmRestoreLocalSource {
3478                file_name: "backup.json".to_string(),
3479            }),
3480            backup_ref: None,
3481        };
3482        let safety = SafetyBackupConfig {
3483            repository_ref: None,
3484            skip: false,
3485        };
3486        let restore = make_restore(source, Some(safety));
3487        assert!(validate_safety_backup_config(&restore).is_err());
3488    }
3489
3490    #[test]
3491    fn local_source_accepts_repository_ref_when_not_skipping() {
3492        let source = KanidmRestoreSource {
3493            local: Some(super::KanidmRestoreLocalSource {
3494                file_name: "backup.json".to_string(),
3495            }),
3496            backup_ref: None,
3497        };
3498        let safety = SafetyBackupConfig {
3499            repository_ref: Some(SafetyBackupRepositoryRef {
3500                name: "offsite".to_string(),
3501            }),
3502            skip: false,
3503        };
3504        let restore = make_restore(source, Some(safety));
3505        assert!(validate_safety_backup_config(&restore).is_ok());
3506    }
3507
3508    #[test]
3509    fn skip_without_break_glass_annotations_is_rejected() {
3510        let source = KanidmRestoreSource {
3511            local: Some(super::KanidmRestoreLocalSource {
3512                file_name: "backup.json".to_string(),
3513            }),
3514            backup_ref: None,
3515        };
3516        let safety = SafetyBackupConfig {
3517            repository_ref: None,
3518            skip: true,
3519        };
3520        let restore = make_restore(source, Some(safety));
3521        assert!(validate_safety_backup_config(&restore).is_err());
3522    }
3523
3524    #[test]
3525    fn has_accepted_condition_true() {
3526        let conditions = vec![Condition {
3527            type_: "Ready".to_string(),
3528            status: "True".to_string(),
3529            reason: "Accepted".to_string(),
3530            message: String::new(),
3531            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3532                k8s_openapi::jiff::Timestamp::now(),
3533            ),
3534            observed_generation: None,
3535        }];
3536        assert!(has_accepted_condition(&conditions));
3537    }
3538
3539    #[test]
3540    fn has_accepted_condition_false_when_not_accepted_reason() {
3541        let conditions = vec![Condition {
3542            type_: "Ready".to_string(),
3543            status: "True".to_string(),
3544            reason: "SomeOtherReason".to_string(),
3545            message: String::new(),
3546            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3547                k8s_openapi::jiff::Timestamp::now(),
3548            ),
3549            observed_generation: None,
3550        }];
3551        assert!(!has_accepted_condition(&conditions));
3552    }
3553
3554    #[test]
3555    fn has_accepted_condition_false_when_status_false() {
3556        let conditions = vec![Condition {
3557            type_: "Ready".to_string(),
3558            status: "False".to_string(),
3559            reason: "Accepted".to_string(),
3560            message: String::new(),
3561            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3562                k8s_openapi::jiff::Timestamp::now(),
3563            ),
3564            observed_generation: None,
3565        }];
3566        assert!(!has_accepted_condition(&conditions));
3567    }
3568
3569    #[test]
3570    fn has_accepted_condition_empty() {
3571        assert!(!has_accepted_condition(&[]));
3572    }
3573
3574    #[test]
3575    fn safety_backup_upload_operation_doc_is_valid() {
3576        let source = KanidmRestoreSource {
3577            local: None,
3578            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3579                name: "backup-1".to_string(),
3580            }),
3581        };
3582        let mut restore = make_restore(source, None);
3583        restore.metadata.namespace = Some("test-ns".to_string());
3584        let target = super::super::crd::Kanidm::default();
3585        let doc_str = super::build_safety_upload_operation_doc(
3586            &restore,
3587            &target,
3588            "test-backup-id",
3589            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3590            "my-bucket",
3591            "prod",
3592            "https://s3.example.com",
3593            "us-east-1",
3594            false,
3595            false,
3596            None,
3597            None,
3598        )
3599        .unwrap();
3600        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3601        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3602        assert_eq!(parsed["kind"], "OperationDocument");
3603        assert_eq!(parsed["operation"], "upload");
3604        assert_eq!(parsed["bucket"], "my-bucket");
3605        assert_eq!(parsed["prefix"], "prod");
3606        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3607        assert_eq!(parsed["region"], "us-east-1");
3608        assert_eq!(parsed["backupId"], "test-backup-id");
3609        assert_eq!(parsed["consistency"], "kanidm-offline");
3610        assert_eq!(parsed["reason"], "restore-safety");
3611        assert_eq!(
3612            parsed["payloadPath"],
3613            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3614        );
3615        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3616    }
3617
3618    #[test]
3619    fn download_operation_doc_uses_real_repository_data() {
3620        let source = KanidmRestoreSource {
3621            local: None,
3622            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3623                name: "backup-1".to_string(),
3624            }),
3625        };
3626        let _restore = make_restore(source, None);
3627        let target = super::super::crd::Kanidm::default();
3628        let doc_str = super::build_download_operation_doc(
3629            &target,
3630            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3631            "019c7c76-f423-7a12-8f41-2bea7588a303",
3632            "source-uid-456",
3633            "real-bucket",
3634            "real-prefix",
3635            "https://real-endpoint.com",
3636            "eu-west-1",
3637            true,
3638            false,
3639            None,
3640            None,
3641        );
3642        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3643        assert_eq!(parsed["bucket"], "real-bucket");
3644        assert_eq!(parsed["prefix"], "real-prefix");
3645        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3646        assert_eq!(parsed["region"], "eu-west-1");
3647        assert_eq!(parsed["forcePathStyle"], true);
3648        assert_eq!(
3649            parsed["manifestKey"],
3650            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3651        );
3652        assert_eq!(
3653            parsed["expectedBackupId"],
3654            "019c7c76-f423-7a12-8f41-2bea7588a303"
3655        );
3656        assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3657    }
3658
3659    #[test]
3660    fn safety_backup_ref_is_set_after_verification() {
3661        let status = KanidmRestoreStatus {
3662            phase: KanidmRestorePhase::SafetyBackup,
3663            safety_backup_job_name: Some("test-safety-backup".to_string()),
3664            safety_backup_ref: Some("safety-test-restore".to_string()),
3665            safety_backup_expected_backup_id: Some(
3666                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3667            ),
3668            safety_backup_manifest_key: Some(
3669                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3670            ),
3671            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3672            ..Default::default()
3673        };
3674        assert!(status.safety_backup_ref.is_some());
3675        assert!(
3676            status
3677                .safety_backup_ref
3678                .as_ref()
3679                .unwrap()
3680                .starts_with("safety-")
3681        );
3682        assert!(status.safety_backup_expected_backup_id.is_some());
3683        assert!(status.safety_backup_manifest_key.is_some());
3684        assert!(status.safety_backup_payload_sha256.is_some());
3685    }
3686
3687    #[test]
3688    fn database_mutation_not_started_without_safety_backup_ref() {
3689        let status = KanidmRestoreStatus {
3690            phase: KanidmRestorePhase::PreparingSource,
3691            safety_backup_ref: None,
3692            database_mutation_started: false,
3693            ..Default::default()
3694        };
3695        assert!(!status.database_mutation_started);
3696        assert!(status.safety_backup_ref.is_none());
3697    }
3698
3699    #[test]
3700    fn pre_boundary_failure_does_not_set_mutation_started() {
3701        let status = KanidmRestoreStatus {
3702            phase: KanidmRestorePhase::Failed,
3703            message: Some(
3704                "safety backup job failed; target restored to original state".to_string(),
3705            ),
3706            database_mutation_started: false,
3707            ..Default::default()
3708        };
3709        assert!(!status.database_mutation_started);
3710    }
3711
3712    #[test]
3713    fn shared_volume_constants_are_distinct() {
3714        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3715        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3716    }
3717
3718    #[test]
3719    fn validate_backup_compatibility_matching_versions_passes() {
3720        let source = KanidmRestoreSource {
3721            local: None,
3722            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3723                name: "backup-1".to_string(),
3724            }),
3725        };
3726        let restore = make_restore(source, None);
3727        let target = kanidm_with_version("1.10.4");
3728        let backup = kaniop_backup_core::crd::KanidmBackup {
3729            metadata: kube::api::ObjectMeta::default(),
3730            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3731                backup_id: "id".to_string(),
3732                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3733                    name: "k".to_string(),
3734                    uid: "uid".to_string(),
3735                },
3736                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3737                    name: "repo".to_string(),
3738                },
3739                manifest_key: "key".to_string(),
3740            },
3741            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3742                kanidm_version: Some("1.10.4".to_string()),
3743                ..Default::default()
3744            }),
3745        };
3746        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3747    }
3748
3749    #[test]
3750    fn validate_backup_compatibility_version_mismatch_fails() {
3751        let source = KanidmRestoreSource {
3752            local: None,
3753            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3754                name: "backup-1".to_string(),
3755            }),
3756        };
3757        let restore = make_restore(source, None);
3758        let target = kanidm_with_version("1.11.0");
3759        let backup = kaniop_backup_core::crd::KanidmBackup {
3760            metadata: kube::api::ObjectMeta::default(),
3761            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3762                backup_id: "id".to_string(),
3763                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3764                    name: "k".to_string(),
3765                    uid: "uid".to_string(),
3766                },
3767                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3768                    name: "repo".to_string(),
3769                },
3770                manifest_key: "key".to_string(),
3771            },
3772            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3773                kanidm_version: Some("1.10.4".to_string()),
3774                ..Default::default()
3775            }),
3776        };
3777        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3778    }
3779
3780    #[test]
3781    fn validate_backup_compatibility_no_backup_version_passes() {
3782        let source = KanidmRestoreSource {
3783            local: None,
3784            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3785                name: "backup-1".to_string(),
3786            }),
3787        };
3788        let restore = make_restore(source, None);
3789        let target = super::super::crd::Kanidm::default();
3790        let backup = kaniop_backup_core::crd::KanidmBackup {
3791            metadata: kube::api::ObjectMeta::default(),
3792            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3793                backup_id: "id".to_string(),
3794                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3795                    name: "k".to_string(),
3796                    uid: "uid".to_string(),
3797                },
3798                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3799                    name: "repo".to_string(),
3800                },
3801                manifest_key: "key".to_string(),
3802            },
3803            status: None,
3804        };
3805        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3806    }
3807
3808    #[test]
3809    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3810        let source = KanidmRestoreSource {
3811            local: None,
3812            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3813                name: "backup-1".to_string(),
3814            }),
3815        };
3816        let mut restore = make_restore(source, None);
3817        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3818        let target = super::super::crd::Kanidm::default();
3819        let backup = kaniop_backup_core::crd::KanidmBackup {
3820            metadata: kube::api::ObjectMeta::default(),
3821            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3822                backup_id: "id".to_string(),
3823                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3824                    name: "k".to_string(),
3825                    uid: "uid".to_string(),
3826                },
3827                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3828                    name: "repo".to_string(),
3829                },
3830                manifest_key: "key".to_string(),
3831            },
3832            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3833                image_digest: Some("sha256:originaldigest".to_string()),
3834                ..Default::default()
3835            }),
3836        };
3837        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3838    }
3839
3840    #[test]
3841    fn validate_backup_compatibility_image_digest_match_passes() {
3842        let source = KanidmRestoreSource {
3843            local: None,
3844            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3845                name: "backup-1".to_string(),
3846            }),
3847        };
3848        let mut restore = make_restore(source, None);
3849        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3850        let target = super::super::crd::Kanidm::default();
3851        let backup = kaniop_backup_core::crd::KanidmBackup {
3852            metadata: kube::api::ObjectMeta::default(),
3853            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3854                backup_id: "id".to_string(),
3855                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3856                    name: "k".to_string(),
3857                    uid: "uid".to_string(),
3858                },
3859                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3860                    name: "repo".to_string(),
3861                },
3862                manifest_key: "key".to_string(),
3863            },
3864            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3865                image_digest: Some("sha256:abc123".to_string()),
3866                ..Default::default()
3867            }),
3868        };
3869        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3870    }
3871
3872    #[test]
3873    fn safety_backup_id_is_deterministic() {
3874        let source = KanidmRestoreSource {
3875            local: None,
3876            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3877                name: "backup-1".to_string(),
3878            }),
3879        };
3880        let restore = make_restore(source, None);
3881        let id1 = super::compute_safety_backup_id(&restore).unwrap();
3882        let id2 = super::compute_safety_backup_id(&restore).unwrap();
3883        assert_eq!(id1, id2);
3884        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3885    }
3886
3887    #[test]
3888    fn safety_manifest_key_is_deterministic() {
3889        let source = KanidmRestoreSource {
3890            local: None,
3891            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3892                name: "backup-1".to_string(),
3893            }),
3894        };
3895        let restore = make_restore(source, None);
3896        let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3897        let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3898        assert_eq!(key1, key2);
3899        assert!(key1.contains("manifest.json"));
3900        assert!(key1.starts_with("v1/tenants/"));
3901        // The tenant segment is the Kubernetes namespace NAME from `make_restore`, never the
3902        // Namespace object UID or the restore UID.
3903        assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3904        assert!(
3905            !key1.contains("a1b2c3d4"),
3906            "safety manifest key must not key the tenant segment by object UID: {key1}"
3907        );
3908    }
3909
3910    /// `compute_safety_manifest_key` hand-rolls the keyspace with `format!` instead of going through
3911    /// `kaniop_backup_core::paths::RepositoryPath`, so it is pinned here byte-for-byte against the
3912    /// deployed layout: `v1/tenants/<namespace-name>/clusters/<kanidm-uid>/backups/<backup-id>/`
3913    /// `manifest.json`. The `<backup-id>` is the UUIDv5 chain `DNS -> namespace name -> restore UID`
3914    /// and is load-bearing for transport re-upload idempotency, so both the segment and the derived
3915    /// ID are asserted as literals. The tenant segment carries the namespace NAME even though the
3916    /// `namespaceUid` wire field it mirrors is named like a UID (misnomer tracked in issue #1084).
3917    #[test]
3918    fn safety_manifest_key_matches_deployed_keyspace_layout() {
3919        let source = KanidmRestoreSource {
3920            local: None,
3921            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3922                name: "backup-1".to_string(),
3923            }),
3924        };
3925        let restore = make_restore(source, None);
3926        let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3927        assert_eq!(
3928            super::compute_safety_backup_id(&restore).unwrap(),
3929            expected_backup_id
3930        );
3931        assert_eq!(
3932            super::compute_safety_manifest_key(&restore).unwrap(),
3933            format!(
3934                "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
3935            )
3936        );
3937    }
3938
3939    #[test]
3940    fn safety_backup_id_survives_restart() {
3941        let source = KanidmRestoreSource {
3942            local: None,
3943            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3944                name: "backup-1".to_string(),
3945            }),
3946        };
3947        let mut restore = make_restore(source, None);
3948        let original_id = super::compute_safety_backup_id(&restore).unwrap();
3949        // Pinning the derived ID pins its UUIDv5 inputs: the namespace NAME and the restore UID.
3950        // Feeding the Namespace object UID here would change every safety backup ID and orphan an
3951        // in-flight safety backup (issue #1084).
3952        assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
3953        restore.status = Some(KanidmRestoreStatus {
3954            phase: KanidmRestorePhase::SafetyBackup,
3955            safety_backup_expected_backup_id: Some(original_id.clone()),
3956            ..Default::default()
3957        });
3958        let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3959        assert_eq!(original_id, recomputed_id);
3960    }
3961
3962    #[test]
3963    fn safety_backup_id_derived_from_uid_not_name() {
3964        let source = KanidmRestoreSource {
3965            local: None,
3966            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3967                name: "backup-1".to_string(),
3968            }),
3969        };
3970        let restore1 = make_restore(source.clone(), None);
3971        let mut restore2 = make_restore(source, None);
3972        restore2.metadata.name = Some("different-name-same-uid".to_string());
3973        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3974        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3975        assert_eq!(id1, id2);
3976    }
3977
3978    #[test]
3979    fn safety_backup_id_differs_for_different_uids() {
3980        let source = KanidmRestoreSource {
3981            local: None,
3982            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3983                name: "backup-1".to_string(),
3984            }),
3985        };
3986        let restore1 = make_restore(source.clone(), None);
3987        let mut restore2 = make_restore(source, None);
3988        restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3989        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3990        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3991        assert_ne!(id1, id2);
3992    }
3993
3994    #[test]
3995    fn safety_backup_id_errors_without_uid() {
3996        let source = KanidmRestoreSource {
3997            local: None,
3998            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3999                name: "backup-1".to_string(),
4000            }),
4001        };
4002        let mut restore = make_restore(source, None);
4003        restore.metadata.uid = None;
4004        assert!(super::compute_safety_backup_id(&restore).is_err());
4005    }
4006
4007    #[test]
4008    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4009        let source = KanidmRestoreSource {
4010            local: None,
4011            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4012                name: "backup-1".to_string(),
4013            }),
4014        };
4015        let restore = make_restore(source, None);
4016        assert!(super::requires_safety_backup(&restore));
4017        let status_without_ref = KanidmRestoreStatus {
4018            phase: KanidmRestorePhase::RestoringPrimary,
4019            safety_backup_ref: None,
4020            database_mutation_started: false,
4021            ..Default::default()
4022        };
4023        assert!(status_without_ref.safety_backup_ref.is_none());
4024        assert!(!status_without_ref.database_mutation_started);
4025    }
4026
4027    #[test]
4028    fn mutation_boundary_allows_progression_when_safety_ref_set() {
4029        let status_with_ref = KanidmRestoreStatus {
4030            phase: KanidmRestorePhase::RestoringPrimary,
4031            safety_backup_ref: Some("safety-test-restore".to_string()),
4032            safety_backup_manifest_key: Some(
4033                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4034            ),
4035            safety_backup_payload_sha256: Some("abc123".to_string()),
4036            database_mutation_started: false,
4037            ..Default::default()
4038        };
4039        assert!(status_with_ref.safety_backup_ref.is_some());
4040        assert!(!status_with_ref.database_mutation_started);
4041    }
4042
4043    #[test]
4044    fn result_document_verification_rejects_missing_backup_id() {
4045        let result_json = r#"{
4046            "apiVersion": "backup.kaniop.rs/v1alpha1",
4047            "kind": "ResultDocument",
4048            "operation": "upload",
4049            "success": true,
4050            "exitCode": "success",
4051            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4052            "payloadSha256": "abc123"
4053        }"#;
4054        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4055        assert!(result.backup_id.is_none());
4056    }
4057
4058    #[test]
4059    fn result_document_verification_rejects_missing_manifest_key() {
4060        let result_json = r#"{
4061            "apiVersion": "backup.kaniop.rs/v1alpha1",
4062            "kind": "ResultDocument",
4063            "operation": "upload",
4064            "success": true,
4065            "exitCode": "success",
4066            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4067            "payloadSha256": "abc123"
4068        }"#;
4069        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4070        assert!(result.manifest_key.is_none());
4071    }
4072
4073    #[test]
4074    fn result_document_verification_rejects_missing_payload_sha256() {
4075        let result_json = r#"{
4076            "apiVersion": "backup.kaniop.rs/v1alpha1",
4077            "kind": "ResultDocument",
4078            "operation": "upload",
4079            "success": true,
4080            "exitCode": "success",
4081            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4082            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4083        }"#;
4084        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4085        assert!(result.payload_sha256.is_none());
4086    }
4087
4088    #[test]
4089    fn result_document_verification_rejects_failure_exit_code() {
4090        let result_json = r#"{
4091            "apiVersion": "backup.kaniop.rs/v1alpha1",
4092            "kind": "ResultDocument",
4093            "operation": "upload",
4094            "success": false,
4095            "exitCode": "integrity",
4096            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4097            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4098            "payloadSha256": "abc123"
4099        }"#;
4100        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4101        assert!(!result.success);
4102        assert_eq!(
4103            result.exit_code,
4104            kaniop_backup_core::result::ExitCode::Integrity
4105        );
4106    }
4107
4108    #[test]
4109    fn result_document_verification_rejects_wrong_operation() {
4110        let result_json = r#"{
4111            "apiVersion": "backup.kaniop.rs/v1alpha1",
4112            "kind": "ResultDocument",
4113            "operation": "download",
4114            "success": true,
4115            "exitCode": "success",
4116            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4117            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4118            "payloadSha256": "abc123"
4119        }"#;
4120        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4121        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4122    }
4123
4124    #[test]
4125    fn result_document_verification_accepts_valid_document() {
4126        let result_json = r#"{
4127            "apiVersion": "backup.kaniop.rs/v1alpha1",
4128            "kind": "ResultDocument",
4129            "operation": "upload",
4130            "success": true,
4131            "exitCode": "success",
4132            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4133            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4134            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4135        }"#;
4136        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4137        assert!(result.success);
4138        assert_eq!(
4139            result.exit_code,
4140            kaniop_backup_core::result::ExitCode::Success
4141        );
4142        assert_eq!(result.operation, "upload");
4143        assert!(result.backup_id.is_some());
4144        assert!(result.manifest_key.is_some());
4145        assert!(result.payload_sha256.is_some());
4146    }
4147
4148    #[test]
4149    fn termination_message_path_constant_is_valid() {
4150        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4151        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4152    }
4153
4154    #[test]
4155    fn status_tracks_safety_backup_evidence() {
4156        let status = KanidmRestoreStatus {
4157            phase: KanidmRestorePhase::PreparingSource,
4158            safety_backup_ref: Some("safety-test-restore".to_string()),
4159            safety_backup_expected_backup_id: Some(
4160                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4161            ),
4162            safety_backup_manifest_key: Some(
4163                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4164            ),
4165            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4166            ..Default::default()
4167        };
4168        assert!(status.safety_backup_ref.is_some());
4169        assert!(status.safety_backup_expected_backup_id.is_some());
4170        assert!(status.safety_backup_manifest_key.is_some());
4171        assert!(status.safety_backup_payload_sha256.is_some());
4172    }
4173
4174    #[test]
4175    fn fail_closed_semantics_preserved_on_result_parse_failure() {
4176        let invalid_json = "not valid json";
4177        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4178        assert!(result.is_err());
4179    }
4180
4181    #[test]
4182    fn fail_closed_semantics_preserved_on_oversized_result() {
4183        let oversized_json = format!(
4184            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4185            kaniop_backup_core::result::RESULT_DOC_VERSION,
4186            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4187        );
4188        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4189        assert!(result.is_err());
4190    }
4191
4192    #[test]
4193    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4194        let source = KanidmRestoreSource {
4195            local: None,
4196            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4197                name: "backup-1".to_string(),
4198            }),
4199        };
4200        let mut restore = make_restore(source, None);
4201        restore.metadata.namespace = Some("test-ns".to_string());
4202        let target = super::super::crd::Kanidm::default();
4203        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4204        let doc_str = super::build_safety_upload_operation_doc(
4205            &restore,
4206            &target,
4207            "test-backup-id",
4208            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4209            "my-bucket",
4210            "prod",
4211            "https://s3.example.com",
4212            "us-east-1",
4213            false,
4214            false,
4215            Some(&ca_path),
4216            None,
4217        )
4218        .unwrap();
4219        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4220        assert_eq!(parsed["caBundlePath"], ca_path);
4221    }
4222
4223    #[test]
4224    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4225        let source = KanidmRestoreSource {
4226            local: None,
4227            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4228                name: "backup-1".to_string(),
4229            }),
4230        };
4231        let mut restore = make_restore(source, None);
4232        restore.metadata.namespace = Some("test-ns".to_string());
4233        let target = super::super::crd::Kanidm::default();
4234        let doc_str = super::build_safety_upload_operation_doc(
4235            &restore,
4236            &target,
4237            "test-backup-id",
4238            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4239            "my-bucket",
4240            "prod",
4241            "https://s3.example.com",
4242            "us-east-1",
4243            false,
4244            false,
4245            None,
4246            None,
4247        )
4248        .unwrap();
4249        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4250        assert!(parsed["caBundlePath"].is_null());
4251    }
4252
4253    #[test]
4254    fn download_operation_doc_includes_ca_bundle_path_when_set() {
4255        let source = KanidmRestoreSource {
4256            local: None,
4257            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4258                name: "backup-1".to_string(),
4259            }),
4260        };
4261        let _restore = make_restore(source, None);
4262        let target = super::super::crd::Kanidm::default();
4263        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4264        let doc_str = super::build_download_operation_doc(
4265            &target,
4266            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4267            "019c7c76-f423-7a12-8f41-2bea7588a303",
4268            "source-uid-456",
4269            "real-bucket",
4270            "real-prefix",
4271            "https://real-endpoint.com",
4272            "eu-west-1",
4273            true,
4274            false,
4275            Some(&ca_path),
4276            None,
4277        );
4278        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4279        assert_eq!(parsed["caBundlePath"], ca_path);
4280    }
4281
4282    #[test]
4283    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4284        let source = KanidmRestoreSource {
4285            local: None,
4286            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4287                name: "backup-1".to_string(),
4288            }),
4289        };
4290        let _restore = make_restore(source, None);
4291        let target = super::super::crd::Kanidm::default();
4292        let doc_str = super::build_download_operation_doc(
4293            &target,
4294            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4295            "019c7c76-f423-7a12-8f41-2bea7588a303",
4296            "source-uid-456",
4297            "real-bucket",
4298            "real-prefix",
4299            "https://real-endpoint.com",
4300            "eu-west-1",
4301            true,
4302            false,
4303            None,
4304            None,
4305        );
4306        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4307        assert!(parsed["caBundlePath"].is_null());
4308    }
4309
4310    #[test]
4311    fn source_prep_result_operation_is_download() {
4312        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4313        assert_ne!(
4314            super::SOURCE_PREP_RESULT_OPERATION,
4315            super::SAFETY_BACKUP_RESULT_OPERATION
4316        );
4317    }
4318
4319    #[test]
4320    fn download_result_document_verification_accepts_valid_document() {
4321        let result_json = r#"{
4322            "apiVersion": "backup.kaniop.rs/v1alpha1",
4323            "kind": "ResultDocument",
4324            "operation": "download",
4325            "success": true,
4326            "exitCode": "success",
4327            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4328            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4329            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4330        }"#;
4331        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4332        assert!(result.success);
4333        assert_eq!(
4334            result.exit_code,
4335            kaniop_backup_core::result::ExitCode::Success
4336        );
4337        assert_eq!(result.operation, "download");
4338        assert_eq!(
4339            result.backup_id.as_deref(),
4340            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4341        );
4342        assert!(result.manifest_key.is_some());
4343        assert!(result.payload_sha256.is_some());
4344    }
4345
4346    #[test]
4347    fn download_result_document_rejects_wrong_operation() {
4348        let result_json = r#"{
4349            "apiVersion": "backup.kaniop.rs/v1alpha1",
4350            "kind": "ResultDocument",
4351            "operation": "upload",
4352            "success": true,
4353            "exitCode": "success",
4354            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4355            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4356            "payloadSha256": "abc123"
4357        }"#;
4358        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4359        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4360    }
4361
4362    #[test]
4363    fn download_result_document_rejects_failure_success_field() {
4364        let result_json = r#"{
4365            "apiVersion": "backup.kaniop.rs/v1alpha1",
4366            "kind": "ResultDocument",
4367            "operation": "download",
4368            "success": false,
4369            "exitCode": "retryable",
4370            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4371            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4372            "payloadSha256": "abc123"
4373        }"#;
4374        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4375        assert!(!result.success);
4376        assert_ne!(
4377            result.exit_code,
4378            kaniop_backup_core::result::ExitCode::Success
4379        );
4380    }
4381
4382    #[test]
4383    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4384        let status = KanidmRestoreStatus {
4385            phase: KanidmRestorePhase::PreparingSource,
4386            database_mutation_started: false,
4387            ..Default::default()
4388        };
4389        assert!(!status.database_mutation_started);
4390        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4391    }
4392
4393    #[test]
4394    fn backup_job_volume_size_default_is_10gi() {
4395        let qty = crate::controller::backup_job_volume_size();
4396        assert_eq!(qty, Quantity("10Gi".to_string()));
4397    }
4398
4399    #[test]
4400    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4401        use kaniop_backup_core::auth::build_encryption_env_vars;
4402        use kaniop_backup_core::crd::SecretRef;
4403
4404        let key_ref = SecretRef {
4405            name: "kek-secret".to_string(),
4406        };
4407        let env_vars = build_encryption_env_vars(Some(&key_ref));
4408        assert_eq!(env_vars.len(), 1);
4409        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4410        let secret_ref = env_vars[0]
4411            .value_from
4412            .as_ref()
4413            .unwrap()
4414            .secret_key_ref
4415            .as_ref()
4416            .unwrap();
4417        assert_eq!(secret_ref.name, "kek-secret");
4418        assert_eq!(secret_ref.key, "encryption-key");
4419    }
4420
4421    #[test]
4422    fn encryption_env_vars_absent_when_no_key_ref() {
4423        use kaniop_backup_core::auth::build_encryption_env_vars;
4424
4425        let env_vars = build_encryption_env_vars(None);
4426        assert!(
4427            env_vars.is_empty(),
4428            "encryption env vars must be empty when key_ref is None"
4429        );
4430    }
4431
4432    #[test]
4433    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4434        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4435
4436        let source = KanidmRestoreSource {
4437            local: None,
4438            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4439                name: "backup-1".to_string(),
4440            }),
4441        };
4442        let mut restore = make_restore(source, None);
4443        restore.metadata.namespace = Some("test-ns".to_string());
4444        let target = super::super::crd::Kanidm::default();
4445        let encryption = RepositoryEncryption {
4446            mode: EncryptionMode::ClientSide,
4447            key_id: None,
4448            key_ref: Some(SecretRef {
4449                name: "kek-secret".to_string(),
4450            }),
4451        };
4452        let doc_str = super::build_safety_upload_operation_doc(
4453            &restore,
4454            &target,
4455            "test-backup-id",
4456            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4457            "my-bucket",
4458            "prod",
4459            "https://s3.example.com",
4460            "us-east-1",
4461            false,
4462            false,
4463            None,
4464            Some(&encryption),
4465        )
4466        .unwrap();
4467        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4468        assert_eq!(parsed["encryptionMode"], "clientSide");
4469        assert!(parsed["encryptionKeyId"].is_null());
4470    }
4471
4472    #[test]
4473    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4474        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4475
4476        let source = KanidmRestoreSource {
4477            local: None,
4478            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4479                name: "backup-1".to_string(),
4480            }),
4481        };
4482        let _restore = make_restore(source, None);
4483        let target = super::super::crd::Kanidm::default();
4484        let encryption = RepositoryEncryption {
4485            mode: EncryptionMode::ProviderKms,
4486            key_id: Some("alias/kaniop-backups".to_string()),
4487            key_ref: None,
4488        };
4489        let doc_str = super::build_download_operation_doc(
4490            &target,
4491            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4492            "019c7c76-f423-7a12-8f41-2bea7588a303",
4493            "source-uid-456",
4494            "real-bucket",
4495            "real-prefix",
4496            "https://real-endpoint.com",
4497            "eu-west-1",
4498            true,
4499            false,
4500            None,
4501            Some(&encryption),
4502        );
4503        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4504        assert_eq!(parsed["encryptionMode"], "providerKms");
4505        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4506    }
4507
4508    #[test]
4509    fn post_mutation_failure_records_job_names() {
4510        let existing = KanidmRestoreStatus {
4511            phase: KanidmRestorePhase::RestoringPrimary,
4512            database_mutation_started: true,
4513            ..Default::default()
4514        };
4515
4516        let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4517        assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4518        assert_eq!(
4519            restore_failed.restore_job_name.as_deref(),
4520            Some("my-restore-job")
4521        );
4522        assert!(restore_failed.verify_job_name.is_none());
4523        assert!(restore_failed.database_mutation_started);
4524        assert_eq!(
4525            restore_failed.message.as_deref(),
4526            Some("database restore job failed")
4527        );
4528
4529        let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4530        assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4531        assert_eq!(
4532            verify_failed.verify_job_name.as_deref(),
4533            Some("my-verify-job")
4534        );
4535        assert!(verify_failed.restore_job_name.is_none());
4536        assert!(verify_failed.database_mutation_started);
4537        assert_eq!(
4538            verify_failed.message.as_deref(),
4539            Some("database verification failed")
4540        );
4541    }
4542
4543    #[test]
4544    fn restore_and_verify_job_names_are_deterministic() {
4545        let source = KanidmRestoreSource {
4546            local: Some(super::KanidmRestoreLocalSource {
4547                file_name: "backup.json".to_string(),
4548            }),
4549            backup_ref: None,
4550        };
4551        let restore = make_restore(source, None);
4552        assert_eq!(restore_job_name(&restore), "test-restore-restore");
4553        assert_eq!(verify_job_name(&restore), "test-restore-verify");
4554    }
4555}