Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76    feature = "schemars",
77    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80    category = "kaniop",
81    group = "kaniop.rs",
82    version = "v1beta1",
83    kind = "KanidmRestore",
84    plural = "kanidmrestores",
85    singular = "kanidmrestore",
86    shortname = "idmrestore",
87    namespaced,
88    status = "KanidmRestoreStatus",
89    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95    pub target_ref: KanidmRestoreTargetRef,
96    pub source: KanidmRestoreSource,
97    pub restore_image: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106    pub name: String,
107    pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113    feature = "schemars",
114    schemars(extend("x-kubernetes-validations" = [
115        {
116            "message": "exactly one of local or backupRef must be set",
117            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118        }
119    ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123    /// Local file source. Mutually exclusive with `backupRef`.
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub local: Option<KanidmRestoreLocalSource>,
126    /// Remote cataloged backup reference. Mutually exclusive with `local`.
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
136    pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
144    pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub repository_ref: Option<SafetyBackupRepositoryRef>,
154    /// Skip the mandatory safety backup. Requires break-glass annotations.
155    #[serde(default)]
156    pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163    pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169    #[default]
170    Pending,
171    Validating,
172    Quiescing,
173    SafetyBackup,
174    PreparingSource,
175    RestoringPrimary,
176    Verifying,
177    RebuildingReplicas,
178    Resuming,
179    Completed,
180    Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub observed_generation: Option<i64>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_target_uid: Option<String>,
191    #[serde(default)]
192    pub phase: KanidmRestorePhase,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub restore_job_name: Option<String>,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub verify_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub safety_backup_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub source_prep_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub safety_backup_ref: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_expected_backup_id: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_manifest_key: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_payload_sha256: Option<String>,
209    #[serde(default)]
210    pub replicas_cleared: bool,
211    #[serde(default)]
212    pub certificates_cleared: bool,
213    #[serde(default)]
214    pub database_mutation_started: bool,
215    #[serde(default, skip_serializing_if = "Vec::is_empty")]
216    pub original_replicas: Vec<ReplicaCountEntry>,
217    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218    pub phase_timestamps: std::collections::BTreeMap<String, String>,
219    #[serde(default, skip_serializing_if = "Vec::is_empty")]
220    #[cfg_attr(
221        feature = "schemars",
222        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223    )]
224    pub conditions: Vec<Condition>,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233    pub group: String,
234    pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239    attempts: Counter<u64>,
240    outcomes: Counter<u64>,
241    duration_seconds: Histogram<f64>,
242    break_glass_total: Counter<u64>,
243    safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247    fn new() -> Self {
248        let meter = global::meter("kaniop");
249        Self {
250            attempts: meter
251                .u64_counter("kaniop_restore_attempts_total")
252                .with_description("Number of Kanidm restore attempts started")
253                .build(),
254            outcomes: meter
255                .u64_counter("kaniop_restore_outcomes_total")
256                .with_description("Number of terminal Kanidm restore outcomes")
257                .build(),
258            duration_seconds: meter
259                .f64_histogram("kaniop_restore_duration_seconds")
260                .with_description("Kanidm restore duration from object creation to terminal phase")
261                .with_unit("s")
262                .build(),
263            break_glass_total: meter
264                .u64_counter("kaniop_restore_break_glass_total")
265                .with_description("Number of break-glass safety backup overrides")
266                .build(),
267            safety_backup_duration_seconds: meter
268                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269                .with_description("Duration of pre-restore safety backup creation")
270                .with_unit("s")
271                .build(),
272        }
273    }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278    client: Client,
279    recorder: Recorder,
280    metrics: RestoreMetrics,
281}
282
283pub async fn run(client: Client) {
284    let api = Api::<KanidmRestore>::all(client.clone());
285    let recorder = Recorder::new(
286        client.clone(),
287        Reporter {
288            controller: CONTROLLER_ID.into(),
289            instance: None,
290        },
291    );
292    let ctx = Arc::new(RestoreContext {
293        client,
294        recorder,
295        metrics: RestoreMetrics::new(),
296    });
297    info!("starting {CONTROLLER_ID} controller");
298    Controller::new(api, watcher::Config::default().any_semantic())
299        .shutdown_on_signal()
300        .run(reconcile_restore, error_policy, ctx)
301        .for_each(|result| async move {
302            if let Err(error) = result {
303                error!(%error, "KanidmRestore reconciliation failed");
304            }
305        })
306        .await;
307}
308
309fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
310    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
311    Action::requeue(Duration::from_secs(5))
312}
313
314async fn reconcile_restore(
315    restore: Arc<KanidmRestore>,
316    ctx: Arc<RestoreContext>,
317) -> Result<Action> {
318    let namespace = restore
319        .namespace()
320        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
321    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
322    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
323        let ctx = ctx.clone();
324        async move {
325            match event {
326                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
327                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
328            }
329        }
330    })
331    .await
332    .map_err(|error| {
333        Error::FinalizerError(
334            "failed on KanidmRestore finalizer".to_string(),
335            Box::new(error),
336        )
337    })
338}
339
340async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
341    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
342    match phase {
343        KanidmRestorePhase::Pending => {
344            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
345            Ok(Action::requeue(REQUEUE))
346        }
347        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
348            Ok(target) => {
349                mark_restoring(&restore, &target, &ctx).await?;
350                let mut status = restore.status.clone().unwrap_or_default();
351                status.observed_target_uid = target.uid();
352                status.original_replicas = capture_original_replicas(&target);
353                status.phase = KanidmRestorePhase::Quiescing;
354                status.message = None;
355                patch_status(&restore, &ctx, status).await?;
356                Ok(Action::requeue(REQUEUE))
357            }
358            Err(error) => {
359                set_phase(
360                    &restore,
361                    &ctx,
362                    KanidmRestorePhase::Failed,
363                    Some(error.to_string()),
364                )
365                .await?;
366                Ok(Action::requeue(Duration::from_secs(300)))
367            }
368        },
369        KanidmRestorePhase::Quiescing => {
370            let target = get_target(&restore, &ctx).await?;
371            scale_all(&target, &ctx, 0).await?;
372            if target_pods_stopped(&target, &ctx).await?
373                && target_volumes_detached(&target, &ctx).await?
374            {
375                let next_phase = if requires_safety_backup(&restore) {
376                    KanidmRestorePhase::SafetyBackup
377                } else {
378                    record_break_glass(&restore, &ctx).await?;
379                    KanidmRestorePhase::PreparingSource
380                };
381                if next_phase == KanidmRestorePhase::PreparingSource {
382                    let refreshed = Api::<KanidmRestore>::namespaced(
383                        ctx.client.clone(),
384                        restore.namespace().as_deref().unwrap_or_default(),
385                    )
386                    .get(&restore.name_any())
387                    .await
388                    .map_err(|error| {
389                        Error::kube_error(
390                            "get",
391                            "KanidmRestore",
392                            restore.namespace().as_deref().unwrap_or_default(),
393                            restore.name_any(),
394                            error,
395                        )
396                    })?;
397                    set_phase(&refreshed, &ctx, next_phase, None).await?;
398                } else {
399                    set_phase(&restore, &ctx, next_phase, None).await?;
400                }
401            }
402            Ok(Action::requeue(REQUEUE))
403        }
404        KanidmRestorePhase::SafetyBackup => {
405            let target = get_target(&restore, &ctx).await?;
406            let name = safety_backup_job_name(&restore);
407            let expected_backup_id = compute_safety_backup_id(&restore);
408            let expected_manifest_key = compute_safety_manifest_key(&restore);
409            ensure_safety_backup_job(
410                &restore,
411                &target,
412                &ctx,
413                &name,
414                &expected_backup_id,
415                &expected_manifest_key,
416            )
417            .await?;
418            let mut status = restore.status.clone().unwrap_or_default();
419            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
420            match job_state(&restore, &ctx, &name).await? {
421                JobState::Complete => {
422                    let result =
423                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
424                    match result {
425                        Ok(verified) => {
426                            status.safety_backup_job_name = Some(name);
427                            status.safety_backup_ref =
428                                Some(format!("safety-{}", restore.name_any()));
429                            status.safety_backup_manifest_key = Some(verified.manifest_key);
430                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
431                            record_safety_backup_duration(&restore, &ctx).await;
432                            status.phase = KanidmRestorePhase::PreparingSource;
433                            status.message = None;
434                            patch_status(&restore, &ctx, status).await?;
435                        }
436                        Err(error) => {
437                            resume_before_mutation(&restore, &ctx).await?;
438                            set_phase(
439                                &restore,
440                                &ctx,
441                                KanidmRestorePhase::Failed,
442                                Some(format!("safety backup result verification failed: {error}")),
443                            )
444                            .await?;
445                        }
446                    }
447                }
448                JobState::Failed => {
449                    resume_before_mutation(&restore, &ctx).await?;
450                    set_phase(
451                        &restore,
452                        &ctx,
453                        KanidmRestorePhase::Failed,
454                        Some(
455                            "safety backup job failed; target restored to original state"
456                                .to_string(),
457                        ),
458                    )
459                    .await?;
460                }
461                JobState::Running => {
462                    patch_status(&restore, &ctx, status).await?;
463                }
464            }
465            Ok(Action::requeue(REQUEUE))
466        }
467        KanidmRestorePhase::PreparingSource => {
468            if is_remote_source(&restore) {
469                let target = get_target(&restore, &ctx).await?;
470                let name = source_prep_job_name(&restore);
471                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
472                match job_state(&restore, &ctx, &name).await? {
473                    JobState::Complete => {
474                        let backup_id = restore
475                            .spec
476                            .source
477                            .backup_ref
478                            .as_ref()
479                            .map(|b| b.name.clone())
480                            .unwrap_or_default();
481                        let backup = {
482                            let ns = restore.namespace().unwrap();
483                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
484                                .get(&backup_id)
485                                .await
486                                .map_err(|e| {
487                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
488                                })?
489                        };
490                        let expected_backup_id = &backup.spec.backup_id;
491                        let result =
492                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
493                                .await;
494                        match result {
495                            Ok(verified) => {
496                                if let Some(expected_sha256) = backup
497                                    .status
498                                    .as_ref()
499                                    .and_then(|s| s.payload_sha256.as_deref())
500                                {
501                                    if verified.payload_sha256 != expected_sha256 {
502                                        resume_before_mutation(&restore, &ctx).await?;
503                                        set_phase(
504                                            &restore,
505                                            &ctx,
506                                            KanidmRestorePhase::Failed,
507                                            Some(format!(
508                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
509                                                verified.payload_sha256
510                                            )),
511                                        )
512                                        .await?;
513                                        return Ok(Action::requeue(REQUEUE));
514                                    }
515                                }
516                                let mut status = restore.status.clone().unwrap_or_default();
517                                status.source_prep_job_name = Some(name);
518                                status.phase = KanidmRestorePhase::RestoringPrimary;
519                                status.message = None;
520                                patch_status(&restore, &ctx, status).await?;
521                            }
522                            Err(error) => {
523                                resume_before_mutation(&restore, &ctx).await?;
524                                set_phase(
525                                    &restore,
526                                    &ctx,
527                                    KanidmRestorePhase::Failed,
528                                    Some(format!(
529                                        "source preparation result verification failed: {error}"
530                                    )),
531                                )
532                                .await?;
533                            }
534                        }
535                    }
536                    JobState::Failed => {
537                        resume_before_mutation(&restore, &ctx).await?;
538                        set_phase(
539                            &restore,
540                            &ctx,
541                            KanidmRestorePhase::Failed,
542                            Some(
543                                "source preparation job failed; target restored to original state"
544                                    .to_string(),
545                            ),
546                        )
547                        .await?;
548                    }
549                    JobState::Running => {}
550                }
551            } else {
552                let target = get_target(&restore, &ctx).await?;
553                let name = source_check_job_name(&restore);
554                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
555                match job_state(&restore, &ctx, &name).await? {
556                    JobState::Complete => {
557                        let mut status = restore.status.clone().unwrap_or_default();
558                        status.source_prep_job_name = Some(name);
559                        status.phase = KanidmRestorePhase::RestoringPrimary;
560                        status.message = None;
561                        patch_status(&restore, &ctx, status).await?;
562                    }
563                    JobState::Failed => {
564                        resume_before_mutation(&restore, &ctx).await?;
565                        set_phase(
566                            &restore,
567                            &ctx,
568                            KanidmRestorePhase::Failed,
569                            Some(
570                                "local backup file check failed; target restored to original state"
571                                    .to_string(),
572                            ),
573                        )
574                        .await?;
575                    }
576                    JobState::Running => {}
577                }
578            }
579            Ok(Action::requeue(REQUEUE))
580        }
581        KanidmRestorePhase::RestoringPrimary => {
582            let target = get_target(&restore, &ctx).await?;
583            let status = restore.status.as_ref().cloned().unwrap_or_default();
584            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
585                return Err(Error::MissingData(
586                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
587                ));
588            }
589            if !status.database_mutation_started {
590                let mut status = status;
591                status.database_mutation_started = true;
592                status.message = Some(
593                    "database mutation boundary persisted; starting primary restore".to_string(),
594                );
595                patch_status(&restore, &ctx, status).await?;
596                return Ok(Action::requeue(REQUEUE));
597            }
598            let name = restore_job_name(&restore);
599            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
600            match job_state(&restore, &ctx, &name).await? {
601                JobState::Complete => {
602                    let mut status = restore.status.clone().unwrap_or_default();
603                    status.restore_job_name = Some(name);
604                    status.phase = KanidmRestorePhase::Verifying;
605                    status.message = None;
606                    patch_status(&restore, &ctx, status).await?;
607                }
608                JobState::Failed => {
609                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
610                }
611                JobState::Running => {}
612            }
613            Ok(Action::requeue(REQUEUE))
614        }
615        KanidmRestorePhase::Verifying => {
616            let target = get_target(&restore, &ctx).await?;
617            let name = verify_job_name(&restore);
618            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
619            match job_state(&restore, &ctx, &name).await? {
620                JobState::Complete => {
621                    let mut status = restore.status.clone().unwrap_or_default();
622                    status.verify_job_name = Some(name);
623                    status.phase = KanidmRestorePhase::RebuildingReplicas;
624                    status.message = None;
625                    patch_status(&restore, &ctx, status).await?;
626                }
627                JobState::Failed => {
628                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
629                }
630                JobState::Running => {}
631            }
632            Ok(Action::requeue(REQUEUE))
633        }
634        KanidmRestorePhase::RebuildingReplicas => {
635            let target = get_target(&restore, &ctx).await?;
636            let mut status = restore.status.clone().unwrap_or_default();
637            if !status.replicas_cleared {
638                if delete_secondary_pvcs(&target, &ctx).await? {
639                    status.replicas_cleared = true;
640                    status.message = Some("secondary database state cleared".to_string());
641                    patch_status(&restore, &ctx, status).await?;
642                }
643                return Ok(Action::requeue(REQUEUE));
644            }
645
646            scale_primary(&target, &ctx, 1).await?;
647            if !primary_ready(&target, &ctx).await? {
648                return Ok(Action::requeue(REQUEUE));
649            }
650            if !status.certificates_cleared {
651                delete_replica_cert_secrets(&target, &ctx).await?;
652                delete_admin_secret(&target, &ctx).await?;
653                status.certificates_cleared = true;
654                status.message = Some(
655                    "replica certificates and admin secret cleared for regeneration".to_string(),
656                );
657                patch_status(&restore, &ctx, status).await?;
658                return Ok(Action::requeue(REQUEUE));
659            }
660            status.phase = KanidmRestorePhase::Resuming;
661            status.message = None;
662            patch_status(&restore, &ctx, status).await?;
663            Ok(Action::requeue(REQUEUE))
664        }
665        KanidmRestorePhase::Resuming => {
666            let target = get_target(&restore, &ctx).await?;
667            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
668            clear_restoring(&restore, &target, &ctx).await?;
669            Ok(Action::requeue(Duration::from_secs(3600)))
670        }
671        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
672            Ok(Action::requeue(Duration::from_secs(3600)))
673        }
674    }
675}
676
677async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
678    let status = restore.status.clone().unwrap_or_default();
679    if status.database_mutation_started
680        && status.phase != KanidmRestorePhase::Completed
681        && status.phase != KanidmRestorePhase::Failed
682    {
683        reconcile_apply(restore.clone(), ctx.clone()).await?;
684        return Ok(Action::requeue(REQUEUE));
685    }
686    if let Ok(target) = get_target(&restore, &ctx).await {
687        let owns_maintenance =
688            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
689        if owns_maintenance && !status.database_mutation_started {
690            scale_desired(&target, &ctx).await?;
691        }
692        clear_restoring(&restore, &target, &ctx).await?;
693    }
694    Ok(Action::await_change())
695}
696
697async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
698    let target = get_target(restore, ctx).await?;
699    let actual_uid = target
700        .uid()
701        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
702    if actual_uid != restore.spec.target_ref.uid {
703        return Err(Error::MissingData(format!(
704            "target UID mismatch: expected {}, got {}",
705            restore.spec.target_ref.uid, actual_uid
706        )));
707    }
708    validate_source(restore)?;
709    if is_remote_source(restore) {
710        validate_backup_ref(restore, &target, ctx).await?;
711    }
712    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
713    {
714        return Err(Error::MissingData(format!(
715            "restoreImage must be the target's pinned Kanidm image (target image is {})",
716            target.spec.image
717        )));
718    }
719    let storage = target
720        .spec
721        .storage
722        .as_ref()
723        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
724    if storage.empty_dir.is_some()
725        || storage.ephemeral.is_some()
726        || storage.volume_claim_template.is_none()
727    {
728        return Err(Error::MissingData(
729            "restore requires PVC-backed Kanidm storage".to_string(),
730        ));
731    }
732    let primaries = target
733        .spec
734        .replica_groups
735        .iter()
736        .filter(|rg| rg.primary_node)
737        .count();
738    if primaries != 1 {
739        return Err(Error::MissingData(
740            "backup/restore requires exactly one primary replica group".to_string(),
741        ));
742    }
743    let ns = restore.namespace().unwrap();
744    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
745        .list(&ListParams::default())
746        .await
747        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
748    if restores.items.iter().any(|other| {
749        other.name_any() != restore.name_any()
750            && other.spec.target_ref.name == restore.spec.target_ref.name
751            && !matches!(
752                other.status.as_ref().map(|s| s.phase),
753                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
754            )
755    }) {
756        return Err(Error::MissingData(
757            "another active restore targets this Kanidm".to_string(),
758        ));
759    }
760    validate_safety_backup_config(restore)?;
761    Ok(target)
762}
763
764fn safe_basename(name: &str) -> bool {
765    !name.is_empty()
766        && name != "."
767        && name != ".."
768        && !name.contains('/')
769        && !name.contains('\\')
770        && !name.contains('\0')
771}
772
773fn mutable_image(image: &str) -> bool {
774    image == "kanidm/server:latest"
775        || image.ends_with(":latest")
776        || (!image.contains('@')
777            && !image
778                .rsplit('/')
779                .next()
780                .is_some_and(|part| part.contains(':')))
781}
782
783async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
784    let ns = restore
785        .namespace()
786        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
787    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
788        .get(&restore.spec.target_ref.name)
789        .await
790        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
791}
792
793async fn mark_restoring(
794    restore: &KanidmRestore,
795    target: &Kanidm,
796    ctx: &RestoreContext,
797) -> Result<()> {
798    let ns = target.namespace().unwrap();
799    let uid = restore
800        .uid()
801        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
802    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
803        .patch(
804            &target.name_any(),
805            &PatchParams::default(),
806            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
807        )
808        .await
809        .map(|_| ())
810        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
811}
812
813async fn clear_restoring(
814    restore: &KanidmRestore,
815    target: &Kanidm,
816    ctx: &RestoreContext,
817) -> Result<()> {
818    let ns = target.namespace().unwrap();
819    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
820        return Ok(());
821    }
822    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
823        .patch(
824            &target.name_any(),
825            &PatchParams::default(),
826            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
827        )
828        .await
829        .map(|_| ())
830        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
831}
832
833async fn patch_status(
834    restore: &KanidmRestore,
835    ctx: &RestoreContext,
836    mut status: KanidmRestoreStatus,
837) -> Result<()> {
838    let ns = restore.namespace().unwrap();
839    let previous_phase = restore.status.as_ref().map(|current| current.phase);
840    let next_phase = status.phase;
841    status.observed_generation = restore.metadata.generation;
842    update_restore_conditions(&mut status, restore.metadata.generation);
843
844    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
845        .patch_status(
846            &restore.name_any(),
847            &PatchParams::default(),
848            &Patch::Merge(json!({"status": &status})),
849        )
850        .await
851        .map_err(|e| {
852            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
853        })?;
854
855    if previous_phase != Some(next_phase) {
856        record_restore_transition(
857            restore,
858            ctx,
859            previous_phase,
860            next_phase,
861            status.message.as_deref(),
862        )
863        .await;
864    }
865    Ok(())
866}
867
868fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
869    let previous = status.conditions.clone();
870    let phase = status.phase;
871    let phase_reason = format!("{phase:?}");
872    let message = status
873        .message
874        .clone()
875        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
876    let terminal_success = phase == KanidmRestorePhase::Completed;
877    let terminal_failure = phase == KanidmRestorePhase::Failed;
878    let progressing = !terminal_success && !terminal_failure;
879
880    let break_glass = previous
881        .iter()
882        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
883        .cloned();
884    status.conditions = vec![
885        restore_condition(
886            &previous,
887            CONDITION_PROGRESSING,
888            if progressing {
889                CONDITION_TRUE
890            } else {
891                CONDITION_FALSE
892            },
893            &phase_reason,
894            &message,
895            generation,
896        ),
897        restore_condition(
898            &previous,
899            CONDITION_READY,
900            if terminal_success {
901                CONDITION_TRUE
902            } else {
903                CONDITION_FALSE
904            },
905            if terminal_success {
906                "RestoreCompleted"
907            } else {
908                "RestoreNotCompleted"
909            },
910            if terminal_success {
911                "Kanidm restore completed successfully."
912            } else {
913                "Kanidm restore has not completed successfully."
914            },
915            generation,
916        ),
917        restore_condition(
918            &previous,
919            CONDITION_FAILED,
920            if terminal_failure {
921                CONDITION_TRUE
922            } else {
923                CONDITION_FALSE
924            },
925            if terminal_failure {
926                "RestoreFailed"
927            } else {
928                "NoRestoreFailure"
929            },
930            if terminal_failure {
931                status
932                    .message
933                    .as_deref()
934                    .unwrap_or("Kanidm restore failed.")
935            } else {
936                "No terminal restore failure has been recorded."
937            },
938            generation,
939        ),
940    ];
941    if let Some(condition) = break_glass {
942        status.conditions.push(condition);
943    }
944}
945
946fn restore_condition(
947    previous: &[Condition],
948    condition_type: &str,
949    condition_status: &str,
950    reason: &str,
951    message: &str,
952    generation: Option<i64>,
953) -> Condition {
954    let last_transition_time = previous
955        .iter()
956        .find(|condition| {
957            condition.type_ == condition_type
958                && condition.status == condition_status
959                && condition.reason == reason
960        })
961        .map(|condition| condition.last_transition_time.clone())
962        .unwrap_or_else(|| Time(Timestamp::now()));
963    Condition {
964        type_: condition_type.to_string(),
965        status: condition_status.to_string(),
966        reason: reason.to_string(),
967        message: message.to_string(),
968        last_transition_time,
969        observed_generation: generation,
970    }
971}
972
973async fn record_restore_transition(
974    restore: &KanidmRestore,
975    ctx: &RestoreContext,
976    previous_phase: Option<KanidmRestorePhase>,
977    phase: KanidmRestorePhase,
978    message: Option<&str>,
979) {
980    if phase == KanidmRestorePhase::Validating {
981        ctx.metrics.attempts.add(1, &[]);
982    }
983
984    let result = match phase {
985        KanidmRestorePhase::Completed => Some("success"),
986        KanidmRestorePhase::Failed => Some("failure"),
987        _ => None,
988    };
989    if let Some(result) = result {
990        let attributes = [KeyValue::new("result", result)];
991        ctx.metrics.outcomes.add(1, &attributes);
992        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
993            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
994            ctx.metrics.duration_seconds.record(elapsed, &attributes);
995        }
996    }
997
998    let reason = if phase == KanidmRestorePhase::Failed {
999        "RestoreFailed"
1000    } else {
1001        "RestorePhaseChanged"
1002    };
1003    let note = message.map(str::to_string).or_else(|| {
1004        Some(format!(
1005            "Kanidm restore phase changed from {} to {phase:?}.",
1006            previous_phase
1007                .map(|previous| format!("{previous:?}"))
1008                .unwrap_or_else(|| "None".to_string())
1009        ))
1010    });
1011    if let Err(error) = ctx
1012        .recorder
1013        .publish(
1014            &Event {
1015                type_: if phase == KanidmRestorePhase::Failed {
1016                    EventType::Warning
1017                } else {
1018                    EventType::Normal
1019                },
1020                reason: reason.to_string(),
1021                note,
1022                action: "Restore".to_string(),
1023                secondary: None,
1024            },
1025            &restore.object_ref(&()),
1026        )
1027        .await
1028    {
1029        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1030    }
1031}
1032
1033async fn set_phase(
1034    restore: &KanidmRestore,
1035    ctx: &RestoreContext,
1036    phase: KanidmRestorePhase,
1037    message: Option<String>,
1038) -> Result<()> {
1039    let mut status = restore.status.clone().unwrap_or_default();
1040    status.phase = phase;
1041    status.message = message;
1042    patch_status(restore, ctx, status).await
1043}
1044
1045async fn fail_after_mutation(
1046    restore: &KanidmRestore,
1047    ctx: &RestoreContext,
1048    message: &str,
1049) -> Result<()> {
1050    set_phase(
1051        restore,
1052        ctx,
1053        KanidmRestorePhase::Failed,
1054        Some(message.to_string()),
1055    )
1056    .await
1057}
1058
1059fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1060    target
1061        .spec
1062        .replica_groups
1063        .iter()
1064        .find(|rg| rg.primary_node)
1065        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1066}
1067
1068async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1069    let ns = target.namespace().unwrap();
1070    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1071    for rg in &target.spec.replica_groups {
1072        let name = target.statefulset_name(&rg.name);
1073        api.patch(
1074            &name,
1075            &PatchParams::default(),
1076            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1077        )
1078        .await
1079        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1080    }
1081    Ok(())
1082}
1083
1084async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1085    let ns = target.namespace().unwrap();
1086    let rg = primary_group(target)?;
1087    let name = target.statefulset_name(&rg.name);
1088    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1089        .patch(
1090            &name,
1091            &PatchParams::default(),
1092            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1093        )
1094        .await
1095        .map(|_| ())
1096        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1097}
1098
1099async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1100    let ns = target.namespace().unwrap();
1101    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1102    for rg in &target.spec.replica_groups {
1103        let name = target.statefulset_name(&rg.name);
1104        api.patch(
1105            &name,
1106            &PatchParams::default(),
1107            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1108        )
1109        .await
1110        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1111    }
1112    Ok(())
1113}
1114
1115async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1116    let ns = target.namespace().unwrap();
1117    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1118        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1119        .await
1120        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1121    Ok(pods.items.is_empty())
1122}
1123
1124async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1125    let ns = target.namespace().unwrap();
1126    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1127    let mut pv_names = BTreeSet::new();
1128    let primary_name = primary_pvc_name(target)?;
1129    let primary_pvc = pvc_api
1130        .get(&primary_name)
1131        .await
1132        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1133    let primary_pv = primary_pvc
1134        .spec
1135        .as_ref()
1136        .and_then(|spec| spec.volume_name.clone())
1137        .ok_or_else(|| {
1138            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1139        })?;
1140    pv_names.insert(primary_pv);
1141
1142    for rg in &target.spec.replica_groups {
1143        let sts = target.statefulset_name(&rg.name);
1144        for ordinal in 0..rg.replicas {
1145            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1146            if name == primary_name {
1147                continue;
1148            }
1149            if let Some(pvc) = pvc_api
1150                .get_opt(&name)
1151                .await
1152                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1153                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1154            {
1155                pv_names.insert(pv_name);
1156            }
1157        }
1158    }
1159
1160    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1161        .list(&ListParams::default())
1162        .await
1163        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1164    Ok(!attachments.items.iter().any(|attachment| {
1165        attachment
1166            .spec
1167            .source
1168            .persistent_volume_name
1169            .as_ref()
1170            .is_some_and(|pv_name| pv_names.contains(pv_name))
1171    }))
1172}
1173
1174async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1175    let ns = target.namespace().unwrap();
1176    let name = target.statefulset_name(&primary_group(target)?.name);
1177    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1178        .get(&name)
1179        .await
1180        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1181    Ok(sts
1182        .status
1183        .as_ref()
1184        .and_then(|s| s.ready_replicas)
1185        .unwrap_or(0)
1186        >= 1)
1187}
1188
1189fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1190    let rg = primary_group(target)?;
1191    Ok(format!(
1192        "{DATA_VOLUME}-{}-0",
1193        target.statefulset_name(&rg.name)
1194    ))
1195}
1196
1197async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1198    let ns = target.namespace().unwrap();
1199    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1200    let mut all_absent = true;
1201    for rg in &target.spec.replica_groups {
1202        let sts = target.statefulset_name(&rg.name);
1203        for ordinal in 0..rg.replicas {
1204            if rg.primary_node && ordinal == 0 {
1205                continue;
1206            }
1207            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1208            match api
1209                .get_opt(&name)
1210                .await
1211                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1212            {
1213                None => {}
1214                Some(pvc) => {
1215                    all_absent = false;
1216                    if pvc.metadata.deletion_timestamp.is_none() {
1217                        match api.delete(&name, &DeleteParams::default()).await {
1218                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1219                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1220                            Err(error) => {
1221                                return Err(Error::kube_error(
1222                                    "delete",
1223                                    "PersistentVolumeClaim",
1224                                    &ns,
1225                                    &name,
1226                                    error,
1227                                ));
1228                            }
1229                        }
1230                    }
1231                }
1232            }
1233        }
1234    }
1235    Ok(all_absent)
1236}
1237
1238async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1239    let ns = target.namespace().unwrap();
1240    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1241    let label_selector = format!(
1242        "{}={},{}={}",
1243        CLUSTER_LABEL,
1244        target.name_any(),
1245        SECRET_TYPE_LABEL,
1246        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1247    );
1248    let secrets = api
1249        .list(&ListParams::default().labels(&label_selector))
1250        .await
1251        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1252    let mut all_absent = secrets.items.is_empty();
1253    for secret in secrets.items {
1254        let name = secret.name_any();
1255        if secret.metadata.deletion_timestamp.is_none() {
1256            match api.delete(&name, &DeleteParams::default()).await {
1257                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1258                Err(kube::Error::Api(status)) if status.code == 404 => {}
1259                Err(error) => {
1260                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1261                }
1262            }
1263            all_absent = false;
1264        }
1265    }
1266    Ok(all_absent)
1267}
1268
1269async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1270    let ns = target.namespace().unwrap();
1271    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1272    let name = format!("{}-admin-passwords", target.name_any());
1273    match api
1274        .get_opt(&name)
1275        .await
1276        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1277    {
1278        None => Ok(true),
1279        Some(secret) => {
1280            if secret.metadata.deletion_timestamp.is_some() {
1281                return Ok(false);
1282            }
1283            match api.delete(&name, &DeleteParams::default()).await {
1284                Ok(_) => {
1285                    debug!(secret = %name, "deleting stale admin secret");
1286                    Ok(false)
1287                }
1288                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1289                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1290            }
1291        }
1292    }
1293}
1294
1295fn restore_job_name(restore: &KanidmRestore) -> String {
1296    format!("{}-restore", restore.name_any())
1297}
1298fn verify_job_name(restore: &KanidmRestore) -> String {
1299    format!("{}-verify", restore.name_any())
1300}
1301
1302fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1303    let origin = target
1304        .spec
1305        .origin
1306        .clone()
1307        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1308    vec![
1309        EnvVar {
1310            name: "KANIDM_DB_PATH".to_string(),
1311            value: Some("/data/kanidm.db".to_string()),
1312            ..Default::default()
1313        },
1314        EnvVar {
1315            name: "KANIDM_DOMAIN".to_string(),
1316            value: Some(target.spec.domain.clone()),
1317            ..Default::default()
1318        },
1319        EnvVar {
1320            name: "KANIDM_ORIGIN".to_string(),
1321            value: Some(origin),
1322            ..Default::default()
1323        },
1324        EnvVar {
1325            name: "KANIDM_TLS_CHAIN".to_string(),
1326            value: Some(format!("{TLS_PATH}/tls.crt")),
1327            ..Default::default()
1328        },
1329        EnvVar {
1330            name: "KANIDM_TLS_KEY".to_string(),
1331            value: Some(format!("{TLS_PATH}/tls.key")),
1332            ..Default::default()
1333        },
1334    ]
1335}
1336
1337async fn ensure_database_job(
1338    restore: &KanidmRestore,
1339    target: &Kanidm,
1340    ctx: &RestoreContext,
1341    name: &str,
1342    verify: bool,
1343) -> Result<()> {
1344    let ns = restore.namespace().unwrap();
1345    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1346    if api
1347        .get_opt(name)
1348        .await
1349        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1350        .is_some()
1351    {
1352        return Ok(());
1353    }
1354    let mut command = vec![
1355        "kanidmd".to_string(),
1356        "database".to_string(),
1357        if verify { "backup" } else { "restore" }.to_string(),
1358    ];
1359    if verify {
1360        command.push("/verify/verification.json.gz".to_string());
1361    } else {
1362        if is_remote_source(restore) {
1363            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1364        } else {
1365            let file_name = restore
1366                .spec
1367                .source
1368                .local
1369                .as_ref()
1370                .map(|l| l.file_name.clone())
1371                .unwrap_or_default();
1372            command.push(format!("{BACKUP_PATH}/{file_name}"));
1373        }
1374    }
1375    let job = Job {
1376        metadata: ObjectMeta {
1377            name: Some(name.to_string()),
1378            namespace: Some(ns.clone()),
1379            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1380            ..Default::default()
1381        },
1382        spec: Some(JobSpec {
1383            backoff_limit: Some(0),
1384            template: PodTemplateSpec {
1385                metadata: Some(ObjectMeta::default()),
1386                spec: Some(PodSpec {
1387                    automount_service_account_token: Some(false),
1388                    restart_policy: Some("Never".to_string()),
1389                    security_context: Some(source_job_pod_security_context(target)),
1390                    containers: vec![Container {
1391                        name: if verify { "verify" } else { "restore" }.to_string(),
1392                        image: Some(restore.spec.restore_image.clone()),
1393                        command: Some(command),
1394                        env: Some(build_kanidm_db_env_vars(target)),
1395                        security_context: Some(kanidm_job_security_context()),
1396                        resources: Some(job_resource_requirements()),
1397                        volume_mounts: Some(vec![
1398                            VolumeMount {
1399                                name: DATA_VOLUME.to_string(),
1400                                mount_path: DATA_PATH.to_string(),
1401                                ..Default::default()
1402                            },
1403                            VolumeMount {
1404                                name: TLS_VOLUME.to_string(),
1405                                mount_path: TLS_PATH.to_string(),
1406                                read_only: Some(true),
1407                                ..Default::default()
1408                            },
1409                            VolumeMount {
1410                                name: "verification-output".to_string(),
1411                                mount_path: "/verify".to_string(),
1412                                ..Default::default()
1413                            },
1414                        ]),
1415                        ..Default::default()
1416                    }],
1417                    volumes: Some(vec![
1418                        Volume {
1419                            name: DATA_VOLUME.to_string(),
1420                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1421                                claim_name: primary_pvc_name(target)?,
1422                                read_only: Some(false),
1423                            }),
1424                            ..Default::default()
1425                        },
1426                        Volume {
1427                            name: TLS_VOLUME.to_string(),
1428                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1429                                secret_name: Some(target.effective_tls_secret_name()),
1430                                ..Default::default()
1431                            }),
1432                            ..Default::default()
1433                        },
1434                        Volume {
1435                            name: "verification-output".to_string(),
1436                            empty_dir: Some(Default::default()),
1437                            ..Default::default()
1438                        },
1439                    ]),
1440                    ..Default::default()
1441                }),
1442            },
1443            ..Default::default()
1444        }),
1445        ..Default::default()
1446    };
1447    api.create(&PostParams::default(), &job)
1448        .await
1449        .map(|_| ())
1450        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1451}
1452
1453enum JobState {
1454    Running,
1455    Complete,
1456    Failed,
1457}
1458
1459async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1460    let ns = restore.namespace().unwrap();
1461    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1462        .get(name)
1463        .await
1464        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1465    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1466        return Ok(JobState::Failed);
1467    }
1468    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1469        return Ok(JobState::Complete);
1470    }
1471    Ok(JobState::Running)
1472}
1473
1474fn is_remote_source(restore: &KanidmRestore) -> bool {
1475    restore.spec.source.backup_ref.is_some()
1476}
1477
1478fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1479    let safety = restore.spec.safety_backup.as_ref();
1480    let skip = safety.is_some_and(|s| s.skip);
1481    !skip
1482}
1483
1484fn validate_source(restore: &KanidmRestore) -> Result<()> {
1485    let has_local = restore.spec.source.local.is_some();
1486    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1487    if has_local && has_backup_ref {
1488        return Err(Error::MissingData(
1489            "source.local and source.backupRef are mutually exclusive".to_string(),
1490        ));
1491    }
1492    if !has_local && !has_backup_ref {
1493        return Err(Error::MissingData(
1494            "source must specify either local or backupRef".to_string(),
1495        ));
1496    }
1497    if let Some(local) = &restore.spec.source.local {
1498        if !safe_basename(&local.file_name) {
1499            return Err(Error::MissingData(
1500                "restore source fileName must be a safe basename".to_string(),
1501            ));
1502        }
1503    }
1504    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1505        if backup_ref.name.is_empty() {
1506            return Err(Error::MissingData(
1507                "source.backupRef.name must not be empty".to_string(),
1508            ));
1509        }
1510    }
1511    Ok(())
1512}
1513
1514fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1515    let safety = restore.spec.safety_backup.as_ref();
1516    let skip = safety.is_some_and(|s| s.skip);
1517    if skip {
1518        let annotations = restore.metadata.annotations.as_ref();
1519        let reason = annotations
1520            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1521            .map(|s| s.trim())
1522            .unwrap_or("");
1523        let approver = annotations
1524            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1525            .map(|s| s.trim())
1526            .unwrap_or("");
1527        if reason.is_empty() {
1528            return Err(Error::MissingData(format!(
1529                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1530            )));
1531        }
1532        if approver.is_empty() {
1533            return Err(Error::MissingData(format!(
1534                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1535            )));
1536        }
1537    }
1538    if is_remote_source(restore) && !skip {
1539        let has_repo = safety
1540            .and_then(|s| s.repository_ref.as_ref())
1541            .is_some_and(|r| !r.name.is_empty());
1542        if !has_repo {
1543            return Err(Error::MissingData(
1544                "remote restore requires safetyBackup.repositoryRef".to_string(),
1545            ));
1546        }
1547    }
1548    Ok(())
1549}
1550
1551fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1552    target
1553        .spec
1554        .replica_groups
1555        .iter()
1556        .map(|rg| ReplicaCountEntry {
1557            group: rg.name.clone(),
1558            replicas: rg.replicas,
1559        })
1560        .collect()
1561}
1562
1563async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564    if let Ok(target) = get_target(restore, ctx).await {
1565        let owns_maintenance =
1566            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1567        if owns_maintenance {
1568            scale_desired(&target, ctx).await?;
1569            clear_restoring(restore, &target, ctx).await?;
1570        }
1571    }
1572    Ok(())
1573}
1574
1575async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1576    ctx.metrics.break_glass_total.add(1, &[]);
1577    let reason = restore
1578        .metadata
1579        .annotations
1580        .as_ref()
1581        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1582        .cloned()
1583        .unwrap_or_default();
1584    let approver = restore
1585        .metadata
1586        .annotations
1587        .as_ref()
1588        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1589        .cloned()
1590        .unwrap_or_default();
1591    let mut status = restore.status.clone().unwrap_or_default();
1592    let break_glass_condition = restore_condition(
1593        &status.conditions,
1594        CONDITION_BREAK_GLASS,
1595        CONDITION_TRUE,
1596        "BreakGlassOverride",
1597        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1598        restore.metadata.generation,
1599    );
1600    status
1601        .conditions
1602        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1603    status.conditions.push(break_glass_condition);
1604    patch_status(restore, ctx, status).await?;
1605    if let Err(error) = ctx
1606        .recorder
1607        .publish(
1608            &Event {
1609                type_: EventType::Warning,
1610                reason: "SafetyBackupSkipped".to_string(),
1611                note: Some(format!(
1612                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1613                )),
1614                action: "Restore".to_string(),
1615                secondary: None,
1616            },
1617            &restore.object_ref(&()),
1618        )
1619        .await
1620    {
1621        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1622    }
1623    warn!(
1624        restore = %restore.name_any(),
1625        reason = %reason,
1626        approver = %approver,
1627        "break-glass safety backup override activated"
1628    );
1629    Ok(())
1630}
1631
1632async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1633    let status = restore.status.as_ref().cloned().unwrap_or_default();
1634    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1635        if let Ok(start_ts) = start.parse::<Timestamp>() {
1636            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1637            ctx.metrics
1638                .safety_backup_duration_seconds
1639                .record(elapsed, &[]);
1640        }
1641    }
1642}
1643
1644fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1645    format!("{}-safety-backup", restore.name_any())
1646}
1647
1648fn source_prep_job_name(restore: &KanidmRestore) -> String {
1649    format!("{}-source-prep", restore.name_any())
1650}
1651
1652fn source_check_job_name(restore: &KanidmRestore) -> String {
1653    format!("{}-source-check", restore.name_any())
1654}
1655
1656async fn ensure_source_check_job(
1657    restore: &KanidmRestore,
1658    target: &Kanidm,
1659    ctx: &RestoreContext,
1660    name: &str,
1661) -> Result<()> {
1662    let ns = restore.namespace().unwrap();
1663    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1664    if api
1665        .get_opt(name)
1666        .await
1667        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1668        .is_some()
1669    {
1670        return Ok(());
1671    }
1672    let file_name = restore
1673        .spec
1674        .source
1675        .local
1676        .as_ref()
1677        .map(|l| l.file_name.clone())
1678        .unwrap_or_default();
1679    let operation_doc = serde_json::json!({
1680        "apiVersion": "backup.kaniop.rs/v1alpha1",
1681        "kind": "OperationDocument",
1682        "operation": "check",
1683        "path": format!("{BACKUP_PATH}/{file_name}"),
1684        "resultPath": "/run/kaniop-result/result.json",
1685        "format": "kanidmJsonGzip",
1686    })
1687    .to_string();
1688    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1689    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1690    let data_mover_image = data_mover_image();
1691    let job = Job {
1692        metadata: ObjectMeta {
1693            name: Some(name.to_string()),
1694            namespace: Some(ns.clone()),
1695            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1696            ..Default::default()
1697        },
1698        spec: Some(JobSpec {
1699            backoff_limit: Some(0),
1700            template: PodTemplateSpec {
1701                metadata: Some(ObjectMeta::default()),
1702                spec: Some(PodSpec {
1703                    automount_service_account_token: Some(false),
1704                    restart_policy: Some("Never".to_string()),
1705                    security_context: Some(source_job_pod_security_context(target)),
1706                    containers: vec![Container {
1707                        name: "source-check".to_string(),
1708                        image: Some(data_mover_image),
1709                        command: Some(vec![
1710                            "/bin/kaniop-data-mover".to_string(),
1711                            "check".to_string(),
1712                            "--operation-doc".to_string(),
1713                            "/run/kaniop/operation.json".to_string(),
1714                        ]),
1715                        security_context: Some(hardened_security_context()),
1716                        resources: Some(job_resource_requirements()),
1717                        env: Some(vec![EnvVar {
1718                            name: "RUST_LOG".to_string(),
1719                            value: Some("info".to_string()),
1720                            ..Default::default()
1721                        }]),
1722                        volume_mounts: Some(vec![
1723                            VolumeMount {
1724                                name: DATA_VOLUME.to_string(),
1725                                mount_path: DATA_PATH.to_string(),
1726                                read_only: Some(true),
1727                                ..Default::default()
1728                            },
1729                            VolumeMount {
1730                                name: "operation".to_string(),
1731                                mount_path: "/run/kaniop".to_string(),
1732                                read_only: Some(true),
1733                                ..Default::default()
1734                            },
1735                            VolumeMount {
1736                                name: "result".to_string(),
1737                                mount_path: "/run/kaniop-result".to_string(),
1738                                ..Default::default()
1739                            },
1740                        ]),
1741                        ..Default::default()
1742                    }],
1743                    volumes: Some(vec![
1744                        Volume {
1745                            name: DATA_VOLUME.to_string(),
1746                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1747                                claim_name: primary_pvc_name(target)?,
1748                                read_only: Some(true),
1749                            }),
1750                            ..Default::default()
1751                        },
1752                        Volume {
1753                            name: "operation".to_string(),
1754                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1755                                name: operation_cm_name,
1756                                ..Default::default()
1757                            }),
1758                            ..Default::default()
1759                        },
1760                        Volume {
1761                            name: "result".to_string(),
1762                            empty_dir: Some(Default::default()),
1763                            ..Default::default()
1764                        },
1765                    ]),
1766                    ..Default::default()
1767                }),
1768            },
1769            ..Default::default()
1770        }),
1771        ..Default::default()
1772    };
1773    api.create(&PostParams::default(), &job)
1774        .await
1775        .map(|_| ())
1776        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1777}
1778
1779const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1780
1781fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1782    let ns = restore.namespace().unwrap_or_default();
1783    let name = restore.name_any();
1784    let namespace_uuid =
1785        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1786    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1787    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1788}
1789
1790fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1791    let ns = restore.namespace().unwrap_or_default();
1792    let kanidm_uid = &restore.spec.target_ref.uid;
1793    let backup_id = compute_safety_backup_id(restore);
1794    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1795}
1796
1797struct VerifiedSafetyBackup {
1798    manifest_key: String,
1799    payload_sha256: String,
1800}
1801
1802async fn read_safety_backup_result(
1803    restore: &KanidmRestore,
1804    ctx: &RestoreContext,
1805    job_name: &str,
1806    expected_backup_id: &str,
1807) -> Result<VerifiedSafetyBackup> {
1808    let ns = restore.namespace().unwrap();
1809    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1810    let pod_list = pods
1811        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1812        .await
1813        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1814    let pod = pod_list.items.first().ok_or_else(|| {
1815        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1816    })?;
1817    let pod_name = pod.name_any();
1818    let container_status = pod
1819        .status
1820        .as_ref()
1821        .and_then(|s| s.container_statuses.as_ref())
1822        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1823        .ok_or_else(|| {
1824            Error::MissingData(format!(
1825                "safety-upload container status not found in pod {pod_name}"
1826            ))
1827        })?;
1828    let termination_message = container_status
1829        .state
1830        .as_ref()
1831        .and_then(|state| state.terminated.as_ref())
1832        .and_then(|t| t.message.as_ref())
1833        .ok_or_else(|| {
1834            Error::MissingData(format!(
1835                "termination message not found in pod {pod_name}; result transport failed"
1836            ))
1837        })?;
1838    if termination_message.is_empty() {
1839        return Err(Error::MissingData(format!(
1840            "termination message is empty in pod {pod_name}; result document was not written"
1841        )));
1842    }
1843    let result_doc = parse_result_document(termination_message).map_err(|e| {
1844        Error::ParseError(format!(
1845            "failed to parse result document from pod {pod_name}: {e}"
1846        ))
1847    })?;
1848    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1849        return Err(Error::ParseError(format!(
1850            "unsupported result document version: {}",
1851            result_doc.api_version
1852        )));
1853    }
1854    if result_doc.kind != "ResultDocument" {
1855        return Err(Error::ParseError(format!(
1856            "invalid result document kind: {}",
1857            result_doc.kind
1858        )));
1859    }
1860    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1861        return Err(Error::ParseError(format!(
1862            "result document operation mismatch: expected '{}', got '{}'",
1863            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1864        )));
1865    }
1866    if !result_doc.success {
1867        return Err(Error::ParseError(format!(
1868            "safety backup reported failure with exit code {:?}",
1869            result_doc.exit_code
1870        )));
1871    }
1872    if result_doc.exit_code != ExitCode::Success {
1873        return Err(Error::ParseError(format!(
1874            "safety backup exit code is not Success: {:?}",
1875            result_doc.exit_code
1876        )));
1877    }
1878    let doc_backup_id = result_doc
1879        .backup_id
1880        .as_deref()
1881        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1882    if doc_backup_id != expected_backup_id {
1883        return Err(Error::ParseError(format!(
1884            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1885        )));
1886    }
1887    let manifest_key = result_doc
1888        .manifest_key
1889        .clone()
1890        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1891    if manifest_key.is_empty() {
1892        return Err(Error::ParseError(
1893            "result document manifestKey is empty".to_string(),
1894        ));
1895    }
1896    let payload_sha256 = result_doc
1897        .payload_sha256
1898        .clone()
1899        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1900    if payload_sha256.is_empty() {
1901        return Err(Error::ParseError(
1902            "result document payloadSha256 is empty".to_string(),
1903        ));
1904    }
1905    info!(
1906        restore = %restore.name_any(),
1907        backup_id = %expected_backup_id,
1908        manifest_key = %manifest_key,
1909        "safety backup result verified"
1910    );
1911    Ok(VerifiedSafetyBackup {
1912        manifest_key,
1913        payload_sha256,
1914    })
1915}
1916
1917struct VerifiedSourcePrepResult {
1918    #[allow(dead_code)]
1919    manifest_key: String,
1920    payload_sha256: String,
1921}
1922
1923async fn read_source_prep_result(
1924    restore: &KanidmRestore,
1925    ctx: &RestoreContext,
1926    job_name: &str,
1927    expected_backup_id: &str,
1928) -> Result<VerifiedSourcePrepResult> {
1929    let ns = restore.namespace().unwrap();
1930    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1931    let pod_list = pods
1932        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1933        .await
1934        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1935    let pod = pod_list.items.first().ok_or_else(|| {
1936        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1937    })?;
1938    let pod_name = pod.name_any();
1939    let container_status = pod
1940        .status
1941        .as_ref()
1942        .and_then(|s| s.container_statuses.as_ref())
1943        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1944        .ok_or_else(|| {
1945            Error::MissingData(format!(
1946                "source-prep container status not found in pod {pod_name}"
1947            ))
1948        })?;
1949    let termination_message = container_status
1950        .state
1951        .as_ref()
1952        .and_then(|state| state.terminated.as_ref())
1953        .and_then(|t| t.message.as_ref())
1954        .ok_or_else(|| {
1955            Error::MissingData(format!(
1956                "termination message not found in pod {pod_name}; result transport failed"
1957            ))
1958        })?;
1959    if termination_message.is_empty() {
1960        return Err(Error::MissingData(format!(
1961            "termination message is empty in pod {pod_name}; result document was not written"
1962        )));
1963    }
1964    let result_doc = parse_result_document(termination_message).map_err(|e| {
1965        Error::ParseError(format!(
1966            "failed to parse result document from pod {pod_name}: {e}"
1967        ))
1968    })?;
1969    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1970        return Err(Error::ParseError(format!(
1971            "unsupported result document version: {}",
1972            result_doc.api_version
1973        )));
1974    }
1975    if result_doc.kind != "ResultDocument" {
1976        return Err(Error::ParseError(format!(
1977            "invalid result document kind: {}",
1978            result_doc.kind
1979        )));
1980    }
1981    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1982        return Err(Error::ParseError(format!(
1983            "result document operation mismatch: expected '{}', got '{}'",
1984            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1985        )));
1986    }
1987    if !result_doc.success {
1988        return Err(Error::ParseError(format!(
1989            "source prep reported failure with exit code {:?}",
1990            result_doc.exit_code
1991        )));
1992    }
1993    if result_doc.exit_code != ExitCode::Success {
1994        return Err(Error::ParseError(format!(
1995            "source prep exit code is not Success: {:?}",
1996            result_doc.exit_code
1997        )));
1998    }
1999    let doc_backup_id = result_doc
2000        .backup_id
2001        .as_deref()
2002        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2003    if doc_backup_id != expected_backup_id {
2004        return Err(Error::ParseError(format!(
2005            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2006        )));
2007    }
2008    let manifest_key = result_doc
2009        .manifest_key
2010        .clone()
2011        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2012    if manifest_key.is_empty() {
2013        return Err(Error::ParseError(
2014            "result document manifestKey is empty".to_string(),
2015        ));
2016    }
2017    let payload_sha256 = result_doc
2018        .payload_sha256
2019        .clone()
2020        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2021    if payload_sha256.is_empty() {
2022        return Err(Error::ParseError(
2023            "result document payloadSha256 is empty".to_string(),
2024        ));
2025    }
2026    info!(
2027        restore = %restore.name_any(),
2028        backup_id = %expected_backup_id,
2029        manifest_key = %manifest_key,
2030        payload_sha256 = %payload_sha256,
2031        "source prep result verified"
2032    );
2033    Ok(VerifiedSourcePrepResult {
2034        manifest_key,
2035        payload_sha256,
2036    })
2037}
2038
2039fn source_job_pod_security_context(
2040    target: &Kanidm,
2041) -> k8s_openapi::api::core::v1::PodSecurityContext {
2042    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2043    security_context.seccomp_profile = Some(SeccompProfile {
2044        type_: "RuntimeDefault".to_string(),
2045        ..Default::default()
2046    });
2047    security_context
2048}
2049
2050fn hardened_security_context() -> SecurityContext {
2051    SecurityContext {
2052        allow_privilege_escalation: Some(false),
2053        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2054            drop: Some(vec!["ALL".to_string()]),
2055            ..Default::default()
2056        }),
2057        read_only_root_filesystem: Some(true),
2058        run_as_non_root: Some(true),
2059        seccomp_profile: Some(SeccompProfile {
2060            type_: "RuntimeDefault".to_string(),
2061            ..Default::default()
2062        }),
2063        ..Default::default()
2064    }
2065}
2066
2067fn kanidm_job_security_context() -> SecurityContext {
2068    SecurityContext {
2069        allow_privilege_escalation: Some(false),
2070        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2071            drop: Some(vec!["ALL".to_string()]),
2072            ..Default::default()
2073        }),
2074        seccomp_profile: Some(SeccompProfile {
2075            type_: "RuntimeDefault".to_string(),
2076            ..Default::default()
2077        }),
2078        ..Default::default()
2079    }
2080}
2081
2082fn job_resource_requirements() -> ResourceRequirements {
2083    ResourceRequirements {
2084        requests: Some(std::collections::BTreeMap::from([
2085            ("cpu".to_string(), Quantity("100m".to_string())),
2086            ("memory".to_string(), Quantity("128Mi".to_string())),
2087        ])),
2088        limits: Some(std::collections::BTreeMap::from([
2089            ("cpu".to_string(), Quantity("2".to_string())),
2090            ("memory".to_string(), Quantity("2Gi".to_string())),
2091        ])),
2092        ..Default::default()
2093    }
2094}
2095
2096async fn ensure_safety_backup_job(
2097    restore: &KanidmRestore,
2098    target: &Kanidm,
2099    ctx: &RestoreContext,
2100    name: &str,
2101    backup_id: &str,
2102    manifest_key: &str,
2103) -> Result<()> {
2104    let ns = restore.namespace().unwrap();
2105    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2106    if api
2107        .get_opt(name)
2108        .await
2109        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2110        .is_some()
2111    {
2112        return Ok(());
2113    }
2114    let safety = restore
2115        .spec
2116        .safety_backup
2117        .as_ref()
2118        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2119    let repo_ref = safety
2120        .repository_ref
2121        .as_ref()
2122        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2123    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2124        .get(&repo_ref.name)
2125        .await
2126        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2127    let _namespace_uid = &ns;
2128    let _kanidm_uid = &restore.spec.target_ref.uid;
2129    let endpoint = &repo.spec.s3.endpoint;
2130    let region = &repo.spec.s3.region;
2131    let operation_doc = build_safety_upload_operation_doc(
2132        restore,
2133        target,
2134        backup_id,
2135        manifest_key,
2136        &repo.spec.s3.bucket,
2137        &repo.spec.s3.prefix,
2138        endpoint,
2139        region,
2140        repo.spec.s3.force_path_style,
2141        repo.spec.s3.insecure,
2142        repo.spec
2143            .s3
2144            .ca_bundle_ref
2145            .as_deref()
2146            .map(|_| ca_bundle_path())
2147            .as_deref(),
2148        repo.spec.encryption.as_ref(),
2149    )?;
2150    let operation_cm_name = format!("{name}-op");
2151    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2152    let auth_method = &repo.spec.authentication.writer;
2153    let data_mover_image = data_mover_image();
2154    let init_command = vec![
2155        "kanidmd".to_string(),
2156        "database".to_string(),
2157        "backup".to_string(),
2158        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2159    ];
2160    let upload_script = format!(
2161        r#"set -eu
2162/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2163RESULT_FILE="/run/kaniop-result/result.json"
2164TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2165if [ ! -f "$RESULT_FILE" ]; then
2166  echo "ERROR: result document not found at $RESULT_FILE"
2167  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2168  exit 1
2169fi
2170if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2171  echo "ERROR: safety backup upload reported failure"
2172  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2173  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2174  exit 1
2175fi
2176dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2177echo "safety backup upload completed successfully"
2178"#
2179    );
2180    let job = Job {
2181        metadata: ObjectMeta {
2182            name: Some(name.to_string()),
2183            namespace: Some(ns.clone()),
2184            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2185            ..Default::default()
2186        },
2187        spec: Some(JobSpec {
2188            backoff_limit: Some(0),
2189            template: PodTemplateSpec {
2190                metadata: Some(ObjectMeta::default()),
2191                spec: Some(PodSpec {
2192                    automount_service_account_token: Some(false),
2193                    restart_policy: Some("Never".to_string()),
2194                    security_context: Some(source_job_pod_security_context(target)),
2195                    init_containers: Some(vec![Container {
2196                        name: "safety-backup".to_string(),
2197                        image: Some(restore.spec.restore_image.clone()),
2198                        command: Some(init_command),
2199                        security_context: Some(SecurityContext {
2200                            allow_privilege_escalation: Some(false),
2201                            capabilities: Some(Capabilities {
2202                                drop: Some(vec!["ALL".to_string()]),
2203                                ..Default::default()
2204                            }),
2205                            read_only_root_filesystem: Some(true),
2206                            seccomp_profile: Some(SeccompProfile {
2207                                type_: "RuntimeDefault".to_string(),
2208                                ..Default::default()
2209                            }),
2210                            ..Default::default()
2211                        }),
2212                        resources: Some(job_resource_requirements()),
2213                        env: Some(build_kanidm_db_env_vars(target)),
2214                        volume_mounts: Some(vec![
2215                            VolumeMount {
2216                                name: DATA_VOLUME.to_string(),
2217                                mount_path: DATA_PATH.to_string(),
2218                                read_only: Some(false),
2219                                ..Default::default()
2220                            },
2221                            VolumeMount {
2222                                name: TLS_VOLUME.to_string(),
2223                                mount_path: TLS_PATH.to_string(),
2224                                read_only: Some(true),
2225                                ..Default::default()
2226                            },
2227                            VolumeMount {
2228                                name: SHARED_VOLUME.to_string(),
2229                                mount_path: SHARED_VOL_PATH.to_string(),
2230                                ..Default::default()
2231                            },
2232                        ]),
2233                        ..Default::default()
2234                    }]),
2235                    containers: vec![Container {
2236                        name: "safety-upload".to_string(),
2237                        image: Some(data_mover_image),
2238                        command: Some(vec!["/bin/sh".to_string()]),
2239                        args: Some(vec!["-c".to_string(), upload_script]),
2240                        security_context: Some(hardened_security_context()),
2241                        resources: Some(job_resource_requirements()),
2242                        env: {
2243                            let mut env_vars =
2244                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2245                            env_vars.push(EnvVar {
2246                                name: "RUST_LOG".to_string(),
2247                                value: Some("info".to_string()),
2248                                ..Default::default()
2249                            });
2250                            if repo.spec.s3.ca_bundle_ref.is_some() {
2251                                env_vars.push(ca_bundle_env_var());
2252                            }
2253                            env_vars.extend(build_encryption_env_vars(
2254                                repo.spec
2255                                    .encryption
2256                                    .as_ref()
2257                                    .and_then(|e| e.key_ref.as_ref()),
2258                            ));
2259                            Some(env_vars)
2260                        },
2261                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2262                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2263                        volume_mounts: {
2264                            let mut mounts = vec![
2265                                VolumeMount {
2266                                    name: SHARED_VOLUME.to_string(),
2267                                    mount_path: SHARED_VOL_PATH.to_string(),
2268                                    read_only: Some(true),
2269                                    ..Default::default()
2270                                },
2271                                VolumeMount {
2272                                    name: "operation".to_string(),
2273                                    mount_path: "/run/kaniop".to_string(),
2274                                    read_only: Some(true),
2275                                    ..Default::default()
2276                                },
2277                                VolumeMount {
2278                                    name: "result".to_string(),
2279                                    mount_path: "/run/kaniop-result".to_string(),
2280                                    ..Default::default()
2281                                },
2282                            ];
2283                            mounts.extend(build_auth_volume_mounts(auth_method));
2284                            if repo.spec.s3.ca_bundle_ref.is_some() {
2285                                mounts.push(build_ca_bundle_volume_mount());
2286                            }
2287                            Some(mounts)
2288                        },
2289                        ..Default::default()
2290                    }],
2291                    volumes: {
2292                        let mut vols = vec![
2293                            Volume {
2294                                name: DATA_VOLUME.to_string(),
2295                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2296                                    claim_name: primary_pvc_name(target)?,
2297                                    read_only: Some(false),
2298                                }),
2299                                ..Default::default()
2300                            },
2301                            Volume {
2302                                name: TLS_VOLUME.to_string(),
2303                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2304                                    secret_name: Some(target.effective_tls_secret_name()),
2305                                    ..Default::default()
2306                                }),
2307                                ..Default::default()
2308                            },
2309                            Volume {
2310                                name: SHARED_VOLUME.to_string(),
2311                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2312                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2313                                    ..Default::default()
2314                                }),
2315                                ..Default::default()
2316                            },
2317                            Volume {
2318                                name: "operation".to_string(),
2319                                config_map: Some(ConfigMapVolumeSource {
2320                                    name: operation_cm_name,
2321                                    ..Default::default()
2322                                }),
2323                                ..Default::default()
2324                            },
2325                            Volume {
2326                                name: "result".to_string(),
2327                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2328                                    size_limit: Some(Quantity("16Mi".to_string())),
2329                                    ..Default::default()
2330                                }),
2331                                ..Default::default()
2332                            },
2333                        ];
2334                        vols.extend(build_auth_volumes(auth_method));
2335                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2336                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2337                        }
2338                        Some(vols)
2339                    },
2340                    ..Default::default()
2341                }),
2342            },
2343            ..Default::default()
2344        }),
2345        ..Default::default()
2346    };
2347    api.create(&PostParams::default(), &job)
2348        .await
2349        .map(|_| ())
2350        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2351}
2352
2353async fn ensure_source_prep_job(
2354    restore: &KanidmRestore,
2355    target: &Kanidm,
2356    ctx: &RestoreContext,
2357    name: &str,
2358) -> Result<()> {
2359    let ns = restore.namespace().unwrap();
2360    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2361    if api
2362        .get_opt(name)
2363        .await
2364        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2365        .is_some()
2366    {
2367        return Ok(());
2368    }
2369    let backup_name = restore
2370        .spec
2371        .source
2372        .backup_ref
2373        .as_ref()
2374        .map(|b| b.name.clone())
2375        .unwrap_or_default();
2376    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2377        .get(&backup_name)
2378        .await
2379        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2380    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2381        .get(&backup.spec.repository_ref.name)
2382        .await
2383        .map_err(|e| {
2384            Error::kube_error(
2385                "get",
2386                "KanidmBackupRepository",
2387                &ns,
2388                &backup.spec.repository_ref.name,
2389                e,
2390            )
2391        })?;
2392    let endpoint = &repo.spec.s3.endpoint;
2393    let region = &repo.spec.s3.region;
2394    let operation_doc = build_download_operation_doc(
2395        restore,
2396        target,
2397        &backup.spec.manifest_key,
2398        &backup.spec.backup_id,
2399        &repo.spec.s3.bucket,
2400        &repo.spec.s3.prefix,
2401        endpoint,
2402        region,
2403        repo.spec.s3.force_path_style,
2404        repo.spec.s3.insecure,
2405        repo.spec
2406            .s3
2407            .ca_bundle_ref
2408            .as_deref()
2409            .map(|_| ca_bundle_path())
2410            .as_deref(),
2411        repo.spec.encryption.as_ref(),
2412    );
2413    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2414    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2415    let auth_method = &repo.spec.authentication.reader;
2416    let data_mover_image = data_mover_image();
2417    let download_script = format!(
2418        r#"set -eu
2419/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2420RESULT_FILE="/run/kaniop-result/result.json"
2421TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2422if [ ! -f "$RESULT_FILE" ]; then
2423  echo "ERROR: result document not found at $RESULT_FILE"
2424  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2425  exit 1
2426fi
2427if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2428  echo "ERROR: source preparation download reported failure"
2429  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2430  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2431  exit 1
2432fi
2433dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2434echo "source preparation download completed successfully"
2435"#
2436    );
2437    let job = Job {
2438        metadata: ObjectMeta {
2439            name: Some(name.to_string()),
2440            namespace: Some(ns.clone()),
2441            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2442            ..Default::default()
2443        },
2444        spec: Some(JobSpec {
2445            backoff_limit: Some(0),
2446            template: PodTemplateSpec {
2447                metadata: Some(ObjectMeta::default()),
2448                spec: Some(PodSpec {
2449                    automount_service_account_token: Some(false),
2450                    restart_policy: Some("Never".to_string()),
2451                    security_context: Some(source_job_pod_security_context(target)),
2452                    containers: vec![Container {
2453                        name: "source-prep".to_string(),
2454                        image: Some(data_mover_image),
2455                        command: Some(vec!["/bin/sh".to_string()]),
2456                        args: Some(vec!["-c".to_string(), download_script]),
2457                        security_context: Some(hardened_security_context()),
2458                        resources: Some(job_resource_requirements()),
2459                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2460                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2461                        env: {
2462                            let mut env_vars = build_auth_env_vars(
2463                                auth_method,
2464                                &backup.spec.repository_ref.name,
2465                                AuthRole::Reader,
2466                            );
2467                            env_vars.push(EnvVar {
2468                                name: "RUST_LOG".to_string(),
2469                                value: Some("info".to_string()),
2470                                ..Default::default()
2471                            });
2472                            if repo.spec.s3.ca_bundle_ref.is_some() {
2473                                env_vars.push(ca_bundle_env_var());
2474                            }
2475                            env_vars.extend(build_encryption_env_vars(
2476                                repo.spec
2477                                    .encryption
2478                                    .as_ref()
2479                                    .and_then(|e| e.key_ref.as_ref()),
2480                            ));
2481                            Some(env_vars)
2482                        },
2483                        volume_mounts: {
2484                            let mut mounts = vec![
2485                                VolumeMount {
2486                                    name: DATA_VOLUME.to_string(),
2487                                    mount_path: DATA_PATH.to_string(),
2488                                    ..Default::default()
2489                                },
2490                                VolumeMount {
2491                                    name: "operation".to_string(),
2492                                    mount_path: "/run/kaniop".to_string(),
2493                                    read_only: Some(true),
2494                                    ..Default::default()
2495                                },
2496                                VolumeMount {
2497                                    name: "result".to_string(),
2498                                    mount_path: "/run/kaniop-result".to_string(),
2499                                    ..Default::default()
2500                                },
2501                            ];
2502                            mounts.extend(build_auth_volume_mounts(auth_method));
2503                            if repo.spec.s3.ca_bundle_ref.is_some() {
2504                                mounts.push(build_ca_bundle_volume_mount());
2505                            }
2506                            Some(mounts)
2507                        },
2508                        ..Default::default()
2509                    }],
2510                    volumes: {
2511                        let mut vols = vec![
2512                            Volume {
2513                                name: DATA_VOLUME.to_string(),
2514                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2515                                    claim_name: primary_pvc_name(target)?,
2516                                    read_only: Some(false),
2517                                }),
2518                                ..Default::default()
2519                            },
2520                            Volume {
2521                                name: "operation".to_string(),
2522                                config_map: Some(ConfigMapVolumeSource {
2523                                    name: operation_cm_name,
2524                                    ..Default::default()
2525                                }),
2526                                ..Default::default()
2527                            },
2528                            Volume {
2529                                name: "result".to_string(),
2530                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2531                                    size_limit: Some(Quantity("16Mi".to_string())),
2532                                    ..Default::default()
2533                                }),
2534                                ..Default::default()
2535                            },
2536                        ];
2537                        vols.extend(build_auth_volumes(auth_method));
2538                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2539                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2540                        }
2541                        Some(vols)
2542                    },
2543                    ..Default::default()
2544                }),
2545            },
2546            ..Default::default()
2547        }),
2548        ..Default::default()
2549    };
2550    api.create(&PostParams::default(), &job)
2551        .await
2552        .map(|_| ())
2553        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2554}
2555
2556#[allow(clippy::too_many_arguments)]
2557fn build_safety_upload_operation_doc(
2558    restore: &KanidmRestore,
2559    target: &Kanidm,
2560    backup_id: &str,
2561    manifest_key: &str,
2562    bucket: &str,
2563    prefix: &str,
2564    endpoint: &str,
2565    region: &str,
2566    force_path_style: bool,
2567    insecure: bool,
2568    ca_bundle_path: Option<&str>,
2569    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2570) -> Result<String> {
2571    let enc_mode =
2572        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2573    let enc_key_id = encryption
2574        .and_then(|e| e.key_id.as_ref())
2575        .map(|s| serde_json::Value::String(s.clone()));
2576    let op = UploadOperation {
2577        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2578        bucket: bucket.to_string(),
2579        prefix: prefix.to_string(),
2580        endpoint: endpoint.to_string(),
2581        region: region.to_string(),
2582        force_path_style,
2583        insecure,
2584        ca_bundle_path: ca_bundle_path.map(str::to_string),
2585        backup_id: backup_id.to_string(),
2586        namespace_uid: restore.namespace().unwrap_or_default(),
2587        kanidm_uid: restore.spec.target_ref.uid.clone(),
2588        kanidm_name: restore.spec.target_ref.name.clone(),
2589        domain: target.spec.domain.clone(),
2590        kanidm_version: target
2591            .status
2592            .as_ref()
2593            .and_then(|s| s.version.as_ref())
2594            .map(|v| v.image_tag.clone())
2595            .unwrap_or_default(),
2596        image_digest: None,
2597        consistency: "kanidm-offline".to_string(),
2598        reason: "restore-safety".to_string(),
2599        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2600        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2601        result_path: "/run/kaniop-result/result.json".to_string(),
2602        max_concurrent_parts: 4,
2603        max_retries: 3,
2604    };
2605    let _ = manifest_key;
2606    let doc = OperationDocument {
2607        api_version: OPERATION_DOC_VERSION.to_string(),
2608        kind: "OperationDocument".to_string(),
2609        spec: OperationSpec::Upload(op),
2610    };
2611    serde_json::to_string(&doc)
2612        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2613}
2614
2615async fn validate_backup_ref(
2616    restore: &KanidmRestore,
2617    target: &Kanidm,
2618    ctx: &RestoreContext,
2619) -> Result<()> {
2620    let backup_ref = restore
2621        .spec
2622        .source
2623        .backup_ref
2624        .as_ref()
2625        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2626    let ns = restore.namespace().unwrap();
2627    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2628        .get(&backup_ref.name)
2629        .await
2630        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2631    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2632        return Err(Error::MissingData(format!(
2633            "KanidmBackup '{}' is not Ready (phase: {:?})",
2634            backup_ref.name,
2635            backup.status.as_ref().map(|s| &s.phase)
2636        )));
2637    }
2638    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2639        return Err(Error::MissingData(format!(
2640            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2641            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2642        )));
2643    }
2644    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2645        return Err(Error::MissingData(format!(
2646            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2647            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2648        )));
2649    }
2650    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2651        .get(&backup.spec.repository_ref.name)
2652        .await
2653        .map_err(|e| {
2654            Error::kube_error(
2655                "get",
2656                "KanidmBackupRepository",
2657                &ns,
2658                &backup.spec.repository_ref.name,
2659                e,
2660            )
2661        })?;
2662    if !has_accepted_condition(
2663        &repo
2664            .status
2665            .as_ref()
2666            .map(|s| &s.conditions)
2667            .cloned()
2668            .unwrap_or_default(),
2669    ) {
2670        return Err(Error::MissingData(format!(
2671            "KanidmBackupRepository '{}' configuration has not been accepted",
2672            backup.spec.repository_ref.name
2673        )));
2674    }
2675    validate_backup_compatibility(restore, target, &backup)?;
2676    Ok(())
2677}
2678
2679fn has_accepted_condition(conditions: &[Condition]) -> bool {
2680    conditions
2681        .iter()
2682        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2683}
2684
2685fn validate_backup_compatibility(
2686    restore: &KanidmRestore,
2687    target: &Kanidm,
2688    backup: &KanidmBackup,
2689) -> Result<()> {
2690    let status = backup.status.as_ref();
2691    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2692        let target_version = target
2693            .status
2694            .as_ref()
2695            .and_then(|s| s.version.as_ref())
2696            .map(|v| v.image_tag.as_str());
2697        if let Some(tv) = target_version {
2698            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2699                return Err(Error::MissingData(format!(
2700                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2701                )));
2702            }
2703        }
2704    }
2705    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2706        if !backup_digest.is_empty()
2707            && restore.spec.restore_image.contains('@')
2708            && !restore.spec.restore_image.contains(backup_digest)
2709        {
2710            return Err(Error::MissingData(format!(
2711                "restore image digest does not match backup image digest '{backup_digest}'"
2712            )));
2713        }
2714    }
2715    Ok(())
2716}
2717
2718#[allow(clippy::too_many_arguments)]
2719fn build_download_operation_doc(
2720    restore: &KanidmRestore,
2721    target: &Kanidm,
2722    manifest_key: &str,
2723    expected_backup_id: &str,
2724    bucket: &str,
2725    prefix: &str,
2726    endpoint: &str,
2727    region: &str,
2728    force_path_style: bool,
2729    insecure: bool,
2730    ca_bundle_path: Option<&str>,
2731    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2732) -> String {
2733    let enc_mode =
2734        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2735    let enc_key_id = encryption
2736        .and_then(|e| e.key_id.as_ref())
2737        .map(|s| serde_json::Value::String(s.clone()));
2738    serde_json::json!({
2739        "apiVersion": "backup.kaniop.rs/v1alpha1",
2740        "kind": "OperationDocument",
2741        "operation": "download",
2742        "manifestKey": manifest_key,
2743        "bucket": bucket,
2744        "prefix": prefix,
2745        "endpoint": endpoint,
2746        "region": region,
2747        "forcePathStyle": force_path_style,
2748        "insecure": insecure,
2749        "caBundlePath": ca_bundle_path,
2750        "expectedBackupId": expected_backup_id,
2751        "expectedKanidmUid": restore.spec.target_ref.uid,
2752        "expectedDomain": target.spec.domain,
2753        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2754        "resultPath": "/run/kaniop-result/result.json",
2755        "maxRetries": 3,
2756        "encryptionMode": enc_mode,
2757        "encryptionKeyId": enc_key_id,
2758    })
2759    .to_string()
2760}
2761
2762async fn ensure_operation_configmap(
2763    restore: &KanidmRestore,
2764    name: &str,
2765    data: &str,
2766    ns: &str,
2767    ctx: &RestoreContext,
2768) -> Result<()> {
2769    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2770    if api
2771        .get_opt(name)
2772        .await
2773        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2774        .is_some()
2775    {
2776        return Ok(());
2777    }
2778    let cm = ConfigMap {
2779        metadata: ObjectMeta {
2780            name: Some(name.to_string()),
2781            namespace: Some(ns.to_string()),
2782            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2783            ..Default::default()
2784        },
2785        data: Some(std::collections::BTreeMap::from([(
2786            "operation.json".to_string(),
2787            data.to_string(),
2788        )])),
2789        ..Default::default()
2790    };
2791    api.create(&PostParams::default(), &cm)
2792        .await
2793        .map(|_| ())
2794        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2795}
2796
2797#[cfg(test)]
2798mod tests {
2799    use super::{
2800        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2801        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2802        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2803        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2804        hardened_security_context, has_accepted_condition, is_remote_source,
2805        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2806        validate_safety_backup_config, validate_source,
2807    };
2808    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2809    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2810    use kube::api::ObjectMeta as ApiObjectMeta;
2811    use std::collections::BTreeMap;
2812
2813    fn make_restore(
2814        source: KanidmRestoreSource,
2815        safety: Option<SafetyBackupConfig>,
2816    ) -> KanidmRestore {
2817        KanidmRestore {
2818            metadata: ApiObjectMeta {
2819                name: Some("test-restore".to_string()),
2820                namespace: Some("default".to_string()),
2821                ..Default::default()
2822            },
2823            spec: KanidmRestoreSpec {
2824                target_ref: KanidmRestoreTargetRef {
2825                    name: "corp-idm".to_string(),
2826                    uid: "test-uid-123".to_string(),
2827                },
2828                source,
2829                restore_image: "kanidm/server@sha256:abc".to_string(),
2830                safety_backup: safety,
2831            },
2832            status: None,
2833        }
2834    }
2835
2836    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2837        super::super::crd::Kanidm {
2838            status: Some(super::super::crd::KanidmStatus {
2839                available_replicas: 1,
2840                replicas: 1,
2841                unavailable_replicas: 0,
2842                updated_replicas: 1,
2843                replica_statuses: vec![],
2844                replica_column: "1/1".to_string(),
2845                secret_name: None,
2846                version: Some(super::super::crd::KanidmVersionStatus {
2847                    image_tag: image_tag.to_string(),
2848                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2849                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2850                }),
2851                domain_appearance_image: None,
2852                mail_sender: None,
2853                conditions: None,
2854            }),
2855            ..Default::default()
2856        }
2857    }
2858
2859    #[test]
2860    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2861        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2862    }
2863
2864    #[test]
2865    fn completed_restore_sets_ready_condition() {
2866        let mut status = KanidmRestoreStatus {
2867            phase: KanidmRestorePhase::Completed,
2868            ..Default::default()
2869        };
2870        super::update_restore_conditions(&mut status, Some(7));
2871        assert!(status.conditions.iter().any(|condition| {
2872            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2873        }));
2874        assert!(!status.conditions.iter().any(|condition| {
2875            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2876        }));
2877    }
2878
2879    #[test]
2880    fn failed_restore_sets_failed_condition() {
2881        let mut status = KanidmRestoreStatus {
2882            phase: KanidmRestorePhase::Failed,
2883            message: Some("verification failed".to_string()),
2884            ..Default::default()
2885        };
2886        super::update_restore_conditions(&mut status, Some(8));
2887        assert!(status.conditions.iter().any(|condition| {
2888            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2889        }));
2890    }
2891
2892    #[test]
2893    fn restore_filename_is_confined_to_backup_directory() {
2894        assert!(safe_basename("backup.json.gz"));
2895        assert!(!safe_basename("../backup.json.gz"));
2896        assert!(!safe_basename("a/b"));
2897        assert!(!safe_basename(""));
2898    }
2899
2900    #[test]
2901    fn restore_image_rejects_latest_and_unpinned_names() {
2902        assert!(mutable_image("kanidm/server:latest"));
2903        assert!(mutable_image("kanidm/server"));
2904        assert!(!mutable_image("kanidm/server:1.10.0"));
2905        assert!(!mutable_image("kanidm/server@sha256:abc"));
2906    }
2907
2908    #[test]
2909    fn source_validation_rejects_both_local_and_backup_ref() {
2910        let source = KanidmRestoreSource {
2911            local: Some(super::KanidmRestoreLocalSource {
2912                file_name: "backup.json".to_string(),
2913            }),
2914            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2915                name: "some-backup".to_string(),
2916            }),
2917        };
2918        let restore = make_restore(source, None);
2919        assert!(validate_source(&restore).is_err());
2920    }
2921
2922    #[test]
2923    fn source_validation_rejects_empty_source() {
2924        let source = KanidmRestoreSource {
2925            local: None,
2926            backup_ref: None,
2927        };
2928        let restore = make_restore(source, None);
2929        assert!(validate_source(&restore).is_err());
2930    }
2931
2932    #[test]
2933    fn source_validation_accepts_local_source() {
2934        let source = KanidmRestoreSource {
2935            local: Some(super::KanidmRestoreLocalSource {
2936                file_name: "backup.json".to_string(),
2937            }),
2938            backup_ref: None,
2939        };
2940        let restore = make_restore(source, None);
2941        assert!(validate_source(&restore).is_ok());
2942    }
2943
2944    #[test]
2945    fn source_validation_accepts_backup_ref_source() {
2946        let source = KanidmRestoreSource {
2947            local: None,
2948            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2949                name: "corp-idm-019c7c76".to_string(),
2950            }),
2951        };
2952        let restore = make_restore(source, None);
2953        assert!(validate_source(&restore).is_ok());
2954    }
2955
2956    #[test]
2957    fn source_validation_rejects_empty_backup_ref_name() {
2958        let source = KanidmRestoreSource {
2959            local: None,
2960            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2961                name: "".to_string(),
2962            }),
2963        };
2964        let restore = make_restore(source, None);
2965        assert!(validate_source(&restore).is_err());
2966    }
2967
2968    #[test]
2969    fn is_remote_source_true_for_backup_ref() {
2970        let source = KanidmRestoreSource {
2971            local: None,
2972            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2973                name: "backup-1".to_string(),
2974            }),
2975        };
2976        let restore = make_restore(source, None);
2977        assert!(is_remote_source(&restore));
2978    }
2979
2980    #[test]
2981    fn is_remote_source_false_for_local() {
2982        let source = KanidmRestoreSource {
2983            local: Some(super::KanidmRestoreLocalSource {
2984                file_name: "backup.json".to_string(),
2985            }),
2986            backup_ref: None,
2987        };
2988        let restore = make_restore(source, None);
2989        assert!(!is_remote_source(&restore));
2990    }
2991
2992    #[test]
2993    fn requires_safety_backup_default_true() {
2994        let source = KanidmRestoreSource {
2995            local: None,
2996            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2997                name: "backup-1".to_string(),
2998            }),
2999        };
3000        let restore = make_restore(source, None);
3001        assert!(requires_safety_backup(&restore));
3002    }
3003
3004    #[test]
3005    fn requires_safety_backup_false_when_skip() {
3006        let source = KanidmRestoreSource {
3007            local: None,
3008            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3009                name: "backup-1".to_string(),
3010            }),
3011        };
3012        let safety = SafetyBackupConfig {
3013            repository_ref: Some(SafetyBackupRepositoryRef {
3014                name: "offsite".to_string(),
3015            }),
3016            skip: true,
3017        };
3018        let restore = make_restore(source, Some(safety));
3019        assert!(!requires_safety_backup(&restore));
3020    }
3021
3022    #[test]
3023    fn break_glass_validation_rejects_skip_without_reason() {
3024        let source = KanidmRestoreSource {
3025            local: None,
3026            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3027                name: "backup-1".to_string(),
3028            }),
3029        };
3030        let safety = SafetyBackupConfig {
3031            repository_ref: Some(SafetyBackupRepositoryRef {
3032                name: "offsite".to_string(),
3033            }),
3034            skip: true,
3035        };
3036        let mut restore = make_restore(source, Some(safety));
3037        restore.metadata.annotations = Some(BTreeMap::from([(
3038            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3039            "commander@example.com".to_string(),
3040        )]));
3041        assert!(validate_safety_backup_config(&restore).is_err());
3042    }
3043
3044    #[test]
3045    fn break_glass_validation_rejects_skip_without_approver() {
3046        let source = KanidmRestoreSource {
3047            local: None,
3048            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3049                name: "backup-1".to_string(),
3050            }),
3051        };
3052        let safety = SafetyBackupConfig {
3053            repository_ref: Some(SafetyBackupRepositoryRef {
3054                name: "offsite".to_string(),
3055            }),
3056            skip: true,
3057        };
3058        let mut restore = make_restore(source, Some(safety));
3059        restore.metadata.annotations = Some(BTreeMap::from([(
3060            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3061            "PVC is unreadable".to_string(),
3062        )]));
3063        assert!(validate_safety_backup_config(&restore).is_err());
3064    }
3065
3066    #[test]
3067    fn break_glass_validation_accepts_valid_annotations() {
3068        let source = KanidmRestoreSource {
3069            local: None,
3070            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3071                name: "backup-1".to_string(),
3072            }),
3073        };
3074        let safety = SafetyBackupConfig {
3075            repository_ref: Some(SafetyBackupRepositoryRef {
3076                name: "offsite".to_string(),
3077            }),
3078            skip: true,
3079        };
3080        let mut restore = make_restore(source, Some(safety));
3081        restore.metadata.annotations = Some(BTreeMap::from([
3082            (
3083                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3084                "PVC is unreadable".to_string(),
3085            ),
3086            (
3087                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3088                "commander@example.com".to_string(),
3089            ),
3090        ]));
3091        assert!(validate_safety_backup_config(&restore).is_ok());
3092    }
3093
3094    #[test]
3095    fn remote_source_requires_repository_ref_when_not_skipping() {
3096        let source = KanidmRestoreSource {
3097            local: None,
3098            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3099                name: "backup-1".to_string(),
3100            }),
3101        };
3102        let safety = SafetyBackupConfig {
3103            repository_ref: None,
3104            skip: false,
3105        };
3106        let restore = make_restore(source, Some(safety));
3107        assert!(validate_safety_backup_config(&restore).is_err());
3108    }
3109
3110    #[test]
3111    fn remote_source_accepts_repository_ref() {
3112        let source = KanidmRestoreSource {
3113            local: None,
3114            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3115                name: "backup-1".to_string(),
3116            }),
3117        };
3118        let safety = SafetyBackupConfig {
3119            repository_ref: Some(SafetyBackupRepositoryRef {
3120                name: "offsite".to_string(),
3121            }),
3122            skip: false,
3123        };
3124        let restore = make_restore(source, Some(safety));
3125        assert!(validate_safety_backup_config(&restore).is_ok());
3126    }
3127
3128    #[test]
3129    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3130        let phases = vec![
3131            KanidmRestorePhase::Pending,
3132            KanidmRestorePhase::Validating,
3133            KanidmRestorePhase::Quiescing,
3134            KanidmRestorePhase::SafetyBackup,
3135            KanidmRestorePhase::PreparingSource,
3136            KanidmRestorePhase::RestoringPrimary,
3137            KanidmRestorePhase::Verifying,
3138            KanidmRestorePhase::RebuildingReplicas,
3139            KanidmRestorePhase::Resuming,
3140            KanidmRestorePhase::Completed,
3141        ];
3142        assert_eq!(phases.len(), 10);
3143        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3144        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3145    }
3146
3147    #[test]
3148    fn status_default_has_empty_phase_timestamps() {
3149        let status = KanidmRestoreStatus::default();
3150        assert!(status.phase_timestamps.is_empty());
3151    }
3152
3153    #[test]
3154    fn status_default_has_empty_original_replicas() {
3155        let status = KanidmRestoreStatus::default();
3156        assert!(status.original_replicas.is_empty());
3157    }
3158
3159    #[test]
3160    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3161        let mut target = super::super::crd::Kanidm::default();
3162        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3163            run_as_non_root: Some(false),
3164            run_as_user: Some(389),
3165            run_as_group: Some(389),
3166            fs_group: Some(389),
3167            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3168            supplemental_groups: Some(vec![390, 391]),
3169            ..Default::default()
3170        });
3171
3172        let ctx = super::source_job_pod_security_context(&target);
3173
3174        assert_eq!(ctx.run_as_non_root, Some(false));
3175        assert_eq!(ctx.run_as_user, Some(389));
3176        assert_eq!(ctx.run_as_group, Some(389));
3177        assert_eq!(ctx.fs_group, Some(389));
3178        assert_eq!(
3179            ctx.fs_group_change_policy.as_deref(),
3180            Some("OnRootMismatch")
3181        );
3182        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3183        assert_eq!(
3184            ctx.seccomp_profile
3185                .as_ref()
3186                .map(|profile| profile.type_.as_str()),
3187            Some("RuntimeDefault")
3188        );
3189    }
3190
3191    #[test]
3192    fn hardened_security_context_drops_all_capabilities() {
3193        let ctx = hardened_security_context();
3194        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3195        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3196        assert_eq!(ctx.run_as_non_root, Some(true));
3197        let caps = ctx.capabilities.unwrap();
3198        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3199        assert_eq!(
3200            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3201            Some("RuntimeDefault")
3202        );
3203    }
3204
3205    #[test]
3206    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3207        let ctx = kanidm_job_security_context();
3208        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3209        assert!(ctx.run_as_non_root.is_none());
3210        assert!(ctx.read_only_root_filesystem.is_none());
3211        let caps = ctx.capabilities.unwrap();
3212        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3213        assert_eq!(
3214            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3215            Some("RuntimeDefault")
3216        );
3217    }
3218
3219    #[test]
3220    fn safety_backup_job_name_is_deterministic() {
3221        let source = KanidmRestoreSource {
3222            local: Some(super::KanidmRestoreLocalSource {
3223                file_name: "backup.json".to_string(),
3224            }),
3225            backup_ref: None,
3226        };
3227        let restore = make_restore(source, None);
3228        let name1 = super::safety_backup_job_name(&restore);
3229        let name2 = super::safety_backup_job_name(&restore);
3230        assert_eq!(name1, name2);
3231        assert_eq!(name1, "test-restore-safety-backup");
3232    }
3233
3234    #[test]
3235    fn source_prep_job_name_is_deterministic() {
3236        let source = KanidmRestoreSource {
3237            local: None,
3238            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3239                name: "backup-1".to_string(),
3240            }),
3241        };
3242        let restore = make_restore(source, None);
3243        let name = super::source_prep_job_name(&restore);
3244        assert_eq!(name, "test-restore-source-prep");
3245    }
3246
3247    #[test]
3248    fn break_glass_condition_is_recorded_in_status() {
3249        let mut status = KanidmRestoreStatus::default();
3250        let condition = super::restore_condition(
3251            &status.conditions,
3252            CONDITION_BREAK_GLASS,
3253            CONDITION_TRUE,
3254            "BreakGlassOverride",
3255            "Safety backup skipped",
3256            Some(1),
3257        );
3258        status.conditions.push(condition);
3259        assert!(status.conditions.iter().any(|c| {
3260            c.type_ == CONDITION_BREAK_GLASS
3261                && c.status == CONDITION_TRUE
3262                && c.reason == "BreakGlassOverride"
3263        }));
3264    }
3265
3266    #[test]
3267    fn replica_count_entry_serialization() {
3268        let entry = ReplicaCountEntry {
3269            group: "primary".to_string(),
3270            replicas: 3,
3271        };
3272        let json = serde_json::to_string(&entry).unwrap();
3273        assert!(json.contains("\"group\":\"primary\""));
3274        assert!(json.contains("\"replicas\":3"));
3275    }
3276
3277    #[test]
3278    fn local_source_does_not_require_repository_ref() {
3279        let source = KanidmRestoreSource {
3280            local: Some(super::KanidmRestoreLocalSource {
3281                file_name: "backup.json".to_string(),
3282            }),
3283            backup_ref: None,
3284        };
3285        let safety = SafetyBackupConfig {
3286            repository_ref: None,
3287            skip: false,
3288        };
3289        let restore = make_restore(source, Some(safety));
3290        assert!(validate_safety_backup_config(&restore).is_ok());
3291    }
3292
3293    #[test]
3294    fn skip_without_break_glass_annotations_is_rejected() {
3295        let source = KanidmRestoreSource {
3296            local: Some(super::KanidmRestoreLocalSource {
3297                file_name: "backup.json".to_string(),
3298            }),
3299            backup_ref: None,
3300        };
3301        let safety = SafetyBackupConfig {
3302            repository_ref: None,
3303            skip: true,
3304        };
3305        let restore = make_restore(source, Some(safety));
3306        assert!(validate_safety_backup_config(&restore).is_err());
3307    }
3308
3309    #[test]
3310    fn has_accepted_condition_true() {
3311        let conditions = vec![Condition {
3312            type_: "Ready".to_string(),
3313            status: "True".to_string(),
3314            reason: "Accepted".to_string(),
3315            message: String::new(),
3316            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3317                k8s_openapi::jiff::Timestamp::now(),
3318            ),
3319            observed_generation: None,
3320        }];
3321        assert!(has_accepted_condition(&conditions));
3322    }
3323
3324    #[test]
3325    fn has_accepted_condition_false_when_not_accepted_reason() {
3326        let conditions = vec![Condition {
3327            type_: "Ready".to_string(),
3328            status: "True".to_string(),
3329            reason: "SomeOtherReason".to_string(),
3330            message: String::new(),
3331            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3332                k8s_openapi::jiff::Timestamp::now(),
3333            ),
3334            observed_generation: None,
3335        }];
3336        assert!(!has_accepted_condition(&conditions));
3337    }
3338
3339    #[test]
3340    fn has_accepted_condition_false_when_status_false() {
3341        let conditions = vec![Condition {
3342            type_: "Ready".to_string(),
3343            status: "False".to_string(),
3344            reason: "Accepted".to_string(),
3345            message: String::new(),
3346            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3347                k8s_openapi::jiff::Timestamp::now(),
3348            ),
3349            observed_generation: None,
3350        }];
3351        assert!(!has_accepted_condition(&conditions));
3352    }
3353
3354    #[test]
3355    fn has_accepted_condition_empty() {
3356        assert!(!has_accepted_condition(&[]));
3357    }
3358
3359    #[test]
3360    fn safety_backup_upload_operation_doc_is_valid() {
3361        let source = KanidmRestoreSource {
3362            local: None,
3363            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3364                name: "backup-1".to_string(),
3365            }),
3366        };
3367        let mut restore = make_restore(source, None);
3368        restore.metadata.namespace = Some("test-ns".to_string());
3369        let target = super::super::crd::Kanidm::default();
3370        let doc_str = super::build_safety_upload_operation_doc(
3371            &restore,
3372            &target,
3373            "test-backup-id",
3374            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3375            "my-bucket",
3376            "prod",
3377            "https://s3.example.com",
3378            "us-east-1",
3379            false,
3380            false,
3381            None,
3382            None,
3383        )
3384        .unwrap();
3385        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3386        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3387        assert_eq!(parsed["kind"], "OperationDocument");
3388        assert_eq!(parsed["operation"], "upload");
3389        assert_eq!(parsed["bucket"], "my-bucket");
3390        assert_eq!(parsed["prefix"], "prod");
3391        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3392        assert_eq!(parsed["region"], "us-east-1");
3393        assert_eq!(parsed["backupId"], "test-backup-id");
3394        assert_eq!(parsed["consistency"], "kanidm-offline");
3395        assert_eq!(parsed["reason"], "restore-safety");
3396        assert_eq!(
3397            parsed["payloadPath"],
3398            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3399        );
3400        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3401    }
3402
3403    #[test]
3404    fn download_operation_doc_uses_real_repository_data() {
3405        let source = KanidmRestoreSource {
3406            local: None,
3407            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3408                name: "backup-1".to_string(),
3409            }),
3410        };
3411        let restore = make_restore(source, None);
3412        let target = super::super::crd::Kanidm::default();
3413        let doc_str = super::build_download_operation_doc(
3414            &restore,
3415            &target,
3416            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3417            "019c7c76-f423-7a12-8f41-2bea7588a303",
3418            "real-bucket",
3419            "real-prefix",
3420            "https://real-endpoint.com",
3421            "eu-west-1",
3422            true,
3423            false,
3424            None,
3425            None,
3426        );
3427        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3428        assert_eq!(parsed["bucket"], "real-bucket");
3429        assert_eq!(parsed["prefix"], "real-prefix");
3430        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3431        assert_eq!(parsed["region"], "eu-west-1");
3432        assert_eq!(parsed["forcePathStyle"], true);
3433        assert_eq!(
3434            parsed["manifestKey"],
3435            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3436        );
3437        assert_eq!(
3438            parsed["expectedBackupId"],
3439            "019c7c76-f423-7a12-8f41-2bea7588a303"
3440        );
3441    }
3442
3443    #[test]
3444    fn safety_backup_ref_is_set_after_verification() {
3445        let status = KanidmRestoreStatus {
3446            phase: KanidmRestorePhase::SafetyBackup,
3447            safety_backup_job_name: Some("test-safety-backup".to_string()),
3448            safety_backup_ref: Some("safety-test-restore".to_string()),
3449            safety_backup_expected_backup_id: Some(
3450                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3451            ),
3452            safety_backup_manifest_key: Some(
3453                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3454            ),
3455            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3456            ..Default::default()
3457        };
3458        assert!(status.safety_backup_ref.is_some());
3459        assert!(
3460            status
3461                .safety_backup_ref
3462                .as_ref()
3463                .unwrap()
3464                .starts_with("safety-")
3465        );
3466        assert!(status.safety_backup_expected_backup_id.is_some());
3467        assert!(status.safety_backup_manifest_key.is_some());
3468        assert!(status.safety_backup_payload_sha256.is_some());
3469    }
3470
3471    #[test]
3472    fn database_mutation_not_started_without_safety_backup_ref() {
3473        let status = KanidmRestoreStatus {
3474            phase: KanidmRestorePhase::PreparingSource,
3475            safety_backup_ref: None,
3476            database_mutation_started: false,
3477            ..Default::default()
3478        };
3479        assert!(!status.database_mutation_started);
3480        assert!(status.safety_backup_ref.is_none());
3481    }
3482
3483    #[test]
3484    fn pre_boundary_failure_does_not_set_mutation_started() {
3485        let status = KanidmRestoreStatus {
3486            phase: KanidmRestorePhase::Failed,
3487            message: Some(
3488                "safety backup job failed; target restored to original state".to_string(),
3489            ),
3490            database_mutation_started: false,
3491            ..Default::default()
3492        };
3493        assert!(!status.database_mutation_started);
3494    }
3495
3496    #[test]
3497    fn shared_volume_constants_are_distinct() {
3498        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3499        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3500    }
3501
3502    #[test]
3503    fn validate_backup_compatibility_matching_versions_passes() {
3504        let source = KanidmRestoreSource {
3505            local: None,
3506            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3507                name: "backup-1".to_string(),
3508            }),
3509        };
3510        let restore = make_restore(source, None);
3511        let target = kanidm_with_version("1.10.4");
3512        let backup = kaniop_backup_core::crd::KanidmBackup {
3513            metadata: kube::api::ObjectMeta::default(),
3514            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3515                backup_id: "id".to_string(),
3516                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3517                    name: "k".to_string(),
3518                    uid: "uid".to_string(),
3519                },
3520                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3521                    name: "repo".to_string(),
3522                },
3523                manifest_key: "key".to_string(),
3524            },
3525            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3526                kanidm_version: Some("1.10.4".to_string()),
3527                ..Default::default()
3528            }),
3529        };
3530        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3531    }
3532
3533    #[test]
3534    fn validate_backup_compatibility_version_mismatch_fails() {
3535        let source = KanidmRestoreSource {
3536            local: None,
3537            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3538                name: "backup-1".to_string(),
3539            }),
3540        };
3541        let restore = make_restore(source, None);
3542        let target = kanidm_with_version("1.11.0");
3543        let backup = kaniop_backup_core::crd::KanidmBackup {
3544            metadata: kube::api::ObjectMeta::default(),
3545            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3546                backup_id: "id".to_string(),
3547                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3548                    name: "k".to_string(),
3549                    uid: "uid".to_string(),
3550                },
3551                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3552                    name: "repo".to_string(),
3553                },
3554                manifest_key: "key".to_string(),
3555            },
3556            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3557                kanidm_version: Some("1.10.4".to_string()),
3558                ..Default::default()
3559            }),
3560        };
3561        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3562    }
3563
3564    #[test]
3565    fn validate_backup_compatibility_no_backup_version_passes() {
3566        let source = KanidmRestoreSource {
3567            local: None,
3568            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3569                name: "backup-1".to_string(),
3570            }),
3571        };
3572        let restore = make_restore(source, None);
3573        let target = super::super::crd::Kanidm::default();
3574        let backup = kaniop_backup_core::crd::KanidmBackup {
3575            metadata: kube::api::ObjectMeta::default(),
3576            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3577                backup_id: "id".to_string(),
3578                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3579                    name: "k".to_string(),
3580                    uid: "uid".to_string(),
3581                },
3582                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3583                    name: "repo".to_string(),
3584                },
3585                manifest_key: "key".to_string(),
3586            },
3587            status: None,
3588        };
3589        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3590    }
3591
3592    #[test]
3593    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3594        let source = KanidmRestoreSource {
3595            local: None,
3596            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3597                name: "backup-1".to_string(),
3598            }),
3599        };
3600        let mut restore = make_restore(source, None);
3601        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3602        let target = super::super::crd::Kanidm::default();
3603        let backup = kaniop_backup_core::crd::KanidmBackup {
3604            metadata: kube::api::ObjectMeta::default(),
3605            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3606                backup_id: "id".to_string(),
3607                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3608                    name: "k".to_string(),
3609                    uid: "uid".to_string(),
3610                },
3611                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3612                    name: "repo".to_string(),
3613                },
3614                manifest_key: "key".to_string(),
3615            },
3616            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3617                image_digest: Some("sha256:originaldigest".to_string()),
3618                ..Default::default()
3619            }),
3620        };
3621        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3622    }
3623
3624    #[test]
3625    fn validate_backup_compatibility_image_digest_match_passes() {
3626        let source = KanidmRestoreSource {
3627            local: None,
3628            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3629                name: "backup-1".to_string(),
3630            }),
3631        };
3632        let mut restore = make_restore(source, None);
3633        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3634        let target = super::super::crd::Kanidm::default();
3635        let backup = kaniop_backup_core::crd::KanidmBackup {
3636            metadata: kube::api::ObjectMeta::default(),
3637            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3638                backup_id: "id".to_string(),
3639                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3640                    name: "k".to_string(),
3641                    uid: "uid".to_string(),
3642                },
3643                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3644                    name: "repo".to_string(),
3645                },
3646                manifest_key: "key".to_string(),
3647            },
3648            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3649                image_digest: Some("sha256:abc123".to_string()),
3650                ..Default::default()
3651            }),
3652        };
3653        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3654    }
3655
3656    #[test]
3657    fn safety_backup_id_is_deterministic() {
3658        let source = KanidmRestoreSource {
3659            local: None,
3660            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3661                name: "backup-1".to_string(),
3662            }),
3663        };
3664        let restore = make_restore(source, None);
3665        let id1 = super::compute_safety_backup_id(&restore);
3666        let id2 = super::compute_safety_backup_id(&restore);
3667        assert_eq!(id1, id2);
3668        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3669    }
3670
3671    #[test]
3672    fn safety_backup_id_differs_for_different_restores() {
3673        let source1 = KanidmRestoreSource {
3674            local: None,
3675            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3676                name: "backup-1".to_string(),
3677            }),
3678        };
3679        let restore1 = make_restore(source1, None);
3680        let mut restore2 = make_restore(
3681            KanidmRestoreSource {
3682                local: None,
3683                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3684                    name: "backup-2".to_string(),
3685                }),
3686            },
3687            None,
3688        );
3689        restore2.metadata.name = Some("different-restore".to_string());
3690        let id1 = super::compute_safety_backup_id(&restore1);
3691        let id2 = super::compute_safety_backup_id(&restore2);
3692        assert_ne!(id1, id2);
3693    }
3694
3695    #[test]
3696    fn safety_manifest_key_is_deterministic() {
3697        let source = KanidmRestoreSource {
3698            local: None,
3699            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3700                name: "backup-1".to_string(),
3701            }),
3702        };
3703        let restore = make_restore(source, None);
3704        let key1 = super::compute_safety_manifest_key(&restore);
3705        let key2 = super::compute_safety_manifest_key(&restore);
3706        assert_eq!(key1, key2);
3707        assert!(key1.contains("manifest.json"));
3708        assert!(key1.starts_with("v1/tenants/"));
3709    }
3710
3711    #[test]
3712    fn safety_backup_id_survives_restart() {
3713        let source = KanidmRestoreSource {
3714            local: None,
3715            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3716                name: "backup-1".to_string(),
3717            }),
3718        };
3719        let mut restore = make_restore(source, None);
3720        let original_id = super::compute_safety_backup_id(&restore);
3721        restore.status = Some(KanidmRestoreStatus {
3722            phase: KanidmRestorePhase::SafetyBackup,
3723            safety_backup_expected_backup_id: Some(original_id.clone()),
3724            ..Default::default()
3725        });
3726        let recomputed_id = super::compute_safety_backup_id(&restore);
3727        assert_eq!(original_id, recomputed_id);
3728    }
3729
3730    #[test]
3731    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3732        let source = KanidmRestoreSource {
3733            local: None,
3734            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3735                name: "backup-1".to_string(),
3736            }),
3737        };
3738        let restore = make_restore(source, None);
3739        assert!(super::requires_safety_backup(&restore));
3740        let status_without_ref = KanidmRestoreStatus {
3741            phase: KanidmRestorePhase::RestoringPrimary,
3742            safety_backup_ref: None,
3743            database_mutation_started: false,
3744            ..Default::default()
3745        };
3746        assert!(status_without_ref.safety_backup_ref.is_none());
3747        assert!(!status_without_ref.database_mutation_started);
3748    }
3749
3750    #[test]
3751    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3752        let status_with_ref = KanidmRestoreStatus {
3753            phase: KanidmRestorePhase::RestoringPrimary,
3754            safety_backup_ref: Some("safety-test-restore".to_string()),
3755            safety_backup_manifest_key: Some(
3756                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3757            ),
3758            safety_backup_payload_sha256: Some("abc123".to_string()),
3759            database_mutation_started: false,
3760            ..Default::default()
3761        };
3762        assert!(status_with_ref.safety_backup_ref.is_some());
3763        assert!(!status_with_ref.database_mutation_started);
3764    }
3765
3766    #[test]
3767    fn result_document_verification_rejects_missing_backup_id() {
3768        let result_json = r#"{
3769            "apiVersion": "backup.kaniop.rs/v1alpha1",
3770            "kind": "ResultDocument",
3771            "operation": "upload",
3772            "success": true,
3773            "exitCode": "success",
3774            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3775            "payloadSha256": "abc123"
3776        }"#;
3777        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3778        assert!(result.backup_id.is_none());
3779    }
3780
3781    #[test]
3782    fn result_document_verification_rejects_missing_manifest_key() {
3783        let result_json = r#"{
3784            "apiVersion": "backup.kaniop.rs/v1alpha1",
3785            "kind": "ResultDocument",
3786            "operation": "upload",
3787            "success": true,
3788            "exitCode": "success",
3789            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3790            "payloadSha256": "abc123"
3791        }"#;
3792        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3793        assert!(result.manifest_key.is_none());
3794    }
3795
3796    #[test]
3797    fn result_document_verification_rejects_missing_payload_sha256() {
3798        let result_json = r#"{
3799            "apiVersion": "backup.kaniop.rs/v1alpha1",
3800            "kind": "ResultDocument",
3801            "operation": "upload",
3802            "success": true,
3803            "exitCode": "success",
3804            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3805            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3806        }"#;
3807        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3808        assert!(result.payload_sha256.is_none());
3809    }
3810
3811    #[test]
3812    fn result_document_verification_rejects_failure_exit_code() {
3813        let result_json = r#"{
3814            "apiVersion": "backup.kaniop.rs/v1alpha1",
3815            "kind": "ResultDocument",
3816            "operation": "upload",
3817            "success": false,
3818            "exitCode": "integrity",
3819            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3820            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3821            "payloadSha256": "abc123"
3822        }"#;
3823        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3824        assert!(!result.success);
3825        assert_eq!(
3826            result.exit_code,
3827            kaniop_backup_core::result::ExitCode::Integrity
3828        );
3829    }
3830
3831    #[test]
3832    fn result_document_verification_rejects_wrong_operation() {
3833        let result_json = r#"{
3834            "apiVersion": "backup.kaniop.rs/v1alpha1",
3835            "kind": "ResultDocument",
3836            "operation": "download",
3837            "success": true,
3838            "exitCode": "success",
3839            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3840            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3841            "payloadSha256": "abc123"
3842        }"#;
3843        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3844        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3845    }
3846
3847    #[test]
3848    fn result_document_verification_accepts_valid_document() {
3849        let result_json = r#"{
3850            "apiVersion": "backup.kaniop.rs/v1alpha1",
3851            "kind": "ResultDocument",
3852            "operation": "upload",
3853            "success": true,
3854            "exitCode": "success",
3855            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3856            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3857            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3858        }"#;
3859        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3860        assert!(result.success);
3861        assert_eq!(
3862            result.exit_code,
3863            kaniop_backup_core::result::ExitCode::Success
3864        );
3865        assert_eq!(result.operation, "upload");
3866        assert!(result.backup_id.is_some());
3867        assert!(result.manifest_key.is_some());
3868        assert!(result.payload_sha256.is_some());
3869    }
3870
3871    #[test]
3872    fn termination_message_path_constant_is_valid() {
3873        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3874        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3875    }
3876
3877    #[test]
3878    fn status_tracks_safety_backup_evidence() {
3879        let status = KanidmRestoreStatus {
3880            phase: KanidmRestorePhase::PreparingSource,
3881            safety_backup_ref: Some("safety-test-restore".to_string()),
3882            safety_backup_expected_backup_id: Some(
3883                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3884            ),
3885            safety_backup_manifest_key: Some(
3886                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3887            ),
3888            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3889            ..Default::default()
3890        };
3891        assert!(status.safety_backup_ref.is_some());
3892        assert!(status.safety_backup_expected_backup_id.is_some());
3893        assert!(status.safety_backup_manifest_key.is_some());
3894        assert!(status.safety_backup_payload_sha256.is_some());
3895    }
3896
3897    #[test]
3898    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3899        let invalid_json = "not valid json";
3900        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3901        assert!(result.is_err());
3902    }
3903
3904    #[test]
3905    fn fail_closed_semantics_preserved_on_oversized_result() {
3906        let oversized_json = format!(
3907            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3908            kaniop_backup_core::result::RESULT_DOC_VERSION,
3909            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3910        );
3911        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3912        assert!(result.is_err());
3913    }
3914
3915    #[test]
3916    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3917        let source = KanidmRestoreSource {
3918            local: None,
3919            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3920                name: "backup-1".to_string(),
3921            }),
3922        };
3923        let mut restore = make_restore(source, None);
3924        restore.metadata.namespace = Some("test-ns".to_string());
3925        let target = super::super::crd::Kanidm::default();
3926        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3927        let doc_str = super::build_safety_upload_operation_doc(
3928            &restore,
3929            &target,
3930            "test-backup-id",
3931            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3932            "my-bucket",
3933            "prod",
3934            "https://s3.example.com",
3935            "us-east-1",
3936            false,
3937            false,
3938            Some(&ca_path),
3939            None,
3940        )
3941        .unwrap();
3942        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3943        assert_eq!(parsed["caBundlePath"], ca_path);
3944    }
3945
3946    #[test]
3947    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3948        let source = KanidmRestoreSource {
3949            local: None,
3950            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3951                name: "backup-1".to_string(),
3952            }),
3953        };
3954        let mut restore = make_restore(source, None);
3955        restore.metadata.namespace = Some("test-ns".to_string());
3956        let target = super::super::crd::Kanidm::default();
3957        let doc_str = super::build_safety_upload_operation_doc(
3958            &restore,
3959            &target,
3960            "test-backup-id",
3961            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3962            "my-bucket",
3963            "prod",
3964            "https://s3.example.com",
3965            "us-east-1",
3966            false,
3967            false,
3968            None,
3969            None,
3970        )
3971        .unwrap();
3972        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3973        assert!(parsed["caBundlePath"].is_null());
3974    }
3975
3976    #[test]
3977    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3978        let source = KanidmRestoreSource {
3979            local: None,
3980            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3981                name: "backup-1".to_string(),
3982            }),
3983        };
3984        let restore = make_restore(source, None);
3985        let target = super::super::crd::Kanidm::default();
3986        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3987        let doc_str = super::build_download_operation_doc(
3988            &restore,
3989            &target,
3990            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3991            "019c7c76-f423-7a12-8f41-2bea7588a303",
3992            "real-bucket",
3993            "real-prefix",
3994            "https://real-endpoint.com",
3995            "eu-west-1",
3996            true,
3997            false,
3998            Some(&ca_path),
3999            None,
4000        );
4001        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4002        assert_eq!(parsed["caBundlePath"], ca_path);
4003    }
4004
4005    #[test]
4006    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4007        let source = KanidmRestoreSource {
4008            local: None,
4009            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4010                name: "backup-1".to_string(),
4011            }),
4012        };
4013        let restore = make_restore(source, None);
4014        let target = super::super::crd::Kanidm::default();
4015        let doc_str = super::build_download_operation_doc(
4016            &restore,
4017            &target,
4018            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4019            "019c7c76-f423-7a12-8f41-2bea7588a303",
4020            "real-bucket",
4021            "real-prefix",
4022            "https://real-endpoint.com",
4023            "eu-west-1",
4024            true,
4025            false,
4026            None,
4027            None,
4028        );
4029        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4030        assert!(parsed["caBundlePath"].is_null());
4031    }
4032
4033    #[test]
4034    fn source_prep_result_operation_is_download() {
4035        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4036        assert_ne!(
4037            super::SOURCE_PREP_RESULT_OPERATION,
4038            super::SAFETY_BACKUP_RESULT_OPERATION
4039        );
4040    }
4041
4042    #[test]
4043    fn download_result_document_verification_accepts_valid_document() {
4044        let result_json = r#"{
4045            "apiVersion": "backup.kaniop.rs/v1alpha1",
4046            "kind": "ResultDocument",
4047            "operation": "download",
4048            "success": true,
4049            "exitCode": "success",
4050            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4051            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4052            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4053        }"#;
4054        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4055        assert!(result.success);
4056        assert_eq!(
4057            result.exit_code,
4058            kaniop_backup_core::result::ExitCode::Success
4059        );
4060        assert_eq!(result.operation, "download");
4061        assert_eq!(
4062            result.backup_id.as_deref(),
4063            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4064        );
4065        assert!(result.manifest_key.is_some());
4066        assert!(result.payload_sha256.is_some());
4067    }
4068
4069    #[test]
4070    fn download_result_document_rejects_wrong_operation() {
4071        let result_json = r#"{
4072            "apiVersion": "backup.kaniop.rs/v1alpha1",
4073            "kind": "ResultDocument",
4074            "operation": "upload",
4075            "success": true,
4076            "exitCode": "success",
4077            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4078            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4079            "payloadSha256": "abc123"
4080        }"#;
4081        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4082        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4083    }
4084
4085    #[test]
4086    fn download_result_document_rejects_failure_success_field() {
4087        let result_json = r#"{
4088            "apiVersion": "backup.kaniop.rs/v1alpha1",
4089            "kind": "ResultDocument",
4090            "operation": "download",
4091            "success": false,
4092            "exitCode": "retryable",
4093            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4094            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4095            "payloadSha256": "abc123"
4096        }"#;
4097        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4098        assert!(!result.success);
4099        assert_ne!(
4100            result.exit_code,
4101            kaniop_backup_core::result::ExitCode::Success
4102        );
4103    }
4104
4105    #[test]
4106    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4107        let status = KanidmRestoreStatus {
4108            phase: KanidmRestorePhase::PreparingSource,
4109            database_mutation_started: false,
4110            ..Default::default()
4111        };
4112        assert!(!status.database_mutation_started);
4113        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4114    }
4115
4116    #[test]
4117    fn backup_job_volume_size_default_is_10gi() {
4118        let qty = crate::controller::backup_job_volume_size();
4119        assert_eq!(qty, Quantity("10Gi".to_string()));
4120    }
4121
4122    #[test]
4123    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4124        use kaniop_backup_core::auth::build_encryption_env_vars;
4125        use kaniop_backup_core::crd::SecretRef;
4126
4127        let key_ref = SecretRef {
4128            name: "kek-secret".to_string(),
4129        };
4130        let env_vars = build_encryption_env_vars(Some(&key_ref));
4131        assert_eq!(env_vars.len(), 1);
4132        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4133        let secret_ref = env_vars[0]
4134            .value_from
4135            .as_ref()
4136            .unwrap()
4137            .secret_key_ref
4138            .as_ref()
4139            .unwrap();
4140        assert_eq!(secret_ref.name, "kek-secret");
4141        assert_eq!(secret_ref.key, "encryption-key");
4142    }
4143
4144    #[test]
4145    fn encryption_env_vars_absent_when_no_key_ref() {
4146        use kaniop_backup_core::auth::build_encryption_env_vars;
4147
4148        let env_vars = build_encryption_env_vars(None);
4149        assert!(
4150            env_vars.is_empty(),
4151            "encryption env vars must be empty when key_ref is None"
4152        );
4153    }
4154
4155    #[test]
4156    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4157        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4158
4159        let source = KanidmRestoreSource {
4160            local: None,
4161            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4162                name: "backup-1".to_string(),
4163            }),
4164        };
4165        let mut restore = make_restore(source, None);
4166        restore.metadata.namespace = Some("test-ns".to_string());
4167        let target = super::super::crd::Kanidm::default();
4168        let encryption = RepositoryEncryption {
4169            mode: EncryptionMode::ClientSide,
4170            key_id: None,
4171            key_ref: Some(SecretRef {
4172                name: "kek-secret".to_string(),
4173            }),
4174        };
4175        let doc_str = super::build_safety_upload_operation_doc(
4176            &restore,
4177            &target,
4178            "test-backup-id",
4179            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4180            "my-bucket",
4181            "prod",
4182            "https://s3.example.com",
4183            "us-east-1",
4184            false,
4185            false,
4186            None,
4187            Some(&encryption),
4188        )
4189        .unwrap();
4190        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4191        assert_eq!(parsed["encryptionMode"], "clientSide");
4192        assert!(parsed["encryptionKeyId"].is_null());
4193    }
4194
4195    #[test]
4196    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4197        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4198
4199        let source = KanidmRestoreSource {
4200            local: None,
4201            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4202                name: "backup-1".to_string(),
4203            }),
4204        };
4205        let restore = make_restore(source, None);
4206        let target = super::super::crd::Kanidm::default();
4207        let encryption = RepositoryEncryption {
4208            mode: EncryptionMode::ProviderKms,
4209            key_id: Some("alias/kaniop-backups".to_string()),
4210            key_ref: None,
4211        };
4212        let doc_str = super::build_download_operation_doc(
4213            &restore,
4214            &target,
4215            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4216            "019c7c76-f423-7a12-8f41-2bea7588a303",
4217            "real-bucket",
4218            "real-prefix",
4219            "https://real-endpoint.com",
4220            "eu-west-1",
4221            true,
4222            false,
4223            None,
4224            Some(&encryption),
4225        );
4226        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4227        assert_eq!(parsed["encryptionMode"], "providerKms");
4228        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4229    }
4230}