Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, ca_bundle_env_var, ca_bundle_path,
9};
10use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
11use kaniop_backup_core::operation::{
12    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
13};
14use kaniop_backup_core::result::{ExitCode, parse_result_document};
15use kaniop_k8s_util::error::{Error, Result};
16
17use std::collections::BTreeSet;
18use std::sync::Arc;
19use std::time::Duration;
20
21use futures::StreamExt;
22use k8s_openapi::api::apps::v1::StatefulSet;
23use k8s_openapi::api::batch::v1::{Job, JobSpec};
24use k8s_openapi::api::core::v1::{
25    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
26    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
27    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
28};
29use k8s_openapi::api::storage::v1::VolumeAttachment;
30use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
31use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
32use k8s_openapi::jiff::Timestamp;
33use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
34use kube::runtime::controller::{Action, Controller};
35use kube::runtime::events::{Event, EventType, Recorder, Reporter};
36use kube::runtime::finalizer::{Event as Finalizer, finalizer};
37use kube::runtime::watcher;
38use kube::{Api, Client, CustomResource, Resource, ResourceExt};
39use opentelemetry::metrics::{Counter, Histogram};
40use opentelemetry::{KeyValue, global};
41#[cfg(feature = "schemars")]
42use schemars::JsonSchema;
43use serde::{Deserialize, Serialize};
44use serde_json::json;
45use tracing::{debug, error, info, warn};
46
47pub const CONTROLLER_ID: &str = "kanidm-restore";
48pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
49const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
50const DATA_VOLUME: &str = "kanidm-data";
51const STAGING_VOLUME: &str = "restore-staging";
52const SHARED_VOLUME: &str = "safety-backup-shared";
53const DATA_PATH: &str = "/data";
54const TLS_VOLUME: &str = "kanidm-certs";
55const TLS_PATH: &str = "/etc/kanidm/tls";
56const BACKUP_PATH: &str = "/data";
57const STAGING_PATH: &str = "/staging";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76    feature = "schemars",
77    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80    category = "kaniop",
81    group = "kaniop.rs",
82    version = "v1beta1",
83    kind = "KanidmRestore",
84    plural = "kanidmrestores",
85    singular = "kanidmrestore",
86    shortname = "idmrestore",
87    namespaced,
88    status = "KanidmRestoreStatus",
89    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95    pub target_ref: KanidmRestoreTargetRef,
96    pub source: KanidmRestoreSource,
97    pub restore_image: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106    pub name: String,
107    pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113    feature = "schemars",
114    schemars(extend("x-kubernetes-validations" = [
115        {
116            "message": "exactly one of local or backupRef must be set",
117            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118        }
119    ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123    /// Local file source. Mutually exclusive with `backupRef`.
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub local: Option<KanidmRestoreLocalSource>,
126    /// Remote cataloged backup reference. Mutually exclusive with `local`.
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
136    pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
144    pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub repository_ref: Option<SafetyBackupRepositoryRef>,
154    /// Skip the mandatory safety backup. Requires break-glass annotations.
155    #[serde(default)]
156    pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163    pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169    #[default]
170    Pending,
171    Validating,
172    Quiescing,
173    SafetyBackup,
174    PreparingSource,
175    RestoringPrimary,
176    Verifying,
177    RebuildingReplicas,
178    Resuming,
179    Completed,
180    Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub observed_generation: Option<i64>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_target_uid: Option<String>,
191    #[serde(default)]
192    pub phase: KanidmRestorePhase,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub restore_job_name: Option<String>,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub verify_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub safety_backup_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub source_prep_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub safety_backup_ref: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_expected_backup_id: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_manifest_key: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_payload_sha256: Option<String>,
209    #[serde(default)]
210    pub replicas_cleared: bool,
211    #[serde(default)]
212    pub certificates_cleared: bool,
213    #[serde(default)]
214    pub database_mutation_started: bool,
215    #[serde(default, skip_serializing_if = "Vec::is_empty")]
216    pub original_replicas: Vec<ReplicaCountEntry>,
217    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218    pub phase_timestamps: std::collections::BTreeMap<String, String>,
219    #[serde(default, skip_serializing_if = "Vec::is_empty")]
220    #[cfg_attr(
221        feature = "schemars",
222        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223    )]
224    pub conditions: Vec<Condition>,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233    pub group: String,
234    pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239    attempts: Counter<u64>,
240    outcomes: Counter<u64>,
241    duration_seconds: Histogram<f64>,
242    break_glass_total: Counter<u64>,
243    safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247    fn new() -> Self {
248        let meter = global::meter("kaniop");
249        Self {
250            attempts: meter
251                .u64_counter("kaniop_restore_attempts_total")
252                .with_description("Number of Kanidm restore attempts started")
253                .build(),
254            outcomes: meter
255                .u64_counter("kaniop_restore_outcomes_total")
256                .with_description("Number of terminal Kanidm restore outcomes")
257                .build(),
258            duration_seconds: meter
259                .f64_histogram("kaniop_restore_duration_seconds")
260                .with_description("Kanidm restore duration from object creation to terminal phase")
261                .with_unit("s")
262                .build(),
263            break_glass_total: meter
264                .u64_counter("kaniop_restore_break_glass_total")
265                .with_description("Number of break-glass safety backup overrides")
266                .build(),
267            safety_backup_duration_seconds: meter
268                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269                .with_description("Duration of pre-restore safety backup creation")
270                .with_unit("s")
271                .build(),
272        }
273    }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278    client: Client,
279    recorder: Recorder,
280    metrics: RestoreMetrics,
281}
282
283pub async fn run(client: Client) {
284    let api = Api::<KanidmRestore>::all(client.clone());
285    let recorder = Recorder::new(
286        client.clone(),
287        Reporter {
288            controller: CONTROLLER_ID.into(),
289            instance: None,
290        },
291    );
292    let ctx = Arc::new(RestoreContext {
293        client,
294        recorder,
295        metrics: RestoreMetrics::new(),
296    });
297    info!("starting {CONTROLLER_ID} controller");
298    Controller::new(api, watcher::Config::default().any_semantic())
299        .shutdown_on_signal()
300        .run(reconcile_restore, error_policy, ctx)
301        .for_each(|result| async move {
302            if let Err(error) = result {
303                error!(%error, "KanidmRestore reconciliation failed");
304            }
305        })
306        .await;
307}
308
309fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
310    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
311    Action::requeue(Duration::from_secs(5))
312}
313
314async fn reconcile_restore(
315    restore: Arc<KanidmRestore>,
316    ctx: Arc<RestoreContext>,
317) -> Result<Action> {
318    let namespace = restore
319        .namespace()
320        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
321    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
322    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
323        let ctx = ctx.clone();
324        async move {
325            match event {
326                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
327                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
328            }
329        }
330    })
331    .await
332    .map_err(|error| {
333        Error::FinalizerError(
334            "failed on KanidmRestore finalizer".to_string(),
335            Box::new(error),
336        )
337    })
338}
339
340async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
341    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
342    match phase {
343        KanidmRestorePhase::Pending => {
344            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
345            Ok(Action::requeue(REQUEUE))
346        }
347        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
348            Ok(target) => {
349                mark_restoring(&restore, &target, &ctx).await?;
350                let mut status = restore.status.clone().unwrap_or_default();
351                status.observed_target_uid = target.uid();
352                status.original_replicas = capture_original_replicas(&target);
353                status.phase = KanidmRestorePhase::Quiescing;
354                status.message = None;
355                patch_status(&restore, &ctx, status).await?;
356                Ok(Action::requeue(REQUEUE))
357            }
358            Err(error) => {
359                set_phase(
360                    &restore,
361                    &ctx,
362                    KanidmRestorePhase::Failed,
363                    Some(error.to_string()),
364                )
365                .await?;
366                Ok(Action::requeue(Duration::from_secs(300)))
367            }
368        },
369        KanidmRestorePhase::Quiescing => {
370            let target = get_target(&restore, &ctx).await?;
371            scale_all(&target, &ctx, 0).await?;
372            if target_pods_stopped(&target, &ctx).await?
373                && target_volumes_detached(&target, &ctx).await?
374            {
375                let next_phase = if requires_safety_backup(&restore) {
376                    KanidmRestorePhase::SafetyBackup
377                } else {
378                    record_break_glass(&restore, &ctx).await?;
379                    KanidmRestorePhase::PreparingSource
380                };
381                if next_phase == KanidmRestorePhase::PreparingSource {
382                    let refreshed = Api::<KanidmRestore>::namespaced(
383                        ctx.client.clone(),
384                        restore.namespace().as_deref().unwrap_or_default(),
385                    )
386                    .get(&restore.name_any())
387                    .await
388                    .map_err(|error| {
389                        Error::kube_error(
390                            "get",
391                            "KanidmRestore",
392                            restore.namespace().as_deref().unwrap_or_default(),
393                            restore.name_any(),
394                            error,
395                        )
396                    })?;
397                    set_phase(&refreshed, &ctx, next_phase, None).await?;
398                } else {
399                    set_phase(&restore, &ctx, next_phase, None).await?;
400                }
401            }
402            Ok(Action::requeue(REQUEUE))
403        }
404        KanidmRestorePhase::SafetyBackup => {
405            let target = get_target(&restore, &ctx).await?;
406            let name = safety_backup_job_name(&restore);
407            let expected_backup_id = compute_safety_backup_id(&restore);
408            let expected_manifest_key = compute_safety_manifest_key(&restore);
409            ensure_safety_backup_job(
410                &restore,
411                &target,
412                &ctx,
413                &name,
414                &expected_backup_id,
415                &expected_manifest_key,
416            )
417            .await?;
418            let mut status = restore.status.clone().unwrap_or_default();
419            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
420            match job_state(&restore, &ctx, &name).await? {
421                JobState::Complete => {
422                    let result =
423                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
424                    match result {
425                        Ok(verified) => {
426                            status.safety_backup_job_name = Some(name);
427                            status.safety_backup_ref =
428                                Some(format!("safety-{}", restore.name_any()));
429                            status.safety_backup_manifest_key = Some(verified.manifest_key);
430                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
431                            record_safety_backup_duration(&restore, &ctx).await;
432                            status.phase = KanidmRestorePhase::PreparingSource;
433                            status.message = None;
434                            patch_status(&restore, &ctx, status).await?;
435                        }
436                        Err(error) => {
437                            resume_before_mutation(&restore, &ctx).await?;
438                            set_phase(
439                                &restore,
440                                &ctx,
441                                KanidmRestorePhase::Failed,
442                                Some(format!("safety backup result verification failed: {error}")),
443                            )
444                            .await?;
445                        }
446                    }
447                }
448                JobState::Failed => {
449                    resume_before_mutation(&restore, &ctx).await?;
450                    set_phase(
451                        &restore,
452                        &ctx,
453                        KanidmRestorePhase::Failed,
454                        Some(
455                            "safety backup job failed; target restored to original state"
456                                .to_string(),
457                        ),
458                    )
459                    .await?;
460                }
461                JobState::Running => {
462                    patch_status(&restore, &ctx, status).await?;
463                }
464            }
465            Ok(Action::requeue(REQUEUE))
466        }
467        KanidmRestorePhase::PreparingSource => {
468            if is_remote_source(&restore) {
469                let target = get_target(&restore, &ctx).await?;
470                let name = source_prep_job_name(&restore);
471                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
472                match job_state(&restore, &ctx, &name).await? {
473                    JobState::Complete => {
474                        let backup_id = restore
475                            .spec
476                            .source
477                            .backup_ref
478                            .as_ref()
479                            .map(|b| b.name.clone())
480                            .unwrap_or_default();
481                        let backup = {
482                            let ns = restore.namespace().unwrap();
483                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
484                                .get(&backup_id)
485                                .await
486                                .map_err(|e| {
487                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
488                                })?
489                        };
490                        let expected_backup_id = &backup.spec.backup_id;
491                        let result =
492                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
493                                .await;
494                        match result {
495                            Ok(_verified) => {
496                                let mut status = restore.status.clone().unwrap_or_default();
497                                status.source_prep_job_name = Some(name);
498                                status.phase = KanidmRestorePhase::RestoringPrimary;
499                                status.message = None;
500                                patch_status(&restore, &ctx, status).await?;
501                            }
502                            Err(error) => {
503                                resume_before_mutation(&restore, &ctx).await?;
504                                set_phase(
505                                    &restore,
506                                    &ctx,
507                                    KanidmRestorePhase::Failed,
508                                    Some(format!(
509                                        "source preparation result verification failed: {error}"
510                                    )),
511                                )
512                                .await?;
513                            }
514                        }
515                    }
516                    JobState::Failed => {
517                        resume_before_mutation(&restore, &ctx).await?;
518                        set_phase(
519                            &restore,
520                            &ctx,
521                            KanidmRestorePhase::Failed,
522                            Some(
523                                "source preparation job failed; target restored to original state"
524                                    .to_string(),
525                            ),
526                        )
527                        .await?;
528                    }
529                    JobState::Running => {}
530                }
531            } else {
532                let target = get_target(&restore, &ctx).await?;
533                let name = source_check_job_name(&restore);
534                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
535                match job_state(&restore, &ctx, &name).await? {
536                    JobState::Complete => {
537                        let mut status = restore.status.clone().unwrap_or_default();
538                        status.source_prep_job_name = Some(name);
539                        status.phase = KanidmRestorePhase::RestoringPrimary;
540                        status.message = None;
541                        patch_status(&restore, &ctx, status).await?;
542                    }
543                    JobState::Failed => {
544                        resume_before_mutation(&restore, &ctx).await?;
545                        set_phase(
546                            &restore,
547                            &ctx,
548                            KanidmRestorePhase::Failed,
549                            Some(
550                                "local backup file check failed; target restored to original state"
551                                    .to_string(),
552                            ),
553                        )
554                        .await?;
555                    }
556                    JobState::Running => {}
557                }
558            }
559            Ok(Action::requeue(REQUEUE))
560        }
561        KanidmRestorePhase::RestoringPrimary => {
562            let target = get_target(&restore, &ctx).await?;
563            let status = restore.status.as_ref().cloned().unwrap_or_default();
564            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
565                return Err(Error::MissingData(
566                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
567                ));
568            }
569            if !status.database_mutation_started {
570                let mut status = status;
571                status.database_mutation_started = true;
572                status.message = Some(
573                    "database mutation boundary persisted; starting primary restore".to_string(),
574                );
575                patch_status(&restore, &ctx, status).await?;
576                return Ok(Action::requeue(REQUEUE));
577            }
578            let name = restore_job_name(&restore);
579            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
580            match job_state(&restore, &ctx, &name).await? {
581                JobState::Complete => {
582                    let mut status = restore.status.clone().unwrap_or_default();
583                    status.restore_job_name = Some(name);
584                    status.phase = KanidmRestorePhase::Verifying;
585                    status.message = None;
586                    patch_status(&restore, &ctx, status).await?;
587                }
588                JobState::Failed => {
589                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
590                }
591                JobState::Running => {}
592            }
593            Ok(Action::requeue(REQUEUE))
594        }
595        KanidmRestorePhase::Verifying => {
596            let target = get_target(&restore, &ctx).await?;
597            let name = verify_job_name(&restore);
598            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
599            match job_state(&restore, &ctx, &name).await? {
600                JobState::Complete => {
601                    let mut status = restore.status.clone().unwrap_or_default();
602                    status.verify_job_name = Some(name);
603                    status.phase = KanidmRestorePhase::RebuildingReplicas;
604                    status.message = None;
605                    patch_status(&restore, &ctx, status).await?;
606                }
607                JobState::Failed => {
608                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
609                }
610                JobState::Running => {}
611            }
612            Ok(Action::requeue(REQUEUE))
613        }
614        KanidmRestorePhase::RebuildingReplicas => {
615            let target = get_target(&restore, &ctx).await?;
616            let mut status = restore.status.clone().unwrap_or_default();
617            if !status.replicas_cleared {
618                if delete_secondary_pvcs(&target, &ctx).await? {
619                    status.replicas_cleared = true;
620                    status.message = Some("secondary database state cleared".to_string());
621                    patch_status(&restore, &ctx, status).await?;
622                }
623                return Ok(Action::requeue(REQUEUE));
624            }
625
626            scale_primary(&target, &ctx, 1).await?;
627            if !primary_ready(&target, &ctx).await? {
628                return Ok(Action::requeue(REQUEUE));
629            }
630            if !status.certificates_cleared {
631                delete_replica_cert_secrets(&target, &ctx).await?;
632                delete_admin_secret(&target, &ctx).await?;
633                status.certificates_cleared = true;
634                status.message = Some(
635                    "replica certificates and admin secret cleared for regeneration".to_string(),
636                );
637                patch_status(&restore, &ctx, status).await?;
638                return Ok(Action::requeue(REQUEUE));
639            }
640            status.phase = KanidmRestorePhase::Resuming;
641            status.message = None;
642            patch_status(&restore, &ctx, status).await?;
643            Ok(Action::requeue(REQUEUE))
644        }
645        KanidmRestorePhase::Resuming => {
646            let target = get_target(&restore, &ctx).await?;
647            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
648            clear_restoring(&restore, &target, &ctx).await?;
649            Ok(Action::requeue(Duration::from_secs(3600)))
650        }
651        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
652            Ok(Action::requeue(Duration::from_secs(3600)))
653        }
654    }
655}
656
657async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
658    let status = restore.status.clone().unwrap_or_default();
659    if status.database_mutation_started
660        && status.phase != KanidmRestorePhase::Completed
661        && status.phase != KanidmRestorePhase::Failed
662    {
663        reconcile_apply(restore.clone(), ctx.clone()).await?;
664        return Ok(Action::requeue(REQUEUE));
665    }
666    if let Ok(target) = get_target(&restore, &ctx).await {
667        let owns_maintenance =
668            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
669        if owns_maintenance && !status.database_mutation_started {
670            scale_desired(&target, &ctx).await?;
671        }
672        clear_restoring(&restore, &target, &ctx).await?;
673    }
674    Ok(Action::await_change())
675}
676
677async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
678    let target = get_target(restore, ctx).await?;
679    let actual_uid = target
680        .uid()
681        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
682    if actual_uid != restore.spec.target_ref.uid {
683        return Err(Error::MissingData(format!(
684            "target UID mismatch: expected {}, got {}",
685            restore.spec.target_ref.uid, actual_uid
686        )));
687    }
688    validate_source(restore)?;
689    if is_remote_source(restore) {
690        validate_backup_ref(restore, &target, ctx).await?;
691    }
692    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
693    {
694        return Err(Error::MissingData(format!(
695            "restoreImage must be the target's pinned Kanidm image (target image is {})",
696            target.spec.image
697        )));
698    }
699    let storage = target
700        .spec
701        .storage
702        .as_ref()
703        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
704    if storage.empty_dir.is_some()
705        || storage.ephemeral.is_some()
706        || storage.volume_claim_template.is_none()
707    {
708        return Err(Error::MissingData(
709            "restore requires PVC-backed Kanidm storage".to_string(),
710        ));
711    }
712    let primaries = target
713        .spec
714        .replica_groups
715        .iter()
716        .filter(|rg| rg.primary_node)
717        .count();
718    if primaries != 1 {
719        return Err(Error::MissingData(
720            "backup/restore requires exactly one primary replica group".to_string(),
721        ));
722    }
723    let ns = restore.namespace().unwrap();
724    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
725        .list(&ListParams::default())
726        .await
727        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
728    if restores.items.iter().any(|other| {
729        other.name_any() != restore.name_any()
730            && other.spec.target_ref.name == restore.spec.target_ref.name
731            && !matches!(
732                other.status.as_ref().map(|s| s.phase),
733                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
734            )
735    }) {
736        return Err(Error::MissingData(
737            "another active restore targets this Kanidm".to_string(),
738        ));
739    }
740    validate_safety_backup_config(restore)?;
741    Ok(target)
742}
743
744fn safe_basename(name: &str) -> bool {
745    !name.is_empty()
746        && name != "."
747        && name != ".."
748        && !name.contains('/')
749        && !name.contains('\\')
750        && !name.contains('\0')
751}
752
753fn mutable_image(image: &str) -> bool {
754    image == "kanidm/server:latest"
755        || image.ends_with(":latest")
756        || (!image.contains('@')
757            && !image
758                .rsplit('/')
759                .next()
760                .is_some_and(|part| part.contains(':')))
761}
762
763async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
764    let ns = restore
765        .namespace()
766        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
767    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
768        .get(&restore.spec.target_ref.name)
769        .await
770        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
771}
772
773async fn mark_restoring(
774    restore: &KanidmRestore,
775    target: &Kanidm,
776    ctx: &RestoreContext,
777) -> Result<()> {
778    let ns = target.namespace().unwrap();
779    let uid = restore
780        .uid()
781        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
782    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
783        .patch(
784            &target.name_any(),
785            &PatchParams::default(),
786            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
787        )
788        .await
789        .map(|_| ())
790        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
791}
792
793async fn clear_restoring(
794    restore: &KanidmRestore,
795    target: &Kanidm,
796    ctx: &RestoreContext,
797) -> Result<()> {
798    let ns = target.namespace().unwrap();
799    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
800        return Ok(());
801    }
802    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
803        .patch(
804            &target.name_any(),
805            &PatchParams::default(),
806            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
807        )
808        .await
809        .map(|_| ())
810        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
811}
812
813async fn patch_status(
814    restore: &KanidmRestore,
815    ctx: &RestoreContext,
816    mut status: KanidmRestoreStatus,
817) -> Result<()> {
818    let ns = restore.namespace().unwrap();
819    let previous_phase = restore.status.as_ref().map(|current| current.phase);
820    let next_phase = status.phase;
821    status.observed_generation = restore.metadata.generation;
822    update_restore_conditions(&mut status, restore.metadata.generation);
823
824    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
825        .patch_status(
826            &restore.name_any(),
827            &PatchParams::default(),
828            &Patch::Merge(json!({"status": &status})),
829        )
830        .await
831        .map_err(|e| {
832            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
833        })?;
834
835    if previous_phase != Some(next_phase) {
836        record_restore_transition(
837            restore,
838            ctx,
839            previous_phase,
840            next_phase,
841            status.message.as_deref(),
842        )
843        .await;
844    }
845    Ok(())
846}
847
848fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
849    let previous = status.conditions.clone();
850    let phase = status.phase;
851    let phase_reason = format!("{phase:?}");
852    let message = status
853        .message
854        .clone()
855        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
856    let terminal_success = phase == KanidmRestorePhase::Completed;
857    let terminal_failure = phase == KanidmRestorePhase::Failed;
858    let progressing = !terminal_success && !terminal_failure;
859
860    let break_glass = previous
861        .iter()
862        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
863        .cloned();
864    status.conditions = vec![
865        restore_condition(
866            &previous,
867            CONDITION_PROGRESSING,
868            if progressing {
869                CONDITION_TRUE
870            } else {
871                CONDITION_FALSE
872            },
873            &phase_reason,
874            &message,
875            generation,
876        ),
877        restore_condition(
878            &previous,
879            CONDITION_READY,
880            if terminal_success {
881                CONDITION_TRUE
882            } else {
883                CONDITION_FALSE
884            },
885            if terminal_success {
886                "RestoreCompleted"
887            } else {
888                "RestoreNotCompleted"
889            },
890            if terminal_success {
891                "Kanidm restore completed successfully."
892            } else {
893                "Kanidm restore has not completed successfully."
894            },
895            generation,
896        ),
897        restore_condition(
898            &previous,
899            CONDITION_FAILED,
900            if terminal_failure {
901                CONDITION_TRUE
902            } else {
903                CONDITION_FALSE
904            },
905            if terminal_failure {
906                "RestoreFailed"
907            } else {
908                "NoRestoreFailure"
909            },
910            if terminal_failure {
911                status
912                    .message
913                    .as_deref()
914                    .unwrap_or("Kanidm restore failed.")
915            } else {
916                "No terminal restore failure has been recorded."
917            },
918            generation,
919        ),
920    ];
921    if let Some(condition) = break_glass {
922        status.conditions.push(condition);
923    }
924}
925
926fn restore_condition(
927    previous: &[Condition],
928    condition_type: &str,
929    condition_status: &str,
930    reason: &str,
931    message: &str,
932    generation: Option<i64>,
933) -> Condition {
934    let last_transition_time = previous
935        .iter()
936        .find(|condition| {
937            condition.type_ == condition_type
938                && condition.status == condition_status
939                && condition.reason == reason
940        })
941        .map(|condition| condition.last_transition_time.clone())
942        .unwrap_or_else(|| Time(Timestamp::now()));
943    Condition {
944        type_: condition_type.to_string(),
945        status: condition_status.to_string(),
946        reason: reason.to_string(),
947        message: message.to_string(),
948        last_transition_time,
949        observed_generation: generation,
950    }
951}
952
953async fn record_restore_transition(
954    restore: &KanidmRestore,
955    ctx: &RestoreContext,
956    previous_phase: Option<KanidmRestorePhase>,
957    phase: KanidmRestorePhase,
958    message: Option<&str>,
959) {
960    if phase == KanidmRestorePhase::Validating {
961        ctx.metrics.attempts.add(1, &[]);
962    }
963
964    let result = match phase {
965        KanidmRestorePhase::Completed => Some("success"),
966        KanidmRestorePhase::Failed => Some("failure"),
967        _ => None,
968    };
969    if let Some(result) = result {
970        let attributes = [KeyValue::new("result", result)];
971        ctx.metrics.outcomes.add(1, &attributes);
972        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
973            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
974            ctx.metrics.duration_seconds.record(elapsed, &attributes);
975        }
976    }
977
978    let reason = if phase == KanidmRestorePhase::Failed {
979        "RestoreFailed"
980    } else {
981        "RestorePhaseChanged"
982    };
983    let note = message.map(str::to_string).or_else(|| {
984        Some(format!(
985            "Kanidm restore phase changed from {} to {phase:?}.",
986            previous_phase
987                .map(|previous| format!("{previous:?}"))
988                .unwrap_or_else(|| "None".to_string())
989        ))
990    });
991    if let Err(error) = ctx
992        .recorder
993        .publish(
994            &Event {
995                type_: if phase == KanidmRestorePhase::Failed {
996                    EventType::Warning
997                } else {
998                    EventType::Normal
999                },
1000                reason: reason.to_string(),
1001                note,
1002                action: "Restore".to_string(),
1003                secondary: None,
1004            },
1005            &restore.object_ref(&()),
1006        )
1007        .await
1008    {
1009        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1010    }
1011}
1012
1013async fn set_phase(
1014    restore: &KanidmRestore,
1015    ctx: &RestoreContext,
1016    phase: KanidmRestorePhase,
1017    message: Option<String>,
1018) -> Result<()> {
1019    let mut status = restore.status.clone().unwrap_or_default();
1020    status.phase = phase;
1021    status.message = message;
1022    patch_status(restore, ctx, status).await
1023}
1024
1025async fn fail_after_mutation(
1026    restore: &KanidmRestore,
1027    ctx: &RestoreContext,
1028    message: &str,
1029) -> Result<()> {
1030    set_phase(
1031        restore,
1032        ctx,
1033        KanidmRestorePhase::Failed,
1034        Some(message.to_string()),
1035    )
1036    .await
1037}
1038
1039fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1040    target
1041        .spec
1042        .replica_groups
1043        .iter()
1044        .find(|rg| rg.primary_node)
1045        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1046}
1047
1048async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1049    let ns = target.namespace().unwrap();
1050    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1051    for rg in &target.spec.replica_groups {
1052        let name = target.statefulset_name(&rg.name);
1053        api.patch(
1054            &name,
1055            &PatchParams::default(),
1056            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1057        )
1058        .await
1059        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1060    }
1061    Ok(())
1062}
1063
1064async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1065    let ns = target.namespace().unwrap();
1066    let rg = primary_group(target)?;
1067    let name = target.statefulset_name(&rg.name);
1068    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1069        .patch(
1070            &name,
1071            &PatchParams::default(),
1072            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1073        )
1074        .await
1075        .map(|_| ())
1076        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1077}
1078
1079async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1080    let ns = target.namespace().unwrap();
1081    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1082    for rg in &target.spec.replica_groups {
1083        let name = target.statefulset_name(&rg.name);
1084        api.patch(
1085            &name,
1086            &PatchParams::default(),
1087            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1088        )
1089        .await
1090        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1091    }
1092    Ok(())
1093}
1094
1095async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1096    let ns = target.namespace().unwrap();
1097    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1098        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1099        .await
1100        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1101    Ok(pods.items.is_empty())
1102}
1103
1104async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1105    let ns = target.namespace().unwrap();
1106    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1107    let mut pv_names = BTreeSet::new();
1108    let primary_name = primary_pvc_name(target)?;
1109    let primary_pvc = pvc_api
1110        .get(&primary_name)
1111        .await
1112        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1113    let primary_pv = primary_pvc
1114        .spec
1115        .as_ref()
1116        .and_then(|spec| spec.volume_name.clone())
1117        .ok_or_else(|| {
1118            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1119        })?;
1120    pv_names.insert(primary_pv);
1121
1122    for rg in &target.spec.replica_groups {
1123        let sts = target.statefulset_name(&rg.name);
1124        for ordinal in 0..rg.replicas {
1125            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1126            if name == primary_name {
1127                continue;
1128            }
1129            if let Some(pvc) = pvc_api
1130                .get_opt(&name)
1131                .await
1132                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1133                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1134            {
1135                pv_names.insert(pv_name);
1136            }
1137        }
1138    }
1139
1140    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1141        .list(&ListParams::default())
1142        .await
1143        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1144    Ok(!attachments.items.iter().any(|attachment| {
1145        attachment
1146            .spec
1147            .source
1148            .persistent_volume_name
1149            .as_ref()
1150            .is_some_and(|pv_name| pv_names.contains(pv_name))
1151    }))
1152}
1153
1154async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1155    let ns = target.namespace().unwrap();
1156    let name = target.statefulset_name(&primary_group(target)?.name);
1157    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1158        .get(&name)
1159        .await
1160        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1161    Ok(sts
1162        .status
1163        .as_ref()
1164        .and_then(|s| s.ready_replicas)
1165        .unwrap_or(0)
1166        >= 1)
1167}
1168
1169fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1170    let rg = primary_group(target)?;
1171    Ok(format!(
1172        "{DATA_VOLUME}-{}-0",
1173        target.statefulset_name(&rg.name)
1174    ))
1175}
1176
1177async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1178    let ns = target.namespace().unwrap();
1179    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1180    let mut all_absent = true;
1181    for rg in &target.spec.replica_groups {
1182        let sts = target.statefulset_name(&rg.name);
1183        for ordinal in 0..rg.replicas {
1184            if rg.primary_node && ordinal == 0 {
1185                continue;
1186            }
1187            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1188            match api
1189                .get_opt(&name)
1190                .await
1191                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1192            {
1193                None => {}
1194                Some(pvc) => {
1195                    all_absent = false;
1196                    if pvc.metadata.deletion_timestamp.is_none() {
1197                        match api.delete(&name, &DeleteParams::default()).await {
1198                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1199                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1200                            Err(error) => {
1201                                return Err(Error::kube_error(
1202                                    "delete",
1203                                    "PersistentVolumeClaim",
1204                                    &ns,
1205                                    &name,
1206                                    error,
1207                                ));
1208                            }
1209                        }
1210                    }
1211                }
1212            }
1213        }
1214    }
1215    Ok(all_absent)
1216}
1217
1218async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1219    let ns = target.namespace().unwrap();
1220    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1221    let label_selector = format!(
1222        "{}={},{}={}",
1223        CLUSTER_LABEL,
1224        target.name_any(),
1225        SECRET_TYPE_LABEL,
1226        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1227    );
1228    let secrets = api
1229        .list(&ListParams::default().labels(&label_selector))
1230        .await
1231        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1232    let mut all_absent = secrets.items.is_empty();
1233    for secret in secrets.items {
1234        let name = secret.name_any();
1235        if secret.metadata.deletion_timestamp.is_none() {
1236            match api.delete(&name, &DeleteParams::default()).await {
1237                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1238                Err(kube::Error::Api(status)) if status.code == 404 => {}
1239                Err(error) => {
1240                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1241                }
1242            }
1243            all_absent = false;
1244        }
1245    }
1246    Ok(all_absent)
1247}
1248
1249async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1250    let ns = target.namespace().unwrap();
1251    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1252    let name = format!("{}-admin-passwords", target.name_any());
1253    match api
1254        .get_opt(&name)
1255        .await
1256        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1257    {
1258        None => Ok(true),
1259        Some(secret) => {
1260            if secret.metadata.deletion_timestamp.is_some() {
1261                return Ok(false);
1262            }
1263            match api.delete(&name, &DeleteParams::default()).await {
1264                Ok(_) => {
1265                    debug!(secret = %name, "deleting stale admin secret");
1266                    Ok(false)
1267                }
1268                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1269                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1270            }
1271        }
1272    }
1273}
1274
1275fn restore_job_name(restore: &KanidmRestore) -> String {
1276    format!("{}-restore", restore.name_any())
1277}
1278fn verify_job_name(restore: &KanidmRestore) -> String {
1279    format!("{}-verify", restore.name_any())
1280}
1281
1282fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1283    let origin = target
1284        .spec
1285        .origin
1286        .clone()
1287        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1288    vec![
1289        EnvVar {
1290            name: "KANIDM_DB_PATH".to_string(),
1291            value: Some("/data/kanidm.db".to_string()),
1292            ..Default::default()
1293        },
1294        EnvVar {
1295            name: "KANIDM_DOMAIN".to_string(),
1296            value: Some(target.spec.domain.clone()),
1297            ..Default::default()
1298        },
1299        EnvVar {
1300            name: "KANIDM_ORIGIN".to_string(),
1301            value: Some(origin),
1302            ..Default::default()
1303        },
1304        EnvVar {
1305            name: "KANIDM_TLS_CHAIN".to_string(),
1306            value: Some(format!("{TLS_PATH}/tls.crt")),
1307            ..Default::default()
1308        },
1309        EnvVar {
1310            name: "KANIDM_TLS_KEY".to_string(),
1311            value: Some(format!("{TLS_PATH}/tls.key")),
1312            ..Default::default()
1313        },
1314    ]
1315}
1316
1317async fn ensure_database_job(
1318    restore: &KanidmRestore,
1319    target: &Kanidm,
1320    ctx: &RestoreContext,
1321    name: &str,
1322    verify: bool,
1323) -> Result<()> {
1324    let ns = restore.namespace().unwrap();
1325    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1326    if api
1327        .get_opt(name)
1328        .await
1329        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1330        .is_some()
1331    {
1332        return Ok(());
1333    }
1334    let mut command = vec![
1335        "kanidmd".to_string(),
1336        "database".to_string(),
1337        if verify { "backup" } else { "restore" }.to_string(),
1338    ];
1339    if verify {
1340        command.push("/verify/verification.json.gz".to_string());
1341    } else {
1342        if is_remote_source(restore) {
1343            command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1344        } else {
1345            let file_name = restore
1346                .spec
1347                .source
1348                .local
1349                .as_ref()
1350                .map(|l| l.file_name.clone())
1351                .unwrap_or_default();
1352            command.push(format!("{BACKUP_PATH}/{file_name}"));
1353        }
1354    }
1355    let job = Job {
1356        metadata: ObjectMeta {
1357            name: Some(name.to_string()),
1358            namespace: Some(ns.clone()),
1359            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1360            ..Default::default()
1361        },
1362        spec: Some(JobSpec {
1363            backoff_limit: Some(0),
1364            template: PodTemplateSpec {
1365                metadata: Some(ObjectMeta::default()),
1366                spec: Some(PodSpec {
1367                    automount_service_account_token: Some(false),
1368                    restart_policy: Some("Never".to_string()),
1369                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1370                        seccomp_profile: Some(SeccompProfile {
1371                            type_: "RuntimeDefault".to_string(),
1372                            ..Default::default()
1373                        }),
1374                        ..Default::default()
1375                    }),
1376                    containers: vec![Container {
1377                        name: if verify { "verify" } else { "restore" }.to_string(),
1378                        image: Some(restore.spec.restore_image.clone()),
1379                        command: Some(command),
1380                        env: Some(build_kanidm_db_env_vars(target)),
1381                        security_context: Some(kanidm_job_security_context()),
1382                        resources: Some(job_resource_requirements()),
1383                        volume_mounts: Some(vec![
1384                            VolumeMount {
1385                                name: DATA_VOLUME.to_string(),
1386                                mount_path: DATA_PATH.to_string(),
1387                                ..Default::default()
1388                            },
1389                            VolumeMount {
1390                                name: TLS_VOLUME.to_string(),
1391                                mount_path: TLS_PATH.to_string(),
1392                                read_only: Some(true),
1393                                ..Default::default()
1394                            },
1395                            VolumeMount {
1396                                name: "verification-output".to_string(),
1397                                mount_path: "/verify".to_string(),
1398                                ..Default::default()
1399                            },
1400                        ]),
1401                        ..Default::default()
1402                    }],
1403                    volumes: Some(vec![
1404                        Volume {
1405                            name: DATA_VOLUME.to_string(),
1406                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1407                                claim_name: primary_pvc_name(target)?,
1408                                read_only: Some(false),
1409                            }),
1410                            ..Default::default()
1411                        },
1412                        Volume {
1413                            name: TLS_VOLUME.to_string(),
1414                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1415                                secret_name: Some(target.effective_tls_secret_name()),
1416                                ..Default::default()
1417                            }),
1418                            ..Default::default()
1419                        },
1420                        Volume {
1421                            name: "verification-output".to_string(),
1422                            empty_dir: Some(Default::default()),
1423                            ..Default::default()
1424                        },
1425                    ]),
1426                    ..Default::default()
1427                }),
1428            },
1429            ..Default::default()
1430        }),
1431        ..Default::default()
1432    };
1433    api.create(&PostParams::default(), &job)
1434        .await
1435        .map(|_| ())
1436        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1437}
1438
1439enum JobState {
1440    Running,
1441    Complete,
1442    Failed,
1443}
1444
1445async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1446    let ns = restore.namespace().unwrap();
1447    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1448        .get(name)
1449        .await
1450        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1451    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1452        return Ok(JobState::Failed);
1453    }
1454    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1455        return Ok(JobState::Complete);
1456    }
1457    Ok(JobState::Running)
1458}
1459
1460fn is_remote_source(restore: &KanidmRestore) -> bool {
1461    restore.spec.source.backup_ref.is_some()
1462}
1463
1464fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1465    let safety = restore.spec.safety_backup.as_ref();
1466    let skip = safety.is_some_and(|s| s.skip);
1467    !skip
1468}
1469
1470fn validate_source(restore: &KanidmRestore) -> Result<()> {
1471    let has_local = restore.spec.source.local.is_some();
1472    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1473    if has_local && has_backup_ref {
1474        return Err(Error::MissingData(
1475            "source.local and source.backupRef are mutually exclusive".to_string(),
1476        ));
1477    }
1478    if !has_local && !has_backup_ref {
1479        return Err(Error::MissingData(
1480            "source must specify either local or backupRef".to_string(),
1481        ));
1482    }
1483    if let Some(local) = &restore.spec.source.local {
1484        if !safe_basename(&local.file_name) {
1485            return Err(Error::MissingData(
1486                "restore source fileName must be a safe basename".to_string(),
1487            ));
1488        }
1489    }
1490    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1491        if backup_ref.name.is_empty() {
1492            return Err(Error::MissingData(
1493                "source.backupRef.name must not be empty".to_string(),
1494            ));
1495        }
1496    }
1497    Ok(())
1498}
1499
1500fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1501    let safety = restore.spec.safety_backup.as_ref();
1502    let skip = safety.is_some_and(|s| s.skip);
1503    if skip {
1504        let annotations = restore.metadata.annotations.as_ref();
1505        let reason = annotations
1506            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1507            .map(|s| s.trim())
1508            .unwrap_or("");
1509        let approver = annotations
1510            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1511            .map(|s| s.trim())
1512            .unwrap_or("");
1513        if reason.is_empty() {
1514            return Err(Error::MissingData(format!(
1515                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1516            )));
1517        }
1518        if approver.is_empty() {
1519            return Err(Error::MissingData(format!(
1520                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1521            )));
1522        }
1523    }
1524    if is_remote_source(restore) && !skip {
1525        let has_repo = safety
1526            .and_then(|s| s.repository_ref.as_ref())
1527            .is_some_and(|r| !r.name.is_empty());
1528        if !has_repo {
1529            return Err(Error::MissingData(
1530                "remote restore requires safetyBackup.repositoryRef".to_string(),
1531            ));
1532        }
1533    }
1534    Ok(())
1535}
1536
1537fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1538    target
1539        .spec
1540        .replica_groups
1541        .iter()
1542        .map(|rg| ReplicaCountEntry {
1543            group: rg.name.clone(),
1544            replicas: rg.replicas,
1545        })
1546        .collect()
1547}
1548
1549async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1550    if let Ok(target) = get_target(restore, ctx).await {
1551        let owns_maintenance =
1552            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1553        if owns_maintenance {
1554            scale_desired(&target, ctx).await?;
1555            clear_restoring(restore, &target, ctx).await?;
1556        }
1557    }
1558    Ok(())
1559}
1560
1561async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1562    ctx.metrics.break_glass_total.add(1, &[]);
1563    let reason = restore
1564        .metadata
1565        .annotations
1566        .as_ref()
1567        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1568        .cloned()
1569        .unwrap_or_default();
1570    let approver = restore
1571        .metadata
1572        .annotations
1573        .as_ref()
1574        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1575        .cloned()
1576        .unwrap_or_default();
1577    let mut status = restore.status.clone().unwrap_or_default();
1578    let break_glass_condition = restore_condition(
1579        &status.conditions,
1580        CONDITION_BREAK_GLASS,
1581        CONDITION_TRUE,
1582        "BreakGlassOverride",
1583        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1584        restore.metadata.generation,
1585    );
1586    status
1587        .conditions
1588        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1589    status.conditions.push(break_glass_condition);
1590    patch_status(restore, ctx, status).await?;
1591    if let Err(error) = ctx
1592        .recorder
1593        .publish(
1594            &Event {
1595                type_: EventType::Warning,
1596                reason: "SafetyBackupSkipped".to_string(),
1597                note: Some(format!(
1598                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1599                )),
1600                action: "Restore".to_string(),
1601                secondary: None,
1602            },
1603            &restore.object_ref(&()),
1604        )
1605        .await
1606    {
1607        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1608    }
1609    warn!(
1610        restore = %restore.name_any(),
1611        reason = %reason,
1612        approver = %approver,
1613        "break-glass safety backup override activated"
1614    );
1615    Ok(())
1616}
1617
1618async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1619    let status = restore.status.as_ref().cloned().unwrap_or_default();
1620    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1621        if let Ok(start_ts) = start.parse::<Timestamp>() {
1622            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1623            ctx.metrics
1624                .safety_backup_duration_seconds
1625                .record(elapsed, &[]);
1626        }
1627    }
1628}
1629
1630fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1631    format!("{}-safety-backup", restore.name_any())
1632}
1633
1634fn source_prep_job_name(restore: &KanidmRestore) -> String {
1635    format!("{}-source-prep", restore.name_any())
1636}
1637
1638fn source_check_job_name(restore: &KanidmRestore) -> String {
1639    format!("{}-source-check", restore.name_any())
1640}
1641
1642async fn ensure_source_check_job(
1643    restore: &KanidmRestore,
1644    target: &Kanidm,
1645    ctx: &RestoreContext,
1646    name: &str,
1647) -> Result<()> {
1648    let ns = restore.namespace().unwrap();
1649    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1650    if api
1651        .get_opt(name)
1652        .await
1653        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1654        .is_some()
1655    {
1656        return Ok(());
1657    }
1658    let file_name = restore
1659        .spec
1660        .source
1661        .local
1662        .as_ref()
1663        .map(|l| l.file_name.clone())
1664        .unwrap_or_default();
1665    let operation_doc = serde_json::json!({
1666        "apiVersion": "backup.kaniop.rs/v1alpha1",
1667        "kind": "OperationDocument",
1668        "operation": "check",
1669        "path": format!("{BACKUP_PATH}/{file_name}"),
1670        "resultPath": "/run/kaniop-result/result.json",
1671    })
1672    .to_string();
1673    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1674    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1675    let data_mover_image = data_mover_image();
1676    let job = Job {
1677        metadata: ObjectMeta {
1678            name: Some(name.to_string()),
1679            namespace: Some(ns.clone()),
1680            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1681            ..Default::default()
1682        },
1683        spec: Some(JobSpec {
1684            backoff_limit: Some(0),
1685            template: PodTemplateSpec {
1686                metadata: Some(ObjectMeta::default()),
1687                spec: Some(PodSpec {
1688                    automount_service_account_token: Some(false),
1689                    restart_policy: Some("Never".to_string()),
1690                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1691                        run_as_non_root: Some(true),
1692                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1693                            type_: "RuntimeDefault".to_string(),
1694                            ..Default::default()
1695                        }),
1696                        ..Default::default()
1697                    }),
1698                    containers: vec![Container {
1699                        name: "source-check".to_string(),
1700                        image: Some(data_mover_image),
1701                        command: Some(vec![
1702                            "/bin/kaniop-data-mover".to_string(),
1703                            "check".to_string(),
1704                            "--operation-doc".to_string(),
1705                            "/run/kaniop/operation.json".to_string(),
1706                        ]),
1707                        security_context: Some(hardened_security_context()),
1708                        resources: Some(job_resource_requirements()),
1709                        env: Some(vec![EnvVar {
1710                            name: "RUST_LOG".to_string(),
1711                            value: Some("info".to_string()),
1712                            ..Default::default()
1713                        }]),
1714                        volume_mounts: Some(vec![
1715                            VolumeMount {
1716                                name: DATA_VOLUME.to_string(),
1717                                mount_path: DATA_PATH.to_string(),
1718                                read_only: Some(true),
1719                                ..Default::default()
1720                            },
1721                            VolumeMount {
1722                                name: "operation".to_string(),
1723                                mount_path: "/run/kaniop".to_string(),
1724                                read_only: Some(true),
1725                                ..Default::default()
1726                            },
1727                            VolumeMount {
1728                                name: "result".to_string(),
1729                                mount_path: "/run/kaniop-result".to_string(),
1730                                ..Default::default()
1731                            },
1732                        ]),
1733                        ..Default::default()
1734                    }],
1735                    volumes: Some(vec![
1736                        Volume {
1737                            name: DATA_VOLUME.to_string(),
1738                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1739                                claim_name: primary_pvc_name(target)?,
1740                                read_only: Some(true),
1741                            }),
1742                            ..Default::default()
1743                        },
1744                        Volume {
1745                            name: "operation".to_string(),
1746                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1747                                name: operation_cm_name,
1748                                ..Default::default()
1749                            }),
1750                            ..Default::default()
1751                        },
1752                        Volume {
1753                            name: "result".to_string(),
1754                            empty_dir: Some(Default::default()),
1755                            ..Default::default()
1756                        },
1757                    ]),
1758                    ..Default::default()
1759                }),
1760            },
1761            ..Default::default()
1762        }),
1763        ..Default::default()
1764    };
1765    api.create(&PostParams::default(), &job)
1766        .await
1767        .map(|_| ())
1768        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1769}
1770
1771const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1772
1773fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1774    let ns = restore.namespace().unwrap_or_default();
1775    let name = restore.name_any();
1776    let namespace_uuid =
1777        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1778    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1779    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1780}
1781
1782fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1783    let ns = restore.namespace().unwrap_or_default();
1784    let kanidm_uid = &restore.spec.target_ref.uid;
1785    let backup_id = compute_safety_backup_id(restore);
1786    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1787}
1788
1789struct VerifiedSafetyBackup {
1790    manifest_key: String,
1791    payload_sha256: String,
1792}
1793
1794async fn read_safety_backup_result(
1795    restore: &KanidmRestore,
1796    ctx: &RestoreContext,
1797    job_name: &str,
1798    expected_backup_id: &str,
1799) -> Result<VerifiedSafetyBackup> {
1800    let ns = restore.namespace().unwrap();
1801    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1802    let pod_list = pods
1803        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1804        .await
1805        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1806    let pod = pod_list.items.first().ok_or_else(|| {
1807        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1808    })?;
1809    let pod_name = pod.name_any();
1810    let container_status = pod
1811        .status
1812        .as_ref()
1813        .and_then(|s| s.container_statuses.as_ref())
1814        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1815        .ok_or_else(|| {
1816            Error::MissingData(format!(
1817                "safety-upload container status not found in pod {pod_name}"
1818            ))
1819        })?;
1820    let termination_message = container_status
1821        .state
1822        .as_ref()
1823        .and_then(|state| state.terminated.as_ref())
1824        .and_then(|t| t.message.as_ref())
1825        .ok_or_else(|| {
1826            Error::MissingData(format!(
1827                "termination message not found in pod {pod_name}; result transport failed"
1828            ))
1829        })?;
1830    if termination_message.is_empty() {
1831        return Err(Error::MissingData(format!(
1832            "termination message is empty in pod {pod_name}; result document was not written"
1833        )));
1834    }
1835    let result_doc = parse_result_document(termination_message).map_err(|e| {
1836        Error::ParseError(format!(
1837            "failed to parse result document from pod {pod_name}: {e}"
1838        ))
1839    })?;
1840    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1841        return Err(Error::ParseError(format!(
1842            "unsupported result document version: {}",
1843            result_doc.api_version
1844        )));
1845    }
1846    if result_doc.kind != "ResultDocument" {
1847        return Err(Error::ParseError(format!(
1848            "invalid result document kind: {}",
1849            result_doc.kind
1850        )));
1851    }
1852    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1853        return Err(Error::ParseError(format!(
1854            "result document operation mismatch: expected '{}', got '{}'",
1855            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1856        )));
1857    }
1858    if !result_doc.success {
1859        return Err(Error::ParseError(format!(
1860            "safety backup reported failure with exit code {:?}",
1861            result_doc.exit_code
1862        )));
1863    }
1864    if result_doc.exit_code != ExitCode::Success {
1865        return Err(Error::ParseError(format!(
1866            "safety backup exit code is not Success: {:?}",
1867            result_doc.exit_code
1868        )));
1869    }
1870    let doc_backup_id = result_doc
1871        .backup_id
1872        .as_deref()
1873        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1874    if doc_backup_id != expected_backup_id {
1875        return Err(Error::ParseError(format!(
1876            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1877        )));
1878    }
1879    let manifest_key = result_doc
1880        .manifest_key
1881        .clone()
1882        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1883    if manifest_key.is_empty() {
1884        return Err(Error::ParseError(
1885            "result document manifestKey is empty".to_string(),
1886        ));
1887    }
1888    let payload_sha256 = result_doc
1889        .payload_sha256
1890        .clone()
1891        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1892    if payload_sha256.is_empty() {
1893        return Err(Error::ParseError(
1894            "result document payloadSha256 is empty".to_string(),
1895        ));
1896    }
1897    info!(
1898        restore = %restore.name_any(),
1899        backup_id = %expected_backup_id,
1900        manifest_key = %manifest_key,
1901        "safety backup result verified"
1902    );
1903    Ok(VerifiedSafetyBackup {
1904        manifest_key,
1905        payload_sha256,
1906    })
1907}
1908
1909struct VerifiedSourcePrepResult {
1910    #[allow(dead_code)]
1911    manifest_key: String,
1912    #[allow(dead_code)]
1913    payload_sha256: String,
1914}
1915
1916async fn read_source_prep_result(
1917    restore: &KanidmRestore,
1918    ctx: &RestoreContext,
1919    job_name: &str,
1920    expected_backup_id: &str,
1921) -> Result<VerifiedSourcePrepResult> {
1922    let ns = restore.namespace().unwrap();
1923    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924    let pod_list = pods
1925        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926        .await
1927        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928    let pod = pod_list.items.first().ok_or_else(|| {
1929        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1930    })?;
1931    let pod_name = pod.name_any();
1932    let container_status = pod
1933        .status
1934        .as_ref()
1935        .and_then(|s| s.container_statuses.as_ref())
1936        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1937        .ok_or_else(|| {
1938            Error::MissingData(format!(
1939                "source-prep container status not found in pod {pod_name}"
1940            ))
1941        })?;
1942    let termination_message = container_status
1943        .state
1944        .as_ref()
1945        .and_then(|state| state.terminated.as_ref())
1946        .and_then(|t| t.message.as_ref())
1947        .ok_or_else(|| {
1948            Error::MissingData(format!(
1949                "termination message not found in pod {pod_name}; result transport failed"
1950            ))
1951        })?;
1952    if termination_message.is_empty() {
1953        return Err(Error::MissingData(format!(
1954            "termination message is empty in pod {pod_name}; result document was not written"
1955        )));
1956    }
1957    let result_doc = parse_result_document(termination_message).map_err(|e| {
1958        Error::ParseError(format!(
1959            "failed to parse result document from pod {pod_name}: {e}"
1960        ))
1961    })?;
1962    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963        return Err(Error::ParseError(format!(
1964            "unsupported result document version: {}",
1965            result_doc.api_version
1966        )));
1967    }
1968    if result_doc.kind != "ResultDocument" {
1969        return Err(Error::ParseError(format!(
1970            "invalid result document kind: {}",
1971            result_doc.kind
1972        )));
1973    }
1974    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1975        return Err(Error::ParseError(format!(
1976            "result document operation mismatch: expected '{}', got '{}'",
1977            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1978        )));
1979    }
1980    if !result_doc.success {
1981        return Err(Error::ParseError(format!(
1982            "source prep reported failure with exit code {:?}",
1983            result_doc.exit_code
1984        )));
1985    }
1986    if result_doc.exit_code != ExitCode::Success {
1987        return Err(Error::ParseError(format!(
1988            "source prep exit code is not Success: {:?}",
1989            result_doc.exit_code
1990        )));
1991    }
1992    let doc_backup_id = result_doc
1993        .backup_id
1994        .as_deref()
1995        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996    if doc_backup_id != expected_backup_id {
1997        return Err(Error::ParseError(format!(
1998            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999        )));
2000    }
2001    let manifest_key = result_doc
2002        .manifest_key
2003        .clone()
2004        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005    if manifest_key.is_empty() {
2006        return Err(Error::ParseError(
2007            "result document manifestKey is empty".to_string(),
2008        ));
2009    }
2010    let payload_sha256 = result_doc
2011        .payload_sha256
2012        .clone()
2013        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014    if payload_sha256.is_empty() {
2015        return Err(Error::ParseError(
2016            "result document payloadSha256 is empty".to_string(),
2017        ));
2018    }
2019    info!(
2020        restore = %restore.name_any(),
2021        backup_id = %expected_backup_id,
2022        manifest_key = %manifest_key,
2023        payload_sha256 = %payload_sha256,
2024        "source prep result verified"
2025    );
2026    Ok(VerifiedSourcePrepResult {
2027        manifest_key,
2028        payload_sha256,
2029    })
2030}
2031
2032fn hardened_security_context() -> SecurityContext {
2033    SecurityContext {
2034        allow_privilege_escalation: Some(false),
2035        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2036            drop: Some(vec!["ALL".to_string()]),
2037            ..Default::default()
2038        }),
2039        read_only_root_filesystem: Some(true),
2040        run_as_non_root: Some(true),
2041        seccomp_profile: Some(SeccompProfile {
2042            type_: "RuntimeDefault".to_string(),
2043            ..Default::default()
2044        }),
2045        ..Default::default()
2046    }
2047}
2048
2049fn kanidm_job_security_context() -> SecurityContext {
2050    SecurityContext {
2051        allow_privilege_escalation: Some(false),
2052        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2053            drop: Some(vec!["ALL".to_string()]),
2054            ..Default::default()
2055        }),
2056        seccomp_profile: Some(SeccompProfile {
2057            type_: "RuntimeDefault".to_string(),
2058            ..Default::default()
2059        }),
2060        ..Default::default()
2061    }
2062}
2063
2064fn job_resource_requirements() -> ResourceRequirements {
2065    ResourceRequirements {
2066        requests: Some(std::collections::BTreeMap::from([
2067            ("cpu".to_string(), Quantity("100m".to_string())),
2068            ("memory".to_string(), Quantity("128Mi".to_string())),
2069        ])),
2070        limits: Some(std::collections::BTreeMap::from([
2071            ("cpu".to_string(), Quantity("2".to_string())),
2072            ("memory".to_string(), Quantity("2Gi".to_string())),
2073        ])),
2074        ..Default::default()
2075    }
2076}
2077
2078async fn ensure_safety_backup_job(
2079    restore: &KanidmRestore,
2080    target: &Kanidm,
2081    ctx: &RestoreContext,
2082    name: &str,
2083    backup_id: &str,
2084    manifest_key: &str,
2085) -> Result<()> {
2086    let ns = restore.namespace().unwrap();
2087    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2088    if api
2089        .get_opt(name)
2090        .await
2091        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2092        .is_some()
2093    {
2094        return Ok(());
2095    }
2096    let safety = restore
2097        .spec
2098        .safety_backup
2099        .as_ref()
2100        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2101    let repo_ref = safety
2102        .repository_ref
2103        .as_ref()
2104        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2105    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2106        .get(&repo_ref.name)
2107        .await
2108        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2109    let _namespace_uid = &ns;
2110    let _kanidm_uid = &restore.spec.target_ref.uid;
2111    let endpoint = repo
2112        .spec
2113        .s3
2114        .endpoint
2115        .as_deref()
2116        .unwrap_or("https://s3.amazonaws.com");
2117    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2118    let operation_doc = build_safety_upload_operation_doc(
2119        restore,
2120        target,
2121        backup_id,
2122        manifest_key,
2123        &repo.spec.s3.bucket,
2124        &repo.spec.s3.prefix,
2125        endpoint,
2126        region,
2127        repo.spec.s3.force_path_style,
2128        repo.spec.s3.insecure,
2129        repo.spec
2130            .s3
2131            .ca_bundle_ref
2132            .as_deref()
2133            .map(|_| ca_bundle_path())
2134            .as_deref(),
2135    )?;
2136    let operation_cm_name = format!("{name}-op");
2137    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2138    let auth_method = &repo.spec.authentication.writer;
2139    let data_mover_image = data_mover_image();
2140    let init_command = vec![
2141        "kanidmd".to_string(),
2142        "database".to_string(),
2143        "backup".to_string(),
2144        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2145    ];
2146    let upload_script = format!(
2147        r#"set -eu
2148/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2149RESULT_FILE="/run/kaniop-result/result.json"
2150TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2151if [ ! -f "$RESULT_FILE" ]; then
2152  echo "ERROR: result document not found at $RESULT_FILE"
2153  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2154  exit 1
2155fi
2156if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2157  echo "ERROR: safety backup upload reported failure"
2158  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2159  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2160  exit 1
2161fi
2162dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2163echo "safety backup upload completed successfully"
2164"#
2165    );
2166    let job = Job {
2167        metadata: ObjectMeta {
2168            name: Some(name.to_string()),
2169            namespace: Some(ns.clone()),
2170            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2171            ..Default::default()
2172        },
2173        spec: Some(JobSpec {
2174            backoff_limit: Some(0),
2175            template: PodTemplateSpec {
2176                metadata: Some(ObjectMeta::default()),
2177                spec: Some(PodSpec {
2178                    automount_service_account_token: Some(false),
2179                    restart_policy: Some("Never".to_string()),
2180                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2181                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2182                            type_: "RuntimeDefault".to_string(),
2183                            ..Default::default()
2184                        }),
2185                        ..Default::default()
2186                    }),
2187                    init_containers: Some(vec![Container {
2188                        name: "safety-backup".to_string(),
2189                        image: Some(restore.spec.restore_image.clone()),
2190                        command: Some(init_command),
2191                        security_context: Some(SecurityContext {
2192                            allow_privilege_escalation: Some(false),
2193                            capabilities: Some(Capabilities {
2194                                drop: Some(vec!["ALL".to_string()]),
2195                                ..Default::default()
2196                            }),
2197                            read_only_root_filesystem: Some(true),
2198                            seccomp_profile: Some(SeccompProfile {
2199                                type_: "RuntimeDefault".to_string(),
2200                                ..Default::default()
2201                            }),
2202                            ..Default::default()
2203                        }),
2204                        resources: Some(job_resource_requirements()),
2205                        env: Some(build_kanidm_db_env_vars(target)),
2206                        volume_mounts: Some(vec![
2207                            VolumeMount {
2208                                name: DATA_VOLUME.to_string(),
2209                                mount_path: DATA_PATH.to_string(),
2210                                read_only: Some(false),
2211                                ..Default::default()
2212                            },
2213                            VolumeMount {
2214                                name: TLS_VOLUME.to_string(),
2215                                mount_path: TLS_PATH.to_string(),
2216                                read_only: Some(true),
2217                                ..Default::default()
2218                            },
2219                            VolumeMount {
2220                                name: SHARED_VOLUME.to_string(),
2221                                mount_path: SHARED_VOL_PATH.to_string(),
2222                                ..Default::default()
2223                            },
2224                        ]),
2225                        ..Default::default()
2226                    }]),
2227                    containers: vec![Container {
2228                        name: "safety-upload".to_string(),
2229                        image: Some(data_mover_image),
2230                        command: Some(vec!["/bin/sh".to_string()]),
2231                        args: Some(vec!["-c".to_string(), upload_script]),
2232                        security_context: Some(hardened_security_context()),
2233                        resources: Some(job_resource_requirements()),
2234                        env: {
2235                            let mut env_vars =
2236                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2237                            env_vars.push(EnvVar {
2238                                name: "RUST_LOG".to_string(),
2239                                value: Some("info".to_string()),
2240                                ..Default::default()
2241                            });
2242                            if repo.spec.s3.ca_bundle_ref.is_some() {
2243                                env_vars.push(ca_bundle_env_var());
2244                            }
2245                            Some(env_vars)
2246                        },
2247                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2248                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2249                        volume_mounts: {
2250                            let mut mounts = vec![
2251                                VolumeMount {
2252                                    name: SHARED_VOLUME.to_string(),
2253                                    mount_path: SHARED_VOL_PATH.to_string(),
2254                                    read_only: Some(true),
2255                                    ..Default::default()
2256                                },
2257                                VolumeMount {
2258                                    name: "operation".to_string(),
2259                                    mount_path: "/run/kaniop".to_string(),
2260                                    read_only: Some(true),
2261                                    ..Default::default()
2262                                },
2263                                VolumeMount {
2264                                    name: "result".to_string(),
2265                                    mount_path: "/run/kaniop-result".to_string(),
2266                                    ..Default::default()
2267                                },
2268                            ];
2269                            mounts.extend(build_auth_volume_mounts(auth_method));
2270                            if repo.spec.s3.ca_bundle_ref.is_some() {
2271                                mounts.push(build_ca_bundle_volume_mount());
2272                            }
2273                            Some(mounts)
2274                        },
2275                        ..Default::default()
2276                    }],
2277                    volumes: {
2278                        let mut vols = vec![
2279                            Volume {
2280                                name: DATA_VOLUME.to_string(),
2281                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2282                                    claim_name: primary_pvc_name(target)?,
2283                                    read_only: Some(false),
2284                                }),
2285                                ..Default::default()
2286                            },
2287                            Volume {
2288                                name: TLS_VOLUME.to_string(),
2289                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2290                                    secret_name: Some(target.effective_tls_secret_name()),
2291                                    ..Default::default()
2292                                }),
2293                                ..Default::default()
2294                            },
2295                            Volume {
2296                                name: SHARED_VOLUME.to_string(),
2297                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2298                                    size_limit: Some(Quantity("10Gi".to_string())),
2299                                    ..Default::default()
2300                                }),
2301                                ..Default::default()
2302                            },
2303                            Volume {
2304                                name: "operation".to_string(),
2305                                config_map: Some(ConfigMapVolumeSource {
2306                                    name: operation_cm_name,
2307                                    ..Default::default()
2308                                }),
2309                                ..Default::default()
2310                            },
2311                            Volume {
2312                                name: "result".to_string(),
2313                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2314                                    size_limit: Some(Quantity("16Mi".to_string())),
2315                                    ..Default::default()
2316                                }),
2317                                ..Default::default()
2318                            },
2319                        ];
2320                        vols.extend(build_auth_volumes(auth_method));
2321                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2322                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2323                        }
2324                        Some(vols)
2325                    },
2326                    ..Default::default()
2327                }),
2328            },
2329            ..Default::default()
2330        }),
2331        ..Default::default()
2332    };
2333    api.create(&PostParams::default(), &job)
2334        .await
2335        .map(|_| ())
2336        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2337}
2338
2339async fn ensure_source_prep_job(
2340    restore: &KanidmRestore,
2341    target: &Kanidm,
2342    ctx: &RestoreContext,
2343    name: &str,
2344) -> Result<()> {
2345    let ns = restore.namespace().unwrap();
2346    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2347    if api
2348        .get_opt(name)
2349        .await
2350        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2351        .is_some()
2352    {
2353        return Ok(());
2354    }
2355    let backup_name = restore
2356        .spec
2357        .source
2358        .backup_ref
2359        .as_ref()
2360        .map(|b| b.name.clone())
2361        .unwrap_or_default();
2362    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2363        .get(&backup_name)
2364        .await
2365        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2366    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2367        .get(&backup.spec.repository_ref.name)
2368        .await
2369        .map_err(|e| {
2370            Error::kube_error(
2371                "get",
2372                "KanidmBackupRepository",
2373                &ns,
2374                &backup.spec.repository_ref.name,
2375                e,
2376            )
2377        })?;
2378    let endpoint = repo
2379        .spec
2380        .s3
2381        .endpoint
2382        .as_deref()
2383        .unwrap_or("https://s3.amazonaws.com");
2384    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2385    let operation_doc = build_download_operation_doc(
2386        restore,
2387        target,
2388        &backup.spec.manifest_key,
2389        &backup.spec.backup_id,
2390        &repo.spec.s3.bucket,
2391        &repo.spec.s3.prefix,
2392        endpoint,
2393        region,
2394        repo.spec.s3.force_path_style,
2395        repo.spec.s3.insecure,
2396        repo.spec
2397            .s3
2398            .ca_bundle_ref
2399            .as_deref()
2400            .map(|_| ca_bundle_path())
2401            .as_deref(),
2402    );
2403    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2404    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2405    let auth_method = &repo.spec.authentication.reader;
2406    let data_mover_image = data_mover_image();
2407    let download_script = format!(
2408        r#"set -eu
2409/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2410RESULT_FILE="/run/kaniop-result/result.json"
2411TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2412if [ ! -f "$RESULT_FILE" ]; then
2413  echo "ERROR: result document not found at $RESULT_FILE"
2414  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2415  exit 1
2416fi
2417if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2418  echo "ERROR: source preparation download reported failure"
2419  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2420  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2421  exit 1
2422fi
2423dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2424echo "source preparation download completed successfully"
2425"#
2426    );
2427    let job = Job {
2428        metadata: ObjectMeta {
2429            name: Some(name.to_string()),
2430            namespace: Some(ns.clone()),
2431            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2432            ..Default::default()
2433        },
2434        spec: Some(JobSpec {
2435            backoff_limit: Some(0),
2436            template: PodTemplateSpec {
2437                metadata: Some(ObjectMeta::default()),
2438                spec: Some(PodSpec {
2439                    automount_service_account_token: Some(false),
2440                    restart_policy: Some("Never".to_string()),
2441                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2442                        run_as_non_root: Some(true),
2443                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2444                            type_: "RuntimeDefault".to_string(),
2445                            ..Default::default()
2446                        }),
2447                        ..Default::default()
2448                    }),
2449                    containers: vec![Container {
2450                        name: "source-prep".to_string(),
2451                        image: Some(data_mover_image),
2452                        command: Some(vec!["/bin/sh".to_string()]),
2453                        args: Some(vec!["-c".to_string(), download_script]),
2454                        security_context: Some(hardened_security_context()),
2455                        resources: Some(job_resource_requirements()),
2456                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2457                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2458                        env: {
2459                            let mut env_vars = build_auth_env_vars(
2460                                auth_method,
2461                                &backup.spec.repository_ref.name,
2462                                AuthRole::Reader,
2463                            );
2464                            env_vars.push(EnvVar {
2465                                name: "RUST_LOG".to_string(),
2466                                value: Some("info".to_string()),
2467                                ..Default::default()
2468                            });
2469                            if repo.spec.s3.ca_bundle_ref.is_some() {
2470                                env_vars.push(ca_bundle_env_var());
2471                            }
2472                            Some(env_vars)
2473                        },
2474                        volume_mounts: {
2475                            let mut mounts = vec![
2476                                VolumeMount {
2477                                    name: STAGING_VOLUME.to_string(),
2478                                    mount_path: STAGING_PATH.to_string(),
2479                                    ..Default::default()
2480                                },
2481                                VolumeMount {
2482                                    name: "operation".to_string(),
2483                                    mount_path: "/run/kaniop".to_string(),
2484                                    read_only: Some(true),
2485                                    ..Default::default()
2486                                },
2487                                VolumeMount {
2488                                    name: "result".to_string(),
2489                                    mount_path: "/run/kaniop-result".to_string(),
2490                                    ..Default::default()
2491                                },
2492                            ];
2493                            mounts.extend(build_auth_volume_mounts(auth_method));
2494                            if repo.spec.s3.ca_bundle_ref.is_some() {
2495                                mounts.push(build_ca_bundle_volume_mount());
2496                            }
2497                            Some(mounts)
2498                        },
2499                        ..Default::default()
2500                    }],
2501                    volumes: {
2502                        let mut vols = vec![
2503                            Volume {
2504                                name: STAGING_VOLUME.to_string(),
2505                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2506                                    size_limit: Some(Quantity("10Gi".to_string())),
2507                                    ..Default::default()
2508                                }),
2509                                ..Default::default()
2510                            },
2511                            Volume {
2512                                name: "operation".to_string(),
2513                                config_map: Some(ConfigMapVolumeSource {
2514                                    name: operation_cm_name,
2515                                    ..Default::default()
2516                                }),
2517                                ..Default::default()
2518                            },
2519                            Volume {
2520                                name: "result".to_string(),
2521                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2522                                    size_limit: Some(Quantity("16Mi".to_string())),
2523                                    ..Default::default()
2524                                }),
2525                                ..Default::default()
2526                            },
2527                        ];
2528                        vols.extend(build_auth_volumes(auth_method));
2529                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2530                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2531                        }
2532                        Some(vols)
2533                    },
2534                    ..Default::default()
2535                }),
2536            },
2537            ..Default::default()
2538        }),
2539        ..Default::default()
2540    };
2541    api.create(&PostParams::default(), &job)
2542        .await
2543        .map(|_| ())
2544        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2545}
2546fn data_mover_image() -> String {
2547    std::env::var("DATA_MOVER_IMAGE")
2548        .unwrap_or_else(|_| "ghcr.io/pando85/kaniop-data-mover:latest".to_string())
2549}
2550
2551#[allow(clippy::too_many_arguments)]
2552fn build_safety_upload_operation_doc(
2553    restore: &KanidmRestore,
2554    target: &Kanidm,
2555    backup_id: &str,
2556    manifest_key: &str,
2557    bucket: &str,
2558    prefix: &str,
2559    endpoint: &str,
2560    region: &str,
2561    force_path_style: bool,
2562    insecure: bool,
2563    ca_bundle_path: Option<&str>,
2564) -> Result<String> {
2565    let op = UploadOperation {
2566        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2567        bucket: bucket.to_string(),
2568        prefix: prefix.to_string(),
2569        endpoint: endpoint.to_string(),
2570        region: region.to_string(),
2571        force_path_style,
2572        insecure,
2573        ca_bundle_path: ca_bundle_path.map(str::to_string),
2574        backup_id: backup_id.to_string(),
2575        namespace_uid: restore.namespace().unwrap_or_default(),
2576        kanidm_uid: restore.spec.target_ref.uid.clone(),
2577        kanidm_name: restore.spec.target_ref.name.clone(),
2578        domain: target.spec.domain.clone(),
2579        kanidm_version: target
2580            .status
2581            .as_ref()
2582            .and_then(|s| s.version.as_ref())
2583            .map(|v| v.image_tag.clone())
2584            .unwrap_or_default(),
2585        image_digest: None,
2586        consistency: "kanidm-offline".to_string(),
2587        reason: "restore-safety".to_string(),
2588        encryption_mode: None,
2589        encryption_key_id: None,
2590        result_path: "/run/kaniop-result/result.json".to_string(),
2591        max_concurrent_parts: 4,
2592        max_retries: 3,
2593    };
2594    let _ = manifest_key;
2595    let doc = OperationDocument {
2596        api_version: OPERATION_DOC_VERSION.to_string(),
2597        kind: "OperationDocument".to_string(),
2598        spec: OperationSpec::Upload(op),
2599    };
2600    serde_json::to_string(&doc)
2601        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2602}
2603
2604async fn validate_backup_ref(
2605    restore: &KanidmRestore,
2606    target: &Kanidm,
2607    ctx: &RestoreContext,
2608) -> Result<()> {
2609    let backup_ref = restore
2610        .spec
2611        .source
2612        .backup_ref
2613        .as_ref()
2614        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2615    let ns = restore.namespace().unwrap();
2616    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2617        .get(&backup_ref.name)
2618        .await
2619        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2620    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2621        return Err(Error::MissingData(format!(
2622            "KanidmBackup '{}' is not Ready (phase: {:?})",
2623            backup_ref.name,
2624            backup.status.as_ref().map(|s| &s.phase)
2625        )));
2626    }
2627    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2628        return Err(Error::MissingData(format!(
2629            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2630            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2631        )));
2632    }
2633    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2634        return Err(Error::MissingData(format!(
2635            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2636            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2637        )));
2638    }
2639    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2640        .get(&backup.spec.repository_ref.name)
2641        .await
2642        .map_err(|e| {
2643            Error::kube_error(
2644                "get",
2645                "KanidmBackupRepository",
2646                &ns,
2647                &backup.spec.repository_ref.name,
2648                e,
2649            )
2650        })?;
2651    if !has_accepted_condition(
2652        &repo
2653            .status
2654            .as_ref()
2655            .map(|s| &s.conditions)
2656            .cloned()
2657            .unwrap_or_default(),
2658    ) {
2659        return Err(Error::MissingData(format!(
2660            "KanidmBackupRepository '{}' configuration has not been accepted",
2661            backup.spec.repository_ref.name
2662        )));
2663    }
2664    validate_backup_compatibility(restore, target, &backup)?;
2665    Ok(())
2666}
2667
2668fn has_accepted_condition(conditions: &[Condition]) -> bool {
2669    conditions
2670        .iter()
2671        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2672}
2673
2674fn validate_backup_compatibility(
2675    restore: &KanidmRestore,
2676    target: &Kanidm,
2677    backup: &KanidmBackup,
2678) -> Result<()> {
2679    let status = backup.status.as_ref();
2680    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2681        let target_version = target
2682            .status
2683            .as_ref()
2684            .and_then(|s| s.version.as_ref())
2685            .map(|v| v.image_tag.as_str());
2686        if let Some(tv) = target_version {
2687            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2688                return Err(Error::MissingData(format!(
2689                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2690                )));
2691            }
2692        }
2693    }
2694    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2695        if !backup_digest.is_empty()
2696            && restore.spec.restore_image.contains('@')
2697            && !restore.spec.restore_image.contains(backup_digest)
2698        {
2699            return Err(Error::MissingData(format!(
2700                "restore image digest does not match backup image digest '{backup_digest}'"
2701            )));
2702        }
2703    }
2704    Ok(())
2705}
2706
2707#[allow(clippy::too_many_arguments)]
2708fn build_download_operation_doc(
2709    restore: &KanidmRestore,
2710    target: &Kanidm,
2711    manifest_key: &str,
2712    expected_backup_id: &str,
2713    bucket: &str,
2714    prefix: &str,
2715    endpoint: &str,
2716    region: &str,
2717    force_path_style: bool,
2718    insecure: bool,
2719    ca_bundle_path: Option<&str>,
2720) -> String {
2721    serde_json::json!({
2722        "apiVersion": "backup.kaniop.rs/v1alpha1",
2723        "kind": "OperationDocument",
2724        "operation": "download",
2725        "manifestKey": manifest_key,
2726        "bucket": bucket,
2727        "prefix": prefix,
2728        "endpoint": endpoint,
2729        "region": region,
2730        "forcePathStyle": force_path_style,
2731        "insecure": insecure,
2732        "caBundlePath": ca_bundle_path,
2733        "expectedBackupId": expected_backup_id,
2734        "expectedKanidmUid": restore.spec.target_ref.uid,
2735        "expectedDomain": target.spec.domain,
2736        "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2737        "resultPath": "/run/kaniop-result/result.json",
2738        "maxRetries": 3,
2739    })
2740    .to_string()
2741}
2742
2743async fn ensure_operation_configmap(
2744    restore: &KanidmRestore,
2745    name: &str,
2746    data: &str,
2747    ns: &str,
2748    ctx: &RestoreContext,
2749) -> Result<()> {
2750    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2751    if api
2752        .get_opt(name)
2753        .await
2754        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2755        .is_some()
2756    {
2757        return Ok(());
2758    }
2759    let cm = ConfigMap {
2760        metadata: ObjectMeta {
2761            name: Some(name.to_string()),
2762            namespace: Some(ns.to_string()),
2763            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2764            ..Default::default()
2765        },
2766        data: Some(std::collections::BTreeMap::from([(
2767            "operation.json".to_string(),
2768            data.to_string(),
2769        )])),
2770        ..Default::default()
2771    };
2772    api.create(&PostParams::default(), &cm)
2773        .await
2774        .map(|_| ())
2775        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2776}
2777
2778#[cfg(test)]
2779mod tests {
2780    use super::{
2781        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2782        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2783        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2784        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2785        hardened_security_context, has_accepted_condition, is_remote_source,
2786        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2787        validate_safety_backup_config, validate_source,
2788    };
2789    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2790    use kube::api::ObjectMeta as ApiObjectMeta;
2791    use std::collections::BTreeMap;
2792
2793    fn make_restore(
2794        source: KanidmRestoreSource,
2795        safety: Option<SafetyBackupConfig>,
2796    ) -> KanidmRestore {
2797        KanidmRestore {
2798            metadata: ApiObjectMeta {
2799                name: Some("test-restore".to_string()),
2800                namespace: Some("default".to_string()),
2801                ..Default::default()
2802            },
2803            spec: KanidmRestoreSpec {
2804                target_ref: KanidmRestoreTargetRef {
2805                    name: "corp-idm".to_string(),
2806                    uid: "test-uid-123".to_string(),
2807                },
2808                source,
2809                restore_image: "kanidm/server@sha256:abc".to_string(),
2810                safety_backup: safety,
2811            },
2812            status: None,
2813        }
2814    }
2815
2816    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2817        super::super::crd::Kanidm {
2818            status: Some(super::super::crd::KanidmStatus {
2819                available_replicas: 1,
2820                replicas: 1,
2821                unavailable_replicas: 0,
2822                updated_replicas: 1,
2823                replica_statuses: vec![],
2824                replica_column: "1/1".to_string(),
2825                secret_name: None,
2826                version: Some(super::super::crd::KanidmVersionStatus {
2827                    image_tag: image_tag.to_string(),
2828                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2829                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2830                }),
2831                domain_appearance_image: None,
2832                mail_sender: None,
2833                conditions: None,
2834            }),
2835            ..Default::default()
2836        }
2837    }
2838
2839    #[test]
2840    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2841        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2842    }
2843
2844    #[test]
2845    fn completed_restore_sets_ready_condition() {
2846        let mut status = KanidmRestoreStatus {
2847            phase: KanidmRestorePhase::Completed,
2848            ..Default::default()
2849        };
2850        super::update_restore_conditions(&mut status, Some(7));
2851        assert!(status.conditions.iter().any(|condition| {
2852            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2853        }));
2854        assert!(!status.conditions.iter().any(|condition| {
2855            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2856        }));
2857    }
2858
2859    #[test]
2860    fn failed_restore_sets_failed_condition() {
2861        let mut status = KanidmRestoreStatus {
2862            phase: KanidmRestorePhase::Failed,
2863            message: Some("verification failed".to_string()),
2864            ..Default::default()
2865        };
2866        super::update_restore_conditions(&mut status, Some(8));
2867        assert!(status.conditions.iter().any(|condition| {
2868            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2869        }));
2870    }
2871
2872    #[test]
2873    fn restore_filename_is_confined_to_backup_directory() {
2874        assert!(safe_basename("backup.json.gz"));
2875        assert!(!safe_basename("../backup.json.gz"));
2876        assert!(!safe_basename("a/b"));
2877        assert!(!safe_basename(""));
2878    }
2879
2880    #[test]
2881    fn restore_image_rejects_latest_and_unpinned_names() {
2882        assert!(mutable_image("kanidm/server:latest"));
2883        assert!(mutable_image("kanidm/server"));
2884        assert!(!mutable_image("kanidm/server:1.10.0"));
2885        assert!(!mutable_image("kanidm/server@sha256:abc"));
2886    }
2887
2888    #[test]
2889    fn source_validation_rejects_both_local_and_backup_ref() {
2890        let source = KanidmRestoreSource {
2891            local: Some(super::KanidmRestoreLocalSource {
2892                file_name: "backup.json".to_string(),
2893            }),
2894            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2895                name: "some-backup".to_string(),
2896            }),
2897        };
2898        let restore = make_restore(source, None);
2899        assert!(validate_source(&restore).is_err());
2900    }
2901
2902    #[test]
2903    fn source_validation_rejects_empty_source() {
2904        let source = KanidmRestoreSource {
2905            local: None,
2906            backup_ref: None,
2907        };
2908        let restore = make_restore(source, None);
2909        assert!(validate_source(&restore).is_err());
2910    }
2911
2912    #[test]
2913    fn source_validation_accepts_local_source() {
2914        let source = KanidmRestoreSource {
2915            local: Some(super::KanidmRestoreLocalSource {
2916                file_name: "backup.json".to_string(),
2917            }),
2918            backup_ref: None,
2919        };
2920        let restore = make_restore(source, None);
2921        assert!(validate_source(&restore).is_ok());
2922    }
2923
2924    #[test]
2925    fn source_validation_accepts_backup_ref_source() {
2926        let source = KanidmRestoreSource {
2927            local: None,
2928            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2929                name: "corp-idm-019c7c76".to_string(),
2930            }),
2931        };
2932        let restore = make_restore(source, None);
2933        assert!(validate_source(&restore).is_ok());
2934    }
2935
2936    #[test]
2937    fn source_validation_rejects_empty_backup_ref_name() {
2938        let source = KanidmRestoreSource {
2939            local: None,
2940            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2941                name: "".to_string(),
2942            }),
2943        };
2944        let restore = make_restore(source, None);
2945        assert!(validate_source(&restore).is_err());
2946    }
2947
2948    #[test]
2949    fn is_remote_source_true_for_backup_ref() {
2950        let source = KanidmRestoreSource {
2951            local: None,
2952            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2953                name: "backup-1".to_string(),
2954            }),
2955        };
2956        let restore = make_restore(source, None);
2957        assert!(is_remote_source(&restore));
2958    }
2959
2960    #[test]
2961    fn is_remote_source_false_for_local() {
2962        let source = KanidmRestoreSource {
2963            local: Some(super::KanidmRestoreLocalSource {
2964                file_name: "backup.json".to_string(),
2965            }),
2966            backup_ref: None,
2967        };
2968        let restore = make_restore(source, None);
2969        assert!(!is_remote_source(&restore));
2970    }
2971
2972    #[test]
2973    fn requires_safety_backup_default_true() {
2974        let source = KanidmRestoreSource {
2975            local: None,
2976            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2977                name: "backup-1".to_string(),
2978            }),
2979        };
2980        let restore = make_restore(source, None);
2981        assert!(requires_safety_backup(&restore));
2982    }
2983
2984    #[test]
2985    fn requires_safety_backup_false_when_skip() {
2986        let source = KanidmRestoreSource {
2987            local: None,
2988            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2989                name: "backup-1".to_string(),
2990            }),
2991        };
2992        let safety = SafetyBackupConfig {
2993            repository_ref: Some(SafetyBackupRepositoryRef {
2994                name: "offsite".to_string(),
2995            }),
2996            skip: true,
2997        };
2998        let restore = make_restore(source, Some(safety));
2999        assert!(!requires_safety_backup(&restore));
3000    }
3001
3002    #[test]
3003    fn break_glass_validation_rejects_skip_without_reason() {
3004        let source = KanidmRestoreSource {
3005            local: None,
3006            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3007                name: "backup-1".to_string(),
3008            }),
3009        };
3010        let safety = SafetyBackupConfig {
3011            repository_ref: Some(SafetyBackupRepositoryRef {
3012                name: "offsite".to_string(),
3013            }),
3014            skip: true,
3015        };
3016        let mut restore = make_restore(source, Some(safety));
3017        restore.metadata.annotations = Some(BTreeMap::from([(
3018            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3019            "commander@example.com".to_string(),
3020        )]));
3021        assert!(validate_safety_backup_config(&restore).is_err());
3022    }
3023
3024    #[test]
3025    fn break_glass_validation_rejects_skip_without_approver() {
3026        let source = KanidmRestoreSource {
3027            local: None,
3028            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3029                name: "backup-1".to_string(),
3030            }),
3031        };
3032        let safety = SafetyBackupConfig {
3033            repository_ref: Some(SafetyBackupRepositoryRef {
3034                name: "offsite".to_string(),
3035            }),
3036            skip: true,
3037        };
3038        let mut restore = make_restore(source, Some(safety));
3039        restore.metadata.annotations = Some(BTreeMap::from([(
3040            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3041            "PVC is unreadable".to_string(),
3042        )]));
3043        assert!(validate_safety_backup_config(&restore).is_err());
3044    }
3045
3046    #[test]
3047    fn break_glass_validation_accepts_valid_annotations() {
3048        let source = KanidmRestoreSource {
3049            local: None,
3050            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3051                name: "backup-1".to_string(),
3052            }),
3053        };
3054        let safety = SafetyBackupConfig {
3055            repository_ref: Some(SafetyBackupRepositoryRef {
3056                name: "offsite".to_string(),
3057            }),
3058            skip: true,
3059        };
3060        let mut restore = make_restore(source, Some(safety));
3061        restore.metadata.annotations = Some(BTreeMap::from([
3062            (
3063                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3064                "PVC is unreadable".to_string(),
3065            ),
3066            (
3067                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3068                "commander@example.com".to_string(),
3069            ),
3070        ]));
3071        assert!(validate_safety_backup_config(&restore).is_ok());
3072    }
3073
3074    #[test]
3075    fn remote_source_requires_repository_ref_when_not_skipping() {
3076        let source = KanidmRestoreSource {
3077            local: None,
3078            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3079                name: "backup-1".to_string(),
3080            }),
3081        };
3082        let safety = SafetyBackupConfig {
3083            repository_ref: None,
3084            skip: false,
3085        };
3086        let restore = make_restore(source, Some(safety));
3087        assert!(validate_safety_backup_config(&restore).is_err());
3088    }
3089
3090    #[test]
3091    fn remote_source_accepts_repository_ref() {
3092        let source = KanidmRestoreSource {
3093            local: None,
3094            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3095                name: "backup-1".to_string(),
3096            }),
3097        };
3098        let safety = SafetyBackupConfig {
3099            repository_ref: Some(SafetyBackupRepositoryRef {
3100                name: "offsite".to_string(),
3101            }),
3102            skip: false,
3103        };
3104        let restore = make_restore(source, Some(safety));
3105        assert!(validate_safety_backup_config(&restore).is_ok());
3106    }
3107
3108    #[test]
3109    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3110        let phases = vec![
3111            KanidmRestorePhase::Pending,
3112            KanidmRestorePhase::Validating,
3113            KanidmRestorePhase::Quiescing,
3114            KanidmRestorePhase::SafetyBackup,
3115            KanidmRestorePhase::PreparingSource,
3116            KanidmRestorePhase::RestoringPrimary,
3117            KanidmRestorePhase::Verifying,
3118            KanidmRestorePhase::RebuildingReplicas,
3119            KanidmRestorePhase::Resuming,
3120            KanidmRestorePhase::Completed,
3121        ];
3122        assert_eq!(phases.len(), 10);
3123        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3124        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3125    }
3126
3127    #[test]
3128    fn status_default_has_empty_phase_timestamps() {
3129        let status = KanidmRestoreStatus::default();
3130        assert!(status.phase_timestamps.is_empty());
3131    }
3132
3133    #[test]
3134    fn status_default_has_empty_original_replicas() {
3135        let status = KanidmRestoreStatus::default();
3136        assert!(status.original_replicas.is_empty());
3137    }
3138
3139    #[test]
3140    fn hardened_security_context_drops_all_capabilities() {
3141        let ctx = hardened_security_context();
3142        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3143        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3144        assert_eq!(ctx.run_as_non_root, Some(true));
3145        let caps = ctx.capabilities.unwrap();
3146        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3147        assert_eq!(
3148            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3149            Some("RuntimeDefault")
3150        );
3151    }
3152
3153    #[test]
3154    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3155        let ctx = kanidm_job_security_context();
3156        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3157        assert!(ctx.run_as_non_root.is_none());
3158        assert!(ctx.read_only_root_filesystem.is_none());
3159        let caps = ctx.capabilities.unwrap();
3160        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3161        assert_eq!(
3162            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3163            Some("RuntimeDefault")
3164        );
3165    }
3166
3167    #[test]
3168    fn safety_backup_job_name_is_deterministic() {
3169        let source = KanidmRestoreSource {
3170            local: Some(super::KanidmRestoreLocalSource {
3171                file_name: "backup.json".to_string(),
3172            }),
3173            backup_ref: None,
3174        };
3175        let restore = make_restore(source, None);
3176        let name1 = super::safety_backup_job_name(&restore);
3177        let name2 = super::safety_backup_job_name(&restore);
3178        assert_eq!(name1, name2);
3179        assert_eq!(name1, "test-restore-safety-backup");
3180    }
3181
3182    #[test]
3183    fn source_prep_job_name_is_deterministic() {
3184        let source = KanidmRestoreSource {
3185            local: None,
3186            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3187                name: "backup-1".to_string(),
3188            }),
3189        };
3190        let restore = make_restore(source, None);
3191        let name = super::source_prep_job_name(&restore);
3192        assert_eq!(name, "test-restore-source-prep");
3193    }
3194
3195    #[test]
3196    fn break_glass_condition_is_recorded_in_status() {
3197        let mut status = KanidmRestoreStatus::default();
3198        let condition = super::restore_condition(
3199            &status.conditions,
3200            CONDITION_BREAK_GLASS,
3201            CONDITION_TRUE,
3202            "BreakGlassOverride",
3203            "Safety backup skipped",
3204            Some(1),
3205        );
3206        status.conditions.push(condition);
3207        assert!(status.conditions.iter().any(|c| {
3208            c.type_ == CONDITION_BREAK_GLASS
3209                && c.status == CONDITION_TRUE
3210                && c.reason == "BreakGlassOverride"
3211        }));
3212    }
3213
3214    #[test]
3215    fn replica_count_entry_serialization() {
3216        let entry = ReplicaCountEntry {
3217            group: "primary".to_string(),
3218            replicas: 3,
3219        };
3220        let json = serde_json::to_string(&entry).unwrap();
3221        assert!(json.contains("\"group\":\"primary\""));
3222        assert!(json.contains("\"replicas\":3"));
3223    }
3224
3225    #[test]
3226    fn local_source_does_not_require_repository_ref() {
3227        let source = KanidmRestoreSource {
3228            local: Some(super::KanidmRestoreLocalSource {
3229                file_name: "backup.json".to_string(),
3230            }),
3231            backup_ref: None,
3232        };
3233        let safety = SafetyBackupConfig {
3234            repository_ref: None,
3235            skip: false,
3236        };
3237        let restore = make_restore(source, Some(safety));
3238        assert!(validate_safety_backup_config(&restore).is_ok());
3239    }
3240
3241    #[test]
3242    fn skip_without_break_glass_annotations_is_rejected() {
3243        let source = KanidmRestoreSource {
3244            local: Some(super::KanidmRestoreLocalSource {
3245                file_name: "backup.json".to_string(),
3246            }),
3247            backup_ref: None,
3248        };
3249        let safety = SafetyBackupConfig {
3250            repository_ref: None,
3251            skip: true,
3252        };
3253        let restore = make_restore(source, Some(safety));
3254        assert!(validate_safety_backup_config(&restore).is_err());
3255    }
3256
3257    #[test]
3258    fn has_accepted_condition_true() {
3259        let conditions = vec![Condition {
3260            type_: "Ready".to_string(),
3261            status: "True".to_string(),
3262            reason: "Accepted".to_string(),
3263            message: String::new(),
3264            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3265                k8s_openapi::jiff::Timestamp::now(),
3266            ),
3267            observed_generation: None,
3268        }];
3269        assert!(has_accepted_condition(&conditions));
3270    }
3271
3272    #[test]
3273    fn has_accepted_condition_false_when_not_accepted_reason() {
3274        let conditions = vec![Condition {
3275            type_: "Ready".to_string(),
3276            status: "True".to_string(),
3277            reason: "SomeOtherReason".to_string(),
3278            message: String::new(),
3279            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3280                k8s_openapi::jiff::Timestamp::now(),
3281            ),
3282            observed_generation: None,
3283        }];
3284        assert!(!has_accepted_condition(&conditions));
3285    }
3286
3287    #[test]
3288    fn has_accepted_condition_false_when_status_false() {
3289        let conditions = vec![Condition {
3290            type_: "Ready".to_string(),
3291            status: "False".to_string(),
3292            reason: "Accepted".to_string(),
3293            message: String::new(),
3294            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3295                k8s_openapi::jiff::Timestamp::now(),
3296            ),
3297            observed_generation: None,
3298        }];
3299        assert!(!has_accepted_condition(&conditions));
3300    }
3301
3302    #[test]
3303    fn has_accepted_condition_empty() {
3304        assert!(!has_accepted_condition(&[]));
3305    }
3306
3307    #[test]
3308    fn safety_backup_upload_operation_doc_is_valid() {
3309        let source = KanidmRestoreSource {
3310            local: None,
3311            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3312                name: "backup-1".to_string(),
3313            }),
3314        };
3315        let mut restore = make_restore(source, None);
3316        restore.metadata.namespace = Some("test-ns".to_string());
3317        let target = super::super::crd::Kanidm::default();
3318        let doc_str = super::build_safety_upload_operation_doc(
3319            &restore,
3320            &target,
3321            "test-backup-id",
3322            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3323            "my-bucket",
3324            "prod",
3325            "https://s3.example.com",
3326            "us-east-1",
3327            false,
3328            false,
3329            None,
3330        )
3331        .unwrap();
3332        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3333        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3334        assert_eq!(parsed["kind"], "OperationDocument");
3335        assert_eq!(parsed["operation"], "upload");
3336        assert_eq!(parsed["bucket"], "my-bucket");
3337        assert_eq!(parsed["prefix"], "prod");
3338        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3339        assert_eq!(parsed["region"], "us-east-1");
3340        assert_eq!(parsed["backupId"], "test-backup-id");
3341        assert_eq!(parsed["consistency"], "kanidm-offline");
3342        assert_eq!(parsed["reason"], "restore-safety");
3343        assert_eq!(
3344            parsed["payloadPath"],
3345            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3346        );
3347        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3348    }
3349
3350    #[test]
3351    fn download_operation_doc_uses_real_repository_data() {
3352        let source = KanidmRestoreSource {
3353            local: None,
3354            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3355                name: "backup-1".to_string(),
3356            }),
3357        };
3358        let restore = make_restore(source, None);
3359        let target = super::super::crd::Kanidm::default();
3360        let doc_str = super::build_download_operation_doc(
3361            &restore,
3362            &target,
3363            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3364            "019c7c76-f423-7a12-8f41-2bea7588a303",
3365            "real-bucket",
3366            "real-prefix",
3367            "https://real-endpoint.com",
3368            "eu-west-1",
3369            true,
3370            false,
3371            None,
3372        );
3373        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3374        assert_eq!(parsed["bucket"], "real-bucket");
3375        assert_eq!(parsed["prefix"], "real-prefix");
3376        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3377        assert_eq!(parsed["region"], "eu-west-1");
3378        assert_eq!(parsed["forcePathStyle"], true);
3379        assert_eq!(
3380            parsed["manifestKey"],
3381            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3382        );
3383        assert_eq!(
3384            parsed["expectedBackupId"],
3385            "019c7c76-f423-7a12-8f41-2bea7588a303"
3386        );
3387    }
3388
3389    #[test]
3390    fn safety_backup_ref_is_set_after_verification() {
3391        let status = KanidmRestoreStatus {
3392            phase: KanidmRestorePhase::SafetyBackup,
3393            safety_backup_job_name: Some("test-safety-backup".to_string()),
3394            safety_backup_ref: Some("safety-test-restore".to_string()),
3395            safety_backup_expected_backup_id: Some(
3396                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3397            ),
3398            safety_backup_manifest_key: Some(
3399                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3400            ),
3401            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3402            ..Default::default()
3403        };
3404        assert!(status.safety_backup_ref.is_some());
3405        assert!(
3406            status
3407                .safety_backup_ref
3408                .as_ref()
3409                .unwrap()
3410                .starts_with("safety-")
3411        );
3412        assert!(status.safety_backup_expected_backup_id.is_some());
3413        assert!(status.safety_backup_manifest_key.is_some());
3414        assert!(status.safety_backup_payload_sha256.is_some());
3415    }
3416
3417    #[test]
3418    fn database_mutation_not_started_without_safety_backup_ref() {
3419        let status = KanidmRestoreStatus {
3420            phase: KanidmRestorePhase::PreparingSource,
3421            safety_backup_ref: None,
3422            database_mutation_started: false,
3423            ..Default::default()
3424        };
3425        assert!(!status.database_mutation_started);
3426        assert!(status.safety_backup_ref.is_none());
3427    }
3428
3429    #[test]
3430    fn pre_boundary_failure_does_not_set_mutation_started() {
3431        let status = KanidmRestoreStatus {
3432            phase: KanidmRestorePhase::Failed,
3433            message: Some(
3434                "safety backup job failed; target restored to original state".to_string(),
3435            ),
3436            database_mutation_started: false,
3437            ..Default::default()
3438        };
3439        assert!(!status.database_mutation_started);
3440    }
3441
3442    #[test]
3443    fn shared_volume_constants_are_distinct() {
3444        assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3445        assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3446    }
3447
3448    #[test]
3449    fn validate_backup_compatibility_matching_versions_passes() {
3450        let source = KanidmRestoreSource {
3451            local: None,
3452            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3453                name: "backup-1".to_string(),
3454            }),
3455        };
3456        let restore = make_restore(source, None);
3457        let target = kanidm_with_version("1.10.4");
3458        let backup = kaniop_backup_core::crd::KanidmBackup {
3459            metadata: kube::api::ObjectMeta::default(),
3460            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3461                backup_id: "id".to_string(),
3462                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3463                    name: "k".to_string(),
3464                    uid: "uid".to_string(),
3465                },
3466                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3467                    name: "repo".to_string(),
3468                },
3469                manifest_key: "key".to_string(),
3470            },
3471            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3472                kanidm_version: Some("1.10.4".to_string()),
3473                ..Default::default()
3474            }),
3475        };
3476        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3477    }
3478
3479    #[test]
3480    fn validate_backup_compatibility_version_mismatch_fails() {
3481        let source = KanidmRestoreSource {
3482            local: None,
3483            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3484                name: "backup-1".to_string(),
3485            }),
3486        };
3487        let restore = make_restore(source, None);
3488        let target = kanidm_with_version("1.11.0");
3489        let backup = kaniop_backup_core::crd::KanidmBackup {
3490            metadata: kube::api::ObjectMeta::default(),
3491            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3492                backup_id: "id".to_string(),
3493                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3494                    name: "k".to_string(),
3495                    uid: "uid".to_string(),
3496                },
3497                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3498                    name: "repo".to_string(),
3499                },
3500                manifest_key: "key".to_string(),
3501            },
3502            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3503                kanidm_version: Some("1.10.4".to_string()),
3504                ..Default::default()
3505            }),
3506        };
3507        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3508    }
3509
3510    #[test]
3511    fn validate_backup_compatibility_no_backup_version_passes() {
3512        let source = KanidmRestoreSource {
3513            local: None,
3514            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3515                name: "backup-1".to_string(),
3516            }),
3517        };
3518        let restore = make_restore(source, None);
3519        let target = super::super::crd::Kanidm::default();
3520        let backup = kaniop_backup_core::crd::KanidmBackup {
3521            metadata: kube::api::ObjectMeta::default(),
3522            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3523                backup_id: "id".to_string(),
3524                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3525                    name: "k".to_string(),
3526                    uid: "uid".to_string(),
3527                },
3528                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3529                    name: "repo".to_string(),
3530                },
3531                manifest_key: "key".to_string(),
3532            },
3533            status: None,
3534        };
3535        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3536    }
3537
3538    #[test]
3539    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3540        let source = KanidmRestoreSource {
3541            local: None,
3542            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3543                name: "backup-1".to_string(),
3544            }),
3545        };
3546        let mut restore = make_restore(source, None);
3547        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3548        let target = super::super::crd::Kanidm::default();
3549        let backup = kaniop_backup_core::crd::KanidmBackup {
3550            metadata: kube::api::ObjectMeta::default(),
3551            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3552                backup_id: "id".to_string(),
3553                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3554                    name: "k".to_string(),
3555                    uid: "uid".to_string(),
3556                },
3557                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3558                    name: "repo".to_string(),
3559                },
3560                manifest_key: "key".to_string(),
3561            },
3562            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3563                image_digest: Some("sha256:originaldigest".to_string()),
3564                ..Default::default()
3565            }),
3566        };
3567        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3568    }
3569
3570    #[test]
3571    fn validate_backup_compatibility_image_digest_match_passes() {
3572        let source = KanidmRestoreSource {
3573            local: None,
3574            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3575                name: "backup-1".to_string(),
3576            }),
3577        };
3578        let mut restore = make_restore(source, None);
3579        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3580        let target = super::super::crd::Kanidm::default();
3581        let backup = kaniop_backup_core::crd::KanidmBackup {
3582            metadata: kube::api::ObjectMeta::default(),
3583            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3584                backup_id: "id".to_string(),
3585                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3586                    name: "k".to_string(),
3587                    uid: "uid".to_string(),
3588                },
3589                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3590                    name: "repo".to_string(),
3591                },
3592                manifest_key: "key".to_string(),
3593            },
3594            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3595                image_digest: Some("sha256:abc123".to_string()),
3596                ..Default::default()
3597            }),
3598        };
3599        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3600    }
3601
3602    #[test]
3603    fn safety_backup_id_is_deterministic() {
3604        let source = KanidmRestoreSource {
3605            local: None,
3606            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3607                name: "backup-1".to_string(),
3608            }),
3609        };
3610        let restore = make_restore(source, None);
3611        let id1 = super::compute_safety_backup_id(&restore);
3612        let id2 = super::compute_safety_backup_id(&restore);
3613        assert_eq!(id1, id2);
3614        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3615    }
3616
3617    #[test]
3618    fn safety_backup_id_differs_for_different_restores() {
3619        let source1 = KanidmRestoreSource {
3620            local: None,
3621            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3622                name: "backup-1".to_string(),
3623            }),
3624        };
3625        let restore1 = make_restore(source1, None);
3626        let mut restore2 = make_restore(
3627            KanidmRestoreSource {
3628                local: None,
3629                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3630                    name: "backup-2".to_string(),
3631                }),
3632            },
3633            None,
3634        );
3635        restore2.metadata.name = Some("different-restore".to_string());
3636        let id1 = super::compute_safety_backup_id(&restore1);
3637        let id2 = super::compute_safety_backup_id(&restore2);
3638        assert_ne!(id1, id2);
3639    }
3640
3641    #[test]
3642    fn safety_manifest_key_is_deterministic() {
3643        let source = KanidmRestoreSource {
3644            local: None,
3645            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3646                name: "backup-1".to_string(),
3647            }),
3648        };
3649        let restore = make_restore(source, None);
3650        let key1 = super::compute_safety_manifest_key(&restore);
3651        let key2 = super::compute_safety_manifest_key(&restore);
3652        assert_eq!(key1, key2);
3653        assert!(key1.contains("manifest.json"));
3654        assert!(key1.starts_with("v1/tenants/"));
3655    }
3656
3657    #[test]
3658    fn safety_backup_id_survives_restart() {
3659        let source = KanidmRestoreSource {
3660            local: None,
3661            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3662                name: "backup-1".to_string(),
3663            }),
3664        };
3665        let mut restore = make_restore(source, None);
3666        let original_id = super::compute_safety_backup_id(&restore);
3667        restore.status = Some(KanidmRestoreStatus {
3668            phase: KanidmRestorePhase::SafetyBackup,
3669            safety_backup_expected_backup_id: Some(original_id.clone()),
3670            ..Default::default()
3671        });
3672        let recomputed_id = super::compute_safety_backup_id(&restore);
3673        assert_eq!(original_id, recomputed_id);
3674    }
3675
3676    #[test]
3677    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3678        let source = KanidmRestoreSource {
3679            local: None,
3680            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3681                name: "backup-1".to_string(),
3682            }),
3683        };
3684        let restore = make_restore(source, None);
3685        assert!(super::requires_safety_backup(&restore));
3686        let status_without_ref = KanidmRestoreStatus {
3687            phase: KanidmRestorePhase::RestoringPrimary,
3688            safety_backup_ref: None,
3689            database_mutation_started: false,
3690            ..Default::default()
3691        };
3692        assert!(status_without_ref.safety_backup_ref.is_none());
3693        assert!(!status_without_ref.database_mutation_started);
3694    }
3695
3696    #[test]
3697    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3698        let status_with_ref = KanidmRestoreStatus {
3699            phase: KanidmRestorePhase::RestoringPrimary,
3700            safety_backup_ref: Some("safety-test-restore".to_string()),
3701            safety_backup_manifest_key: Some(
3702                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3703            ),
3704            safety_backup_payload_sha256: Some("abc123".to_string()),
3705            database_mutation_started: false,
3706            ..Default::default()
3707        };
3708        assert!(status_with_ref.safety_backup_ref.is_some());
3709        assert!(!status_with_ref.database_mutation_started);
3710    }
3711
3712    #[test]
3713    fn result_document_verification_rejects_missing_backup_id() {
3714        let result_json = r#"{
3715            "apiVersion": "backup.kaniop.rs/v1alpha1",
3716            "kind": "ResultDocument",
3717            "operation": "upload",
3718            "success": true,
3719            "exitCode": "success",
3720            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3721            "payloadSha256": "abc123"
3722        }"#;
3723        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3724        assert!(result.backup_id.is_none());
3725    }
3726
3727    #[test]
3728    fn result_document_verification_rejects_missing_manifest_key() {
3729        let result_json = r#"{
3730            "apiVersion": "backup.kaniop.rs/v1alpha1",
3731            "kind": "ResultDocument",
3732            "operation": "upload",
3733            "success": true,
3734            "exitCode": "success",
3735            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3736            "payloadSha256": "abc123"
3737        }"#;
3738        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3739        assert!(result.manifest_key.is_none());
3740    }
3741
3742    #[test]
3743    fn result_document_verification_rejects_missing_payload_sha256() {
3744        let result_json = r#"{
3745            "apiVersion": "backup.kaniop.rs/v1alpha1",
3746            "kind": "ResultDocument",
3747            "operation": "upload",
3748            "success": true,
3749            "exitCode": "success",
3750            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3751            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3752        }"#;
3753        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3754        assert!(result.payload_sha256.is_none());
3755    }
3756
3757    #[test]
3758    fn result_document_verification_rejects_failure_exit_code() {
3759        let result_json = r#"{
3760            "apiVersion": "backup.kaniop.rs/v1alpha1",
3761            "kind": "ResultDocument",
3762            "operation": "upload",
3763            "success": false,
3764            "exitCode": "integrity",
3765            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3766            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3767            "payloadSha256": "abc123"
3768        }"#;
3769        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3770        assert!(!result.success);
3771        assert_eq!(
3772            result.exit_code,
3773            kaniop_backup_core::result::ExitCode::Integrity
3774        );
3775    }
3776
3777    #[test]
3778    fn result_document_verification_rejects_wrong_operation() {
3779        let result_json = r#"{
3780            "apiVersion": "backup.kaniop.rs/v1alpha1",
3781            "kind": "ResultDocument",
3782            "operation": "download",
3783            "success": true,
3784            "exitCode": "success",
3785            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3786            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3787            "payloadSha256": "abc123"
3788        }"#;
3789        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3790        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3791    }
3792
3793    #[test]
3794    fn result_document_verification_accepts_valid_document() {
3795        let result_json = r#"{
3796            "apiVersion": "backup.kaniop.rs/v1alpha1",
3797            "kind": "ResultDocument",
3798            "operation": "upload",
3799            "success": true,
3800            "exitCode": "success",
3801            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3802            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3803            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3804        }"#;
3805        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3806        assert!(result.success);
3807        assert_eq!(
3808            result.exit_code,
3809            kaniop_backup_core::result::ExitCode::Success
3810        );
3811        assert_eq!(result.operation, "upload");
3812        assert!(result.backup_id.is_some());
3813        assert!(result.manifest_key.is_some());
3814        assert!(result.payload_sha256.is_some());
3815    }
3816
3817    #[test]
3818    fn termination_message_path_constant_is_valid() {
3819        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3820        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3821    }
3822
3823    #[test]
3824    fn status_tracks_safety_backup_evidence() {
3825        let status = KanidmRestoreStatus {
3826            phase: KanidmRestorePhase::PreparingSource,
3827            safety_backup_ref: Some("safety-test-restore".to_string()),
3828            safety_backup_expected_backup_id: Some(
3829                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3830            ),
3831            safety_backup_manifest_key: Some(
3832                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3833            ),
3834            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3835            ..Default::default()
3836        };
3837        assert!(status.safety_backup_ref.is_some());
3838        assert!(status.safety_backup_expected_backup_id.is_some());
3839        assert!(status.safety_backup_manifest_key.is_some());
3840        assert!(status.safety_backup_payload_sha256.is_some());
3841    }
3842
3843    #[test]
3844    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3845        let invalid_json = "not valid json";
3846        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3847        assert!(result.is_err());
3848    }
3849
3850    #[test]
3851    fn fail_closed_semantics_preserved_on_oversized_result() {
3852        let oversized_json = format!(
3853            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3854            kaniop_backup_core::result::RESULT_DOC_VERSION,
3855            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3856        );
3857        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3858        assert!(result.is_err());
3859    }
3860
3861    #[test]
3862    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3863        let source = KanidmRestoreSource {
3864            local: None,
3865            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3866                name: "backup-1".to_string(),
3867            }),
3868        };
3869        let mut restore = make_restore(source, None);
3870        restore.metadata.namespace = Some("test-ns".to_string());
3871        let target = super::super::crd::Kanidm::default();
3872        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3873        let doc_str = super::build_safety_upload_operation_doc(
3874            &restore,
3875            &target,
3876            "test-backup-id",
3877            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3878            "my-bucket",
3879            "prod",
3880            "https://s3.example.com",
3881            "us-east-1",
3882            false,
3883            false,
3884            Some(&ca_path),
3885        )
3886        .unwrap();
3887        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3888        assert_eq!(parsed["caBundlePath"], ca_path);
3889    }
3890
3891    #[test]
3892    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3893        let source = KanidmRestoreSource {
3894            local: None,
3895            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3896                name: "backup-1".to_string(),
3897            }),
3898        };
3899        let mut restore = make_restore(source, None);
3900        restore.metadata.namespace = Some("test-ns".to_string());
3901        let target = super::super::crd::Kanidm::default();
3902        let doc_str = super::build_safety_upload_operation_doc(
3903            &restore,
3904            &target,
3905            "test-backup-id",
3906            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3907            "my-bucket",
3908            "prod",
3909            "https://s3.example.com",
3910            "us-east-1",
3911            false,
3912            false,
3913            None,
3914        )
3915        .unwrap();
3916        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3917        assert!(parsed["caBundlePath"].is_null());
3918    }
3919
3920    #[test]
3921    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3922        let source = KanidmRestoreSource {
3923            local: None,
3924            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3925                name: "backup-1".to_string(),
3926            }),
3927        };
3928        let restore = make_restore(source, None);
3929        let target = super::super::crd::Kanidm::default();
3930        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3931        let doc_str = super::build_download_operation_doc(
3932            &restore,
3933            &target,
3934            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3935            "019c7c76-f423-7a12-8f41-2bea7588a303",
3936            "real-bucket",
3937            "real-prefix",
3938            "https://real-endpoint.com",
3939            "eu-west-1",
3940            true,
3941            false,
3942            Some(&ca_path),
3943        );
3944        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3945        assert_eq!(parsed["caBundlePath"], ca_path);
3946    }
3947
3948    #[test]
3949    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3950        let source = KanidmRestoreSource {
3951            local: None,
3952            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3953                name: "backup-1".to_string(),
3954            }),
3955        };
3956        let restore = make_restore(source, None);
3957        let target = super::super::crd::Kanidm::default();
3958        let doc_str = super::build_download_operation_doc(
3959            &restore,
3960            &target,
3961            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3962            "019c7c76-f423-7a12-8f41-2bea7588a303",
3963            "real-bucket",
3964            "real-prefix",
3965            "https://real-endpoint.com",
3966            "eu-west-1",
3967            true,
3968            false,
3969            None,
3970        );
3971        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3972        assert!(parsed["caBundlePath"].is_null());
3973    }
3974
3975    #[test]
3976    fn source_prep_result_operation_is_download() {
3977        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3978        assert_ne!(
3979            super::SOURCE_PREP_RESULT_OPERATION,
3980            super::SAFETY_BACKUP_RESULT_OPERATION
3981        );
3982    }
3983
3984    #[test]
3985    fn download_result_document_verification_accepts_valid_document() {
3986        let result_json = r#"{
3987            "apiVersion": "backup.kaniop.rs/v1alpha1",
3988            "kind": "ResultDocument",
3989            "operation": "download",
3990            "success": true,
3991            "exitCode": "success",
3992            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3993            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3994            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3995        }"#;
3996        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3997        assert!(result.success);
3998        assert_eq!(
3999            result.exit_code,
4000            kaniop_backup_core::result::ExitCode::Success
4001        );
4002        assert_eq!(result.operation, "download");
4003        assert_eq!(
4004            result.backup_id.as_deref(),
4005            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4006        );
4007        assert!(result.manifest_key.is_some());
4008        assert!(result.payload_sha256.is_some());
4009    }
4010
4011    #[test]
4012    fn download_result_document_rejects_wrong_operation() {
4013        let result_json = r#"{
4014            "apiVersion": "backup.kaniop.rs/v1alpha1",
4015            "kind": "ResultDocument",
4016            "operation": "upload",
4017            "success": true,
4018            "exitCode": "success",
4019            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4020            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4021            "payloadSha256": "abc123"
4022        }"#;
4023        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4024        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4025    }
4026
4027    #[test]
4028    fn download_result_document_rejects_failure_success_field() {
4029        let result_json = r#"{
4030            "apiVersion": "backup.kaniop.rs/v1alpha1",
4031            "kind": "ResultDocument",
4032            "operation": "download",
4033            "success": false,
4034            "exitCode": "retryable",
4035            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4036            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4037            "payloadSha256": "abc123"
4038        }"#;
4039        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4040        assert!(!result.success);
4041        assert_ne!(
4042            result.exit_code,
4043            kaniop_backup_core::result::ExitCode::Success
4044        );
4045    }
4046
4047    #[test]
4048    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4049        let status = KanidmRestoreStatus {
4050            phase: KanidmRestorePhase::PreparingSource,
4051            database_mutation_started: false,
4052            ..Default::default()
4053        };
4054        assert!(!status.database_mutation_started);
4055        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4056    }
4057}