Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const STAGING_VOLUME: &str = "restore-staging";
54const SHARED_VOLUME: &str = "safety-backup-shared";
55const DATA_PATH: &str = "/data";
56const TLS_VOLUME: &str = "kanidm-certs";
57const TLS_PATH: &str = "/etc/kanidm/tls";
58const BACKUP_PATH: &str = "/data";
59const STAGING_PATH: &str = "/staging";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78    feature = "schemars",
79    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82    category = "kaniop",
83    group = "kaniop.rs",
84    version = "v1beta1",
85    kind = "KanidmRestore",
86    plural = "kanidmrestores",
87    singular = "kanidmrestore",
88    shortname = "idmrestore",
89    namespaced,
90    status = "KanidmRestoreStatus",
91    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97    pub target_ref: KanidmRestoreTargetRef,
98    pub source: KanidmRestoreSource,
99    pub restore_image: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108    pub name: String,
109    pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115    feature = "schemars",
116    schemars(extend("x-kubernetes-validations" = [
117        {
118            "message": "exactly one of local or backupRef must be set",
119            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120        }
121    ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125    /// Local file source. Mutually exclusive with `backupRef`.
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub local: Option<KanidmRestoreLocalSource>,
128    /// Remote cataloged backup reference. Mutually exclusive with `local`.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
138    pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
146    pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub repository_ref: Option<SafetyBackupRepositoryRef>,
156    /// Skip the mandatory safety backup. Requires break-glass annotations.
157    #[serde(default)]
158    pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165    pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171    #[default]
172    Pending,
173    Validating,
174    Quiescing,
175    SafetyBackup,
176    PreparingSource,
177    RestoringPrimary,
178    Verifying,
179    RebuildingReplicas,
180    Resuming,
181    Completed,
182    Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_generation: Option<i64>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_target_uid: Option<String>,
193    #[serde(default)]
194    pub phase: KanidmRestorePhase,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub restore_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub verify_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub safety_backup_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub source_prep_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_ref: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_expected_backup_id: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_manifest_key: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_payload_sha256: Option<String>,
211    #[serde(default)]
212    pub replicas_cleared: bool,
213    #[serde(default)]
214    pub certificates_cleared: bool,
215    #[serde(default)]
216    pub database_mutation_started: bool,
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub original_replicas: Vec<ReplicaCountEntry>,
219    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220    pub phase_timestamps: std::collections::BTreeMap<String, String>,
221    #[serde(default, skip_serializing_if = "Vec::is_empty")]
222    #[cfg_attr(
223        feature = "schemars",
224        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225    )]
226    pub conditions: Vec<Condition>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235    pub group: String,
236    pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241    attempts: Counter<u64>,
242    outcomes: Counter<u64>,
243    duration_seconds: Histogram<f64>,
244    break_glass_total: Counter<u64>,
245    safety_backup_duration_seconds: Histogram<f64>,
246}
247
248impl RestoreMetrics {
249    fn new() -> Self {
250        let meter = global::meter("kaniop");
251        Self {
252            attempts: meter
253                .u64_counter("kaniop_restore_attempts_total")
254                .with_description("Number of Kanidm restore attempts started")
255                .build(),
256            outcomes: meter
257                .u64_counter("kaniop_restore_outcomes_total")
258                .with_description("Number of terminal Kanidm restore outcomes")
259                .build(),
260            duration_seconds: meter
261                .f64_histogram("kaniop_restore_duration_seconds")
262                .with_description("Kanidm restore duration from object creation to terminal phase")
263                .with_unit("s")
264                .build(),
265            break_glass_total: meter
266                .u64_counter("kaniop_restore_break_glass_total")
267                .with_description("Number of break-glass safety backup overrides")
268                .build(),
269            safety_backup_duration_seconds: meter
270                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
271                .with_description("Duration of pre-restore safety backup creation")
272                .with_unit("s")
273                .build(),
274        }
275    }
276}
277
278#[derive(Clone)]
279struct RestoreContext {
280    client: Client,
281    recorder: Recorder,
282    metrics: RestoreMetrics,
283}
284
285pub async fn run(client: Client) {
286    let api = Api::<KanidmRestore>::all(client.clone());
287    let recorder = Recorder::new(
288        client.clone(),
289        Reporter {
290            controller: CONTROLLER_ID.into(),
291            instance: None,
292        },
293    );
294    let ctx = Arc::new(RestoreContext {
295        client,
296        recorder,
297        metrics: RestoreMetrics::new(),
298    });
299    info!("starting {CONTROLLER_ID} controller");
300    Controller::new(api, watcher::Config::default().any_semantic())
301        .shutdown_on_signal()
302        .run(reconcile_restore, error_policy, ctx)
303        .for_each(|result| async move {
304            if let Err(error) = result {
305                error!(%error, "KanidmRestore reconciliation failed");
306            }
307        })
308        .await;
309}
310
311fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
312    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
313    Action::requeue(Duration::from_secs(5))
314}
315
316async fn reconcile_restore(
317    restore: Arc<KanidmRestore>,
318    ctx: Arc<RestoreContext>,
319) -> Result<Action> {
320    let namespace = restore
321        .namespace()
322        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
323    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
324    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
325        let ctx = ctx.clone();
326        async move {
327            match event {
328                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
329                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
330            }
331        }
332    })
333    .await
334    .map_err(|error| {
335        Error::FinalizerError(
336            "failed on KanidmRestore finalizer".to_string(),
337            Box::new(error),
338        )
339    })
340}
341
342async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
343    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
344    match phase {
345        KanidmRestorePhase::Pending => {
346            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
347            Ok(Action::requeue(REQUEUE))
348        }
349        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
350            Ok(target) => {
351                mark_restoring(&restore, &target, &ctx).await?;
352                let mut status = restore.status.clone().unwrap_or_default();
353                status.observed_target_uid = target.uid();
354                status.original_replicas = capture_original_replicas(&target);
355                status.phase = KanidmRestorePhase::Quiescing;
356                status.message = None;
357                patch_status(&restore, &ctx, status).await?;
358                Ok(Action::requeue(REQUEUE))
359            }
360            Err(error) => {
361                set_phase(
362                    &restore,
363                    &ctx,
364                    KanidmRestorePhase::Failed,
365                    Some(error.to_string()),
366                )
367                .await?;
368                Ok(Action::requeue(Duration::from_secs(300)))
369            }
370        },
371        KanidmRestorePhase::Quiescing => {
372            let target = get_target(&restore, &ctx).await?;
373            scale_all(&target, &ctx, 0).await?;
374            if target_pods_stopped(&target, &ctx).await?
375                && target_volumes_detached(&target, &ctx).await?
376            {
377                let next_phase = if requires_safety_backup(&restore) {
378                    KanidmRestorePhase::SafetyBackup
379                } else {
380                    record_break_glass(&restore, &ctx).await?;
381                    KanidmRestorePhase::PreparingSource
382                };
383                if next_phase == KanidmRestorePhase::PreparingSource {
384                    let refreshed = Api::<KanidmRestore>::namespaced(
385                        ctx.client.clone(),
386                        restore.namespace().as_deref().unwrap_or_default(),
387                    )
388                    .get(&restore.name_any())
389                    .await
390                    .map_err(|error| {
391                        Error::kube_error(
392                            "get",
393                            "KanidmRestore",
394                            restore.namespace().as_deref().unwrap_or_default(),
395                            restore.name_any(),
396                            error,
397                        )
398                    })?;
399                    set_phase(&refreshed, &ctx, next_phase, None).await?;
400                } else {
401                    set_phase(&restore, &ctx, next_phase, None).await?;
402                }
403            }
404            Ok(Action::requeue(REQUEUE))
405        }
406        KanidmRestorePhase::SafetyBackup => {
407            let target = get_target(&restore, &ctx).await?;
408            let name = safety_backup_job_name(&restore);
409            let expected_backup_id = compute_safety_backup_id(&restore);
410            let expected_manifest_key = compute_safety_manifest_key(&restore);
411            ensure_safety_backup_job(
412                &restore,
413                &target,
414                &ctx,
415                &name,
416                &expected_backup_id,
417                &expected_manifest_key,
418            )
419            .await?;
420            let mut status = restore.status.clone().unwrap_or_default();
421            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
422            match job_state(&restore, &ctx, &name).await? {
423                JobState::Complete => {
424                    let result =
425                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
426                    match result {
427                        Ok(verified) => {
428                            status.safety_backup_job_name = Some(name);
429                            status.safety_backup_ref =
430                                Some(format!("safety-{}", restore.name_any()));
431                            status.safety_backup_manifest_key = Some(verified.manifest_key);
432                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
433                            record_safety_backup_duration(&restore, &ctx).await;
434                            status.phase = KanidmRestorePhase::PreparingSource;
435                            status.message = None;
436                            patch_status(&restore, &ctx, status).await?;
437                        }
438                        Err(error) => {
439                            resume_before_mutation(&restore, &ctx).await?;
440                            set_phase(
441                                &restore,
442                                &ctx,
443                                KanidmRestorePhase::Failed,
444                                Some(format!("safety backup result verification failed: {error}")),
445                            )
446                            .await?;
447                        }
448                    }
449                }
450                JobState::Failed => {
451                    resume_before_mutation(&restore, &ctx).await?;
452                    set_phase(
453                        &restore,
454                        &ctx,
455                        KanidmRestorePhase::Failed,
456                        Some(
457                            "safety backup job failed; target restored to original state"
458                                .to_string(),
459                        ),
460                    )
461                    .await?;
462                }
463                JobState::Running => {
464                    patch_status(&restore, &ctx, status).await?;
465                }
466            }
467            Ok(Action::requeue(REQUEUE))
468        }
469        KanidmRestorePhase::PreparingSource => {
470            if is_remote_source(&restore) {
471                let target = get_target(&restore, &ctx).await?;
472                let name = source_prep_job_name(&restore);
473                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
474                match job_state(&restore, &ctx, &name).await? {
475                    JobState::Complete => {
476                        let backup_id = restore
477                            .spec
478                            .source
479                            .backup_ref
480                            .as_ref()
481                            .map(|b| b.name.clone())
482                            .unwrap_or_default();
483                        let backup = {
484                            let ns = restore.namespace().unwrap();
485                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
486                                .get(&backup_id)
487                                .await
488                                .map_err(|e| {
489                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
490                                })?
491                        };
492                        let expected_backup_id = &backup.spec.backup_id;
493                        let result =
494                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
495                                .await;
496                        match result {
497                            Ok(_verified) => {
498                                let mut status = restore.status.clone().unwrap_or_default();
499                                status.source_prep_job_name = Some(name);
500                                status.phase = KanidmRestorePhase::RestoringPrimary;
501                                status.message = None;
502                                patch_status(&restore, &ctx, status).await?;
503                            }
504                            Err(error) => {
505                                resume_before_mutation(&restore, &ctx).await?;
506                                set_phase(
507                                    &restore,
508                                    &ctx,
509                                    KanidmRestorePhase::Failed,
510                                    Some(format!(
511                                        "source preparation result verification failed: {error}"
512                                    )),
513                                )
514                                .await?;
515                            }
516                        }
517                    }
518                    JobState::Failed => {
519                        resume_before_mutation(&restore, &ctx).await?;
520                        set_phase(
521                            &restore,
522                            &ctx,
523                            KanidmRestorePhase::Failed,
524                            Some(
525                                "source preparation job failed; target restored to original state"
526                                    .to_string(),
527                            ),
528                        )
529                        .await?;
530                    }
531                    JobState::Running => {}
532                }
533            } else {
534                let target = get_target(&restore, &ctx).await?;
535                let name = source_check_job_name(&restore);
536                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
537                match job_state(&restore, &ctx, &name).await? {
538                    JobState::Complete => {
539                        let mut status = restore.status.clone().unwrap_or_default();
540                        status.source_prep_job_name = Some(name);
541                        status.phase = KanidmRestorePhase::RestoringPrimary;
542                        status.message = None;
543                        patch_status(&restore, &ctx, status).await?;
544                    }
545                    JobState::Failed => {
546                        resume_before_mutation(&restore, &ctx).await?;
547                        set_phase(
548                            &restore,
549                            &ctx,
550                            KanidmRestorePhase::Failed,
551                            Some(
552                                "local backup file check failed; target restored to original state"
553                                    .to_string(),
554                            ),
555                        )
556                        .await?;
557                    }
558                    JobState::Running => {}
559                }
560            }
561            Ok(Action::requeue(REQUEUE))
562        }
563        KanidmRestorePhase::RestoringPrimary => {
564            let target = get_target(&restore, &ctx).await?;
565            let status = restore.status.as_ref().cloned().unwrap_or_default();
566            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
567                return Err(Error::MissingData(
568                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
569                ));
570            }
571            if !status.database_mutation_started {
572                let mut status = status;
573                status.database_mutation_started = true;
574                status.message = Some(
575                    "database mutation boundary persisted; starting primary restore".to_string(),
576                );
577                patch_status(&restore, &ctx, status).await?;
578                return Ok(Action::requeue(REQUEUE));
579            }
580            let name = restore_job_name(&restore);
581            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
582            match job_state(&restore, &ctx, &name).await? {
583                JobState::Complete => {
584                    let mut status = restore.status.clone().unwrap_or_default();
585                    status.restore_job_name = Some(name);
586                    status.phase = KanidmRestorePhase::Verifying;
587                    status.message = None;
588                    patch_status(&restore, &ctx, status).await?;
589                }
590                JobState::Failed => {
591                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
592                }
593                JobState::Running => {}
594            }
595            Ok(Action::requeue(REQUEUE))
596        }
597        KanidmRestorePhase::Verifying => {
598            let target = get_target(&restore, &ctx).await?;
599            let name = verify_job_name(&restore);
600            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
601            match job_state(&restore, &ctx, &name).await? {
602                JobState::Complete => {
603                    let mut status = restore.status.clone().unwrap_or_default();
604                    status.verify_job_name = Some(name);
605                    status.phase = KanidmRestorePhase::RebuildingReplicas;
606                    status.message = None;
607                    patch_status(&restore, &ctx, status).await?;
608                }
609                JobState::Failed => {
610                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
611                }
612                JobState::Running => {}
613            }
614            Ok(Action::requeue(REQUEUE))
615        }
616        KanidmRestorePhase::RebuildingReplicas => {
617            let target = get_target(&restore, &ctx).await?;
618            let mut status = restore.status.clone().unwrap_or_default();
619            if !status.replicas_cleared {
620                if delete_secondary_pvcs(&target, &ctx).await? {
621                    status.replicas_cleared = true;
622                    status.message = Some("secondary database state cleared".to_string());
623                    patch_status(&restore, &ctx, status).await?;
624                }
625                return Ok(Action::requeue(REQUEUE));
626            }
627
628            scale_primary(&target, &ctx, 1).await?;
629            if !primary_ready(&target, &ctx).await? {
630                return Ok(Action::requeue(REQUEUE));
631            }
632            if !status.certificates_cleared {
633                delete_replica_cert_secrets(&target, &ctx).await?;
634                delete_admin_secret(&target, &ctx).await?;
635                status.certificates_cleared = true;
636                status.message = Some(
637                    "replica certificates and admin secret cleared for regeneration".to_string(),
638                );
639                patch_status(&restore, &ctx, status).await?;
640                return Ok(Action::requeue(REQUEUE));
641            }
642            status.phase = KanidmRestorePhase::Resuming;
643            status.message = None;
644            patch_status(&restore, &ctx, status).await?;
645            Ok(Action::requeue(REQUEUE))
646        }
647        KanidmRestorePhase::Resuming => {
648            let target = get_target(&restore, &ctx).await?;
649            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
650            clear_restoring(&restore, &target, &ctx).await?;
651            Ok(Action::requeue(Duration::from_secs(3600)))
652        }
653        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
654            Ok(Action::requeue(Duration::from_secs(3600)))
655        }
656    }
657}
658
659async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
660    let status = restore.status.clone().unwrap_or_default();
661    if status.database_mutation_started
662        && status.phase != KanidmRestorePhase::Completed
663        && status.phase != KanidmRestorePhase::Failed
664    {
665        reconcile_apply(restore.clone(), ctx.clone()).await?;
666        return Ok(Action::requeue(REQUEUE));
667    }
668    if let Ok(target) = get_target(&restore, &ctx).await {
669        let owns_maintenance =
670            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
671        if owns_maintenance && !status.database_mutation_started {
672            scale_desired(&target, &ctx).await?;
673        }
674        clear_restoring(&restore, &target, &ctx).await?;
675    }
676    Ok(Action::await_change())
677}
678
679async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
680    let target = get_target(restore, ctx).await?;
681    let actual_uid = target
682        .uid()
683        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
684    if actual_uid != restore.spec.target_ref.uid {
685        return Err(Error::MissingData(format!(
686            "target UID mismatch: expected {}, got {}",
687            restore.spec.target_ref.uid, actual_uid
688        )));
689    }
690    validate_source(restore)?;
691    if is_remote_source(restore) {
692        validate_backup_ref(restore, &target, ctx).await?;
693    }
694    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
695    {
696        return Err(Error::MissingData(format!(
697            "restoreImage must be the target's pinned Kanidm image (target image is {})",
698            target.spec.image
699        )));
700    }
701    let storage = target
702        .spec
703        .storage
704        .as_ref()
705        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
706    if storage.empty_dir.is_some()
707        || storage.ephemeral.is_some()
708        || storage.volume_claim_template.is_none()
709    {
710        return Err(Error::MissingData(
711            "restore requires PVC-backed Kanidm storage".to_string(),
712        ));
713    }
714    let primaries = target
715        .spec
716        .replica_groups
717        .iter()
718        .filter(|rg| rg.primary_node)
719        .count();
720    if primaries != 1 {
721        return Err(Error::MissingData(
722            "backup/restore requires exactly one primary replica group".to_string(),
723        ));
724    }
725    let ns = restore.namespace().unwrap();
726    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
727        .list(&ListParams::default())
728        .await
729        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
730    if restores.items.iter().any(|other| {
731        other.name_any() != restore.name_any()
732            && other.spec.target_ref.name == restore.spec.target_ref.name
733            && !matches!(
734                other.status.as_ref().map(|s| s.phase),
735                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
736            )
737    }) {
738        return Err(Error::MissingData(
739            "another active restore targets this Kanidm".to_string(),
740        ));
741    }
742    validate_safety_backup_config(restore)?;
743    Ok(target)
744}
745
746fn safe_basename(name: &str) -> bool {
747    !name.is_empty()
748        && name != "."
749        && name != ".."
750        && !name.contains('/')
751        && !name.contains('\\')
752        && !name.contains('\0')
753}
754
755fn mutable_image(image: &str) -> bool {
756    image == "kanidm/server:latest"
757        || image.ends_with(":latest")
758        || (!image.contains('@')
759            && !image
760                .rsplit('/')
761                .next()
762                .is_some_and(|part| part.contains(':')))
763}
764
765async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
766    let ns = restore
767        .namespace()
768        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
769    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
770        .get(&restore.spec.target_ref.name)
771        .await
772        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
773}
774
775async fn mark_restoring(
776    restore: &KanidmRestore,
777    target: &Kanidm,
778    ctx: &RestoreContext,
779) -> Result<()> {
780    let ns = target.namespace().unwrap();
781    let uid = restore
782        .uid()
783        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
784    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
785        .patch(
786            &target.name_any(),
787            &PatchParams::default(),
788            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
789        )
790        .await
791        .map(|_| ())
792        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
793}
794
795async fn clear_restoring(
796    restore: &KanidmRestore,
797    target: &Kanidm,
798    ctx: &RestoreContext,
799) -> Result<()> {
800    let ns = target.namespace().unwrap();
801    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
802        return Ok(());
803    }
804    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
805        .patch(
806            &target.name_any(),
807            &PatchParams::default(),
808            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
809        )
810        .await
811        .map(|_| ())
812        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
813}
814
815async fn patch_status(
816    restore: &KanidmRestore,
817    ctx: &RestoreContext,
818    mut status: KanidmRestoreStatus,
819) -> Result<()> {
820    let ns = restore.namespace().unwrap();
821    let previous_phase = restore.status.as_ref().map(|current| current.phase);
822    let next_phase = status.phase;
823    status.observed_generation = restore.metadata.generation;
824    update_restore_conditions(&mut status, restore.metadata.generation);
825
826    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
827        .patch_status(
828            &restore.name_any(),
829            &PatchParams::default(),
830            &Patch::Merge(json!({"status": &status})),
831        )
832        .await
833        .map_err(|e| {
834            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
835        })?;
836
837    if previous_phase != Some(next_phase) {
838        record_restore_transition(
839            restore,
840            ctx,
841            previous_phase,
842            next_phase,
843            status.message.as_deref(),
844        )
845        .await;
846    }
847    Ok(())
848}
849
850fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
851    let previous = status.conditions.clone();
852    let phase = status.phase;
853    let phase_reason = format!("{phase:?}");
854    let message = status
855        .message
856        .clone()
857        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
858    let terminal_success = phase == KanidmRestorePhase::Completed;
859    let terminal_failure = phase == KanidmRestorePhase::Failed;
860    let progressing = !terminal_success && !terminal_failure;
861
862    let break_glass = previous
863        .iter()
864        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
865        .cloned();
866    status.conditions = vec![
867        restore_condition(
868            &previous,
869            CONDITION_PROGRESSING,
870            if progressing {
871                CONDITION_TRUE
872            } else {
873                CONDITION_FALSE
874            },
875            &phase_reason,
876            &message,
877            generation,
878        ),
879        restore_condition(
880            &previous,
881            CONDITION_READY,
882            if terminal_success {
883                CONDITION_TRUE
884            } else {
885                CONDITION_FALSE
886            },
887            if terminal_success {
888                "RestoreCompleted"
889            } else {
890                "RestoreNotCompleted"
891            },
892            if terminal_success {
893                "Kanidm restore completed successfully."
894            } else {
895                "Kanidm restore has not completed successfully."
896            },
897            generation,
898        ),
899        restore_condition(
900            &previous,
901            CONDITION_FAILED,
902            if terminal_failure {
903                CONDITION_TRUE
904            } else {
905                CONDITION_FALSE
906            },
907            if terminal_failure {
908                "RestoreFailed"
909            } else {
910                "NoRestoreFailure"
911            },
912            if terminal_failure {
913                status
914                    .message
915                    .as_deref()
916                    .unwrap_or("Kanidm restore failed.")
917            } else {
918                "No terminal restore failure has been recorded."
919            },
920            generation,
921        ),
922    ];
923    if let Some(condition) = break_glass {
924        status.conditions.push(condition);
925    }
926}
927
928fn restore_condition(
929    previous: &[Condition],
930    condition_type: &str,
931    condition_status: &str,
932    reason: &str,
933    message: &str,
934    generation: Option<i64>,
935) -> Condition {
936    let last_transition_time = previous
937        .iter()
938        .find(|condition| {
939            condition.type_ == condition_type
940                && condition.status == condition_status
941                && condition.reason == reason
942        })
943        .map(|condition| condition.last_transition_time.clone())
944        .unwrap_or_else(|| Time(Timestamp::now()));
945    Condition {
946        type_: condition_type.to_string(),
947        status: condition_status.to_string(),
948        reason: reason.to_string(),
949        message: message.to_string(),
950        last_transition_time,
951        observed_generation: generation,
952    }
953}
954
955async fn record_restore_transition(
956    restore: &KanidmRestore,
957    ctx: &RestoreContext,
958    previous_phase: Option<KanidmRestorePhase>,
959    phase: KanidmRestorePhase,
960    message: Option<&str>,
961) {
962    if phase == KanidmRestorePhase::Validating {
963        ctx.metrics.attempts.add(1, &[]);
964    }
965
966    let result = match phase {
967        KanidmRestorePhase::Completed => Some("success"),
968        KanidmRestorePhase::Failed => Some("failure"),
969        _ => None,
970    };
971    if let Some(result) = result {
972        let attributes = [KeyValue::new("result", result)];
973        ctx.metrics.outcomes.add(1, &attributes);
974        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
975            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
976            ctx.metrics.duration_seconds.record(elapsed, &attributes);
977        }
978    }
979
980    let reason = if phase == KanidmRestorePhase::Failed {
981        "RestoreFailed"
982    } else {
983        "RestorePhaseChanged"
984    };
985    let note = message.map(str::to_string).or_else(|| {
986        Some(format!(
987            "Kanidm restore phase changed from {} to {phase:?}.",
988            previous_phase
989                .map(|previous| format!("{previous:?}"))
990                .unwrap_or_else(|| "None".to_string())
991        ))
992    });
993    if let Err(error) = ctx
994        .recorder
995        .publish(
996            &Event {
997                type_: if phase == KanidmRestorePhase::Failed {
998                    EventType::Warning
999                } else {
1000                    EventType::Normal
1001                },
1002                reason: reason.to_string(),
1003                note,
1004                action: "Restore".to_string(),
1005                secondary: None,
1006            },
1007            &restore.object_ref(&()),
1008        )
1009        .await
1010    {
1011        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1012    }
1013}
1014
1015async fn set_phase(
1016    restore: &KanidmRestore,
1017    ctx: &RestoreContext,
1018    phase: KanidmRestorePhase,
1019    message: Option<String>,
1020) -> Result<()> {
1021    let mut status = restore.status.clone().unwrap_or_default();
1022    status.phase = phase;
1023    status.message = message;
1024    patch_status(restore, ctx, status).await
1025}
1026
1027async fn fail_after_mutation(
1028    restore: &KanidmRestore,
1029    ctx: &RestoreContext,
1030    message: &str,
1031) -> Result<()> {
1032    set_phase(
1033        restore,
1034        ctx,
1035        KanidmRestorePhase::Failed,
1036        Some(message.to_string()),
1037    )
1038    .await
1039}
1040
1041fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1042    target
1043        .spec
1044        .replica_groups
1045        .iter()
1046        .find(|rg| rg.primary_node)
1047        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1048}
1049
1050async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1051    let ns = target.namespace().unwrap();
1052    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1053    for rg in &target.spec.replica_groups {
1054        let name = target.statefulset_name(&rg.name);
1055        api.patch(
1056            &name,
1057            &PatchParams::default(),
1058            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1059        )
1060        .await
1061        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1062    }
1063    Ok(())
1064}
1065
1066async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1067    let ns = target.namespace().unwrap();
1068    let rg = primary_group(target)?;
1069    let name = target.statefulset_name(&rg.name);
1070    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1071        .patch(
1072            &name,
1073            &PatchParams::default(),
1074            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1075        )
1076        .await
1077        .map(|_| ())
1078        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1079}
1080
1081async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1082    let ns = target.namespace().unwrap();
1083    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1084    for rg in &target.spec.replica_groups {
1085        let name = target.statefulset_name(&rg.name);
1086        api.patch(
1087            &name,
1088            &PatchParams::default(),
1089            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1090        )
1091        .await
1092        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1093    }
1094    Ok(())
1095}
1096
1097async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098    let ns = target.namespace().unwrap();
1099    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1100        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1101        .await
1102        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1103    Ok(pods.items.is_empty())
1104}
1105
1106async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1107    let ns = target.namespace().unwrap();
1108    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1109    let mut pv_names = BTreeSet::new();
1110    let primary_name = primary_pvc_name(target)?;
1111    let primary_pvc = pvc_api
1112        .get(&primary_name)
1113        .await
1114        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1115    let primary_pv = primary_pvc
1116        .spec
1117        .as_ref()
1118        .and_then(|spec| spec.volume_name.clone())
1119        .ok_or_else(|| {
1120            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1121        })?;
1122    pv_names.insert(primary_pv);
1123
1124    for rg in &target.spec.replica_groups {
1125        let sts = target.statefulset_name(&rg.name);
1126        for ordinal in 0..rg.replicas {
1127            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1128            if name == primary_name {
1129                continue;
1130            }
1131            if let Some(pvc) = pvc_api
1132                .get_opt(&name)
1133                .await
1134                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1136            {
1137                pv_names.insert(pv_name);
1138            }
1139        }
1140    }
1141
1142    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1143        .list(&ListParams::default())
1144        .await
1145        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1146    Ok(!attachments.items.iter().any(|attachment| {
1147        attachment
1148            .spec
1149            .source
1150            .persistent_volume_name
1151            .as_ref()
1152            .is_some_and(|pv_name| pv_names.contains(pv_name))
1153    }))
1154}
1155
1156async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1157    let ns = target.namespace().unwrap();
1158    let name = target.statefulset_name(&primary_group(target)?.name);
1159    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1160        .get(&name)
1161        .await
1162        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1163    Ok(sts
1164        .status
1165        .as_ref()
1166        .and_then(|s| s.ready_replicas)
1167        .unwrap_or(0)
1168        >= 1)
1169}
1170
1171fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1172    let rg = primary_group(target)?;
1173    Ok(format!(
1174        "{DATA_VOLUME}-{}-0",
1175        target.statefulset_name(&rg.name)
1176    ))
1177}
1178
1179async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1180    let ns = target.namespace().unwrap();
1181    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1182    let mut all_absent = true;
1183    for rg in &target.spec.replica_groups {
1184        let sts = target.statefulset_name(&rg.name);
1185        for ordinal in 0..rg.replicas {
1186            if rg.primary_node && ordinal == 0 {
1187                continue;
1188            }
1189            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1190            match api
1191                .get_opt(&name)
1192                .await
1193                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1194            {
1195                None => {}
1196                Some(pvc) => {
1197                    all_absent = false;
1198                    if pvc.metadata.deletion_timestamp.is_none() {
1199                        match api.delete(&name, &DeleteParams::default()).await {
1200                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1201                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1202                            Err(error) => {
1203                                return Err(Error::kube_error(
1204                                    "delete",
1205                                    "PersistentVolumeClaim",
1206                                    &ns,
1207                                    &name,
1208                                    error,
1209                                ));
1210                            }
1211                        }
1212                    }
1213                }
1214            }
1215        }
1216    }
1217    Ok(all_absent)
1218}
1219
1220async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1221    let ns = target.namespace().unwrap();
1222    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1223    let label_selector = format!(
1224        "{}={},{}={}",
1225        CLUSTER_LABEL,
1226        target.name_any(),
1227        SECRET_TYPE_LABEL,
1228        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1229    );
1230    let secrets = api
1231        .list(&ListParams::default().labels(&label_selector))
1232        .await
1233        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1234    let mut all_absent = secrets.items.is_empty();
1235    for secret in secrets.items {
1236        let name = secret.name_any();
1237        if secret.metadata.deletion_timestamp.is_none() {
1238            match api.delete(&name, &DeleteParams::default()).await {
1239                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1240                Err(kube::Error::Api(status)) if status.code == 404 => {}
1241                Err(error) => {
1242                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1243                }
1244            }
1245            all_absent = false;
1246        }
1247    }
1248    Ok(all_absent)
1249}
1250
1251async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1252    let ns = target.namespace().unwrap();
1253    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1254    let name = format!("{}-admin-passwords", target.name_any());
1255    match api
1256        .get_opt(&name)
1257        .await
1258        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1259    {
1260        None => Ok(true),
1261        Some(secret) => {
1262            if secret.metadata.deletion_timestamp.is_some() {
1263                return Ok(false);
1264            }
1265            match api.delete(&name, &DeleteParams::default()).await {
1266                Ok(_) => {
1267                    debug!(secret = %name, "deleting stale admin secret");
1268                    Ok(false)
1269                }
1270                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1271                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1272            }
1273        }
1274    }
1275}
1276
1277fn restore_job_name(restore: &KanidmRestore) -> String {
1278    format!("{}-restore", restore.name_any())
1279}
1280fn verify_job_name(restore: &KanidmRestore) -> String {
1281    format!("{}-verify", restore.name_any())
1282}
1283
1284fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1285    let origin = target
1286        .spec
1287        .origin
1288        .clone()
1289        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1290    vec![
1291        EnvVar {
1292            name: "KANIDM_DB_PATH".to_string(),
1293            value: Some("/data/kanidm.db".to_string()),
1294            ..Default::default()
1295        },
1296        EnvVar {
1297            name: "KANIDM_DOMAIN".to_string(),
1298            value: Some(target.spec.domain.clone()),
1299            ..Default::default()
1300        },
1301        EnvVar {
1302            name: "KANIDM_ORIGIN".to_string(),
1303            value: Some(origin),
1304            ..Default::default()
1305        },
1306        EnvVar {
1307            name: "KANIDM_TLS_CHAIN".to_string(),
1308            value: Some(format!("{TLS_PATH}/tls.crt")),
1309            ..Default::default()
1310        },
1311        EnvVar {
1312            name: "KANIDM_TLS_KEY".to_string(),
1313            value: Some(format!("{TLS_PATH}/tls.key")),
1314            ..Default::default()
1315        },
1316    ]
1317}
1318
1319async fn ensure_database_job(
1320    restore: &KanidmRestore,
1321    target: &Kanidm,
1322    ctx: &RestoreContext,
1323    name: &str,
1324    verify: bool,
1325) -> Result<()> {
1326    let ns = restore.namespace().unwrap();
1327    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1328    if api
1329        .get_opt(name)
1330        .await
1331        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1332        .is_some()
1333    {
1334        return Ok(());
1335    }
1336    let mut command = vec![
1337        "kanidmd".to_string(),
1338        "database".to_string(),
1339        if verify { "backup" } else { "restore" }.to_string(),
1340    ];
1341    if verify {
1342        command.push("/verify/verification.json.gz".to_string());
1343    } else {
1344        if is_remote_source(restore) {
1345            command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1346        } else {
1347            let file_name = restore
1348                .spec
1349                .source
1350                .local
1351                .as_ref()
1352                .map(|l| l.file_name.clone())
1353                .unwrap_or_default();
1354            command.push(format!("{BACKUP_PATH}/{file_name}"));
1355        }
1356    }
1357    let job = Job {
1358        metadata: ObjectMeta {
1359            name: Some(name.to_string()),
1360            namespace: Some(ns.clone()),
1361            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1362            ..Default::default()
1363        },
1364        spec: Some(JobSpec {
1365            backoff_limit: Some(0),
1366            template: PodTemplateSpec {
1367                metadata: Some(ObjectMeta::default()),
1368                spec: Some(PodSpec {
1369                    automount_service_account_token: Some(false),
1370                    restart_policy: Some("Never".to_string()),
1371                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1372                        seccomp_profile: Some(SeccompProfile {
1373                            type_: "RuntimeDefault".to_string(),
1374                            ..Default::default()
1375                        }),
1376                        ..Default::default()
1377                    }),
1378                    containers: vec![Container {
1379                        name: if verify { "verify" } else { "restore" }.to_string(),
1380                        image: Some(restore.spec.restore_image.clone()),
1381                        command: Some(command),
1382                        env: Some(build_kanidm_db_env_vars(target)),
1383                        security_context: Some(kanidm_job_security_context()),
1384                        resources: Some(job_resource_requirements()),
1385                        volume_mounts: Some(vec![
1386                            VolumeMount {
1387                                name: DATA_VOLUME.to_string(),
1388                                mount_path: DATA_PATH.to_string(),
1389                                ..Default::default()
1390                            },
1391                            VolumeMount {
1392                                name: TLS_VOLUME.to_string(),
1393                                mount_path: TLS_PATH.to_string(),
1394                                read_only: Some(true),
1395                                ..Default::default()
1396                            },
1397                            VolumeMount {
1398                                name: "verification-output".to_string(),
1399                                mount_path: "/verify".to_string(),
1400                                ..Default::default()
1401                            },
1402                        ]),
1403                        ..Default::default()
1404                    }],
1405                    volumes: Some(vec![
1406                        Volume {
1407                            name: DATA_VOLUME.to_string(),
1408                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1409                                claim_name: primary_pvc_name(target)?,
1410                                read_only: Some(false),
1411                            }),
1412                            ..Default::default()
1413                        },
1414                        Volume {
1415                            name: TLS_VOLUME.to_string(),
1416                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1417                                secret_name: Some(target.effective_tls_secret_name()),
1418                                ..Default::default()
1419                            }),
1420                            ..Default::default()
1421                        },
1422                        Volume {
1423                            name: "verification-output".to_string(),
1424                            empty_dir: Some(Default::default()),
1425                            ..Default::default()
1426                        },
1427                    ]),
1428                    ..Default::default()
1429                }),
1430            },
1431            ..Default::default()
1432        }),
1433        ..Default::default()
1434    };
1435    api.create(&PostParams::default(), &job)
1436        .await
1437        .map(|_| ())
1438        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1439}
1440
1441enum JobState {
1442    Running,
1443    Complete,
1444    Failed,
1445}
1446
1447async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1448    let ns = restore.namespace().unwrap();
1449    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1450        .get(name)
1451        .await
1452        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1453    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1454        return Ok(JobState::Failed);
1455    }
1456    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1457        return Ok(JobState::Complete);
1458    }
1459    Ok(JobState::Running)
1460}
1461
1462fn is_remote_source(restore: &KanidmRestore) -> bool {
1463    restore.spec.source.backup_ref.is_some()
1464}
1465
1466fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1467    let safety = restore.spec.safety_backup.as_ref();
1468    let skip = safety.is_some_and(|s| s.skip);
1469    !skip
1470}
1471
1472fn validate_source(restore: &KanidmRestore) -> Result<()> {
1473    let has_local = restore.spec.source.local.is_some();
1474    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1475    if has_local && has_backup_ref {
1476        return Err(Error::MissingData(
1477            "source.local and source.backupRef are mutually exclusive".to_string(),
1478        ));
1479    }
1480    if !has_local && !has_backup_ref {
1481        return Err(Error::MissingData(
1482            "source must specify either local or backupRef".to_string(),
1483        ));
1484    }
1485    if let Some(local) = &restore.spec.source.local {
1486        if !safe_basename(&local.file_name) {
1487            return Err(Error::MissingData(
1488                "restore source fileName must be a safe basename".to_string(),
1489            ));
1490        }
1491    }
1492    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1493        if backup_ref.name.is_empty() {
1494            return Err(Error::MissingData(
1495                "source.backupRef.name must not be empty".to_string(),
1496            ));
1497        }
1498    }
1499    Ok(())
1500}
1501
1502fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1503    let safety = restore.spec.safety_backup.as_ref();
1504    let skip = safety.is_some_and(|s| s.skip);
1505    if skip {
1506        let annotations = restore.metadata.annotations.as_ref();
1507        let reason = annotations
1508            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1509            .map(|s| s.trim())
1510            .unwrap_or("");
1511        let approver = annotations
1512            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1513            .map(|s| s.trim())
1514            .unwrap_or("");
1515        if reason.is_empty() {
1516            return Err(Error::MissingData(format!(
1517                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1518            )));
1519        }
1520        if approver.is_empty() {
1521            return Err(Error::MissingData(format!(
1522                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1523            )));
1524        }
1525    }
1526    if is_remote_source(restore) && !skip {
1527        let has_repo = safety
1528            .and_then(|s| s.repository_ref.as_ref())
1529            .is_some_and(|r| !r.name.is_empty());
1530        if !has_repo {
1531            return Err(Error::MissingData(
1532                "remote restore requires safetyBackup.repositoryRef".to_string(),
1533            ));
1534        }
1535    }
1536    Ok(())
1537}
1538
1539fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1540    target
1541        .spec
1542        .replica_groups
1543        .iter()
1544        .map(|rg| ReplicaCountEntry {
1545            group: rg.name.clone(),
1546            replicas: rg.replicas,
1547        })
1548        .collect()
1549}
1550
1551async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1552    if let Ok(target) = get_target(restore, ctx).await {
1553        let owns_maintenance =
1554            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1555        if owns_maintenance {
1556            scale_desired(&target, ctx).await?;
1557            clear_restoring(restore, &target, ctx).await?;
1558        }
1559    }
1560    Ok(())
1561}
1562
1563async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564    ctx.metrics.break_glass_total.add(1, &[]);
1565    let reason = restore
1566        .metadata
1567        .annotations
1568        .as_ref()
1569        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1570        .cloned()
1571        .unwrap_or_default();
1572    let approver = restore
1573        .metadata
1574        .annotations
1575        .as_ref()
1576        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1577        .cloned()
1578        .unwrap_or_default();
1579    let mut status = restore.status.clone().unwrap_or_default();
1580    let break_glass_condition = restore_condition(
1581        &status.conditions,
1582        CONDITION_BREAK_GLASS,
1583        CONDITION_TRUE,
1584        "BreakGlassOverride",
1585        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1586        restore.metadata.generation,
1587    );
1588    status
1589        .conditions
1590        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1591    status.conditions.push(break_glass_condition);
1592    patch_status(restore, ctx, status).await?;
1593    if let Err(error) = ctx
1594        .recorder
1595        .publish(
1596            &Event {
1597                type_: EventType::Warning,
1598                reason: "SafetyBackupSkipped".to_string(),
1599                note: Some(format!(
1600                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1601                )),
1602                action: "Restore".to_string(),
1603                secondary: None,
1604            },
1605            &restore.object_ref(&()),
1606        )
1607        .await
1608    {
1609        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1610    }
1611    warn!(
1612        restore = %restore.name_any(),
1613        reason = %reason,
1614        approver = %approver,
1615        "break-glass safety backup override activated"
1616    );
1617    Ok(())
1618}
1619
1620async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1621    let status = restore.status.as_ref().cloned().unwrap_or_default();
1622    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1623        if let Ok(start_ts) = start.parse::<Timestamp>() {
1624            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1625            ctx.metrics
1626                .safety_backup_duration_seconds
1627                .record(elapsed, &[]);
1628        }
1629    }
1630}
1631
1632fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1633    format!("{}-safety-backup", restore.name_any())
1634}
1635
1636fn source_prep_job_name(restore: &KanidmRestore) -> String {
1637    format!("{}-source-prep", restore.name_any())
1638}
1639
1640fn source_check_job_name(restore: &KanidmRestore) -> String {
1641    format!("{}-source-check", restore.name_any())
1642}
1643
1644async fn ensure_source_check_job(
1645    restore: &KanidmRestore,
1646    target: &Kanidm,
1647    ctx: &RestoreContext,
1648    name: &str,
1649) -> Result<()> {
1650    let ns = restore.namespace().unwrap();
1651    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1652    if api
1653        .get_opt(name)
1654        .await
1655        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1656        .is_some()
1657    {
1658        return Ok(());
1659    }
1660    let file_name = restore
1661        .spec
1662        .source
1663        .local
1664        .as_ref()
1665        .map(|l| l.file_name.clone())
1666        .unwrap_or_default();
1667    let operation_doc = serde_json::json!({
1668        "apiVersion": "backup.kaniop.rs/v1alpha1",
1669        "kind": "OperationDocument",
1670        "operation": "check",
1671        "path": format!("{BACKUP_PATH}/{file_name}"),
1672        "resultPath": "/run/kaniop-result/result.json",
1673        "format": "kanidmJsonGzip",
1674    })
1675    .to_string();
1676    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1677    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1678    let data_mover_image = data_mover_image();
1679    let job = Job {
1680        metadata: ObjectMeta {
1681            name: Some(name.to_string()),
1682            namespace: Some(ns.clone()),
1683            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1684            ..Default::default()
1685        },
1686        spec: Some(JobSpec {
1687            backoff_limit: Some(0),
1688            template: PodTemplateSpec {
1689                metadata: Some(ObjectMeta::default()),
1690                spec: Some(PodSpec {
1691                    automount_service_account_token: Some(false),
1692                    restart_policy: Some("Never".to_string()),
1693                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1694                        run_as_non_root: Some(true),
1695                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1696                            type_: "RuntimeDefault".to_string(),
1697                            ..Default::default()
1698                        }),
1699                        ..Default::default()
1700                    }),
1701                    containers: vec![Container {
1702                        name: "source-check".to_string(),
1703                        image: Some(data_mover_image),
1704                        command: Some(vec![
1705                            "/bin/kaniop-data-mover".to_string(),
1706                            "check".to_string(),
1707                            "--operation-doc".to_string(),
1708                            "/run/kaniop/operation.json".to_string(),
1709                        ]),
1710                        security_context: Some(hardened_security_context()),
1711                        resources: Some(job_resource_requirements()),
1712                        env: Some(vec![EnvVar {
1713                            name: "RUST_LOG".to_string(),
1714                            value: Some("info".to_string()),
1715                            ..Default::default()
1716                        }]),
1717                        volume_mounts: Some(vec![
1718                            VolumeMount {
1719                                name: DATA_VOLUME.to_string(),
1720                                mount_path: DATA_PATH.to_string(),
1721                                read_only: Some(true),
1722                                ..Default::default()
1723                            },
1724                            VolumeMount {
1725                                name: "operation".to_string(),
1726                                mount_path: "/run/kaniop".to_string(),
1727                                read_only: Some(true),
1728                                ..Default::default()
1729                            },
1730                            VolumeMount {
1731                                name: "result".to_string(),
1732                                mount_path: "/run/kaniop-result".to_string(),
1733                                ..Default::default()
1734                            },
1735                        ]),
1736                        ..Default::default()
1737                    }],
1738                    volumes: Some(vec![
1739                        Volume {
1740                            name: DATA_VOLUME.to_string(),
1741                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1742                                claim_name: primary_pvc_name(target)?,
1743                                read_only: Some(true),
1744                            }),
1745                            ..Default::default()
1746                        },
1747                        Volume {
1748                            name: "operation".to_string(),
1749                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1750                                name: operation_cm_name,
1751                                ..Default::default()
1752                            }),
1753                            ..Default::default()
1754                        },
1755                        Volume {
1756                            name: "result".to_string(),
1757                            empty_dir: Some(Default::default()),
1758                            ..Default::default()
1759                        },
1760                    ]),
1761                    ..Default::default()
1762                }),
1763            },
1764            ..Default::default()
1765        }),
1766        ..Default::default()
1767    };
1768    api.create(&PostParams::default(), &job)
1769        .await
1770        .map(|_| ())
1771        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1772}
1773
1774const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1775
1776fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1777    let ns = restore.namespace().unwrap_or_default();
1778    let name = restore.name_any();
1779    let namespace_uuid =
1780        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1781    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1782    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1783}
1784
1785fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1786    let ns = restore.namespace().unwrap_or_default();
1787    let kanidm_uid = &restore.spec.target_ref.uid;
1788    let backup_id = compute_safety_backup_id(restore);
1789    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1790}
1791
1792struct VerifiedSafetyBackup {
1793    manifest_key: String,
1794    payload_sha256: String,
1795}
1796
1797async fn read_safety_backup_result(
1798    restore: &KanidmRestore,
1799    ctx: &RestoreContext,
1800    job_name: &str,
1801    expected_backup_id: &str,
1802) -> Result<VerifiedSafetyBackup> {
1803    let ns = restore.namespace().unwrap();
1804    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1805    let pod_list = pods
1806        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1807        .await
1808        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1809    let pod = pod_list.items.first().ok_or_else(|| {
1810        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1811    })?;
1812    let pod_name = pod.name_any();
1813    let container_status = pod
1814        .status
1815        .as_ref()
1816        .and_then(|s| s.container_statuses.as_ref())
1817        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1818        .ok_or_else(|| {
1819            Error::MissingData(format!(
1820                "safety-upload container status not found in pod {pod_name}"
1821            ))
1822        })?;
1823    let termination_message = container_status
1824        .state
1825        .as_ref()
1826        .and_then(|state| state.terminated.as_ref())
1827        .and_then(|t| t.message.as_ref())
1828        .ok_or_else(|| {
1829            Error::MissingData(format!(
1830                "termination message not found in pod {pod_name}; result transport failed"
1831            ))
1832        })?;
1833    if termination_message.is_empty() {
1834        return Err(Error::MissingData(format!(
1835            "termination message is empty in pod {pod_name}; result document was not written"
1836        )));
1837    }
1838    let result_doc = parse_result_document(termination_message).map_err(|e| {
1839        Error::ParseError(format!(
1840            "failed to parse result document from pod {pod_name}: {e}"
1841        ))
1842    })?;
1843    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1844        return Err(Error::ParseError(format!(
1845            "unsupported result document version: {}",
1846            result_doc.api_version
1847        )));
1848    }
1849    if result_doc.kind != "ResultDocument" {
1850        return Err(Error::ParseError(format!(
1851            "invalid result document kind: {}",
1852            result_doc.kind
1853        )));
1854    }
1855    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1856        return Err(Error::ParseError(format!(
1857            "result document operation mismatch: expected '{}', got '{}'",
1858            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1859        )));
1860    }
1861    if !result_doc.success {
1862        return Err(Error::ParseError(format!(
1863            "safety backup reported failure with exit code {:?}",
1864            result_doc.exit_code
1865        )));
1866    }
1867    if result_doc.exit_code != ExitCode::Success {
1868        return Err(Error::ParseError(format!(
1869            "safety backup exit code is not Success: {:?}",
1870            result_doc.exit_code
1871        )));
1872    }
1873    let doc_backup_id = result_doc
1874        .backup_id
1875        .as_deref()
1876        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1877    if doc_backup_id != expected_backup_id {
1878        return Err(Error::ParseError(format!(
1879            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1880        )));
1881    }
1882    let manifest_key = result_doc
1883        .manifest_key
1884        .clone()
1885        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1886    if manifest_key.is_empty() {
1887        return Err(Error::ParseError(
1888            "result document manifestKey is empty".to_string(),
1889        ));
1890    }
1891    let payload_sha256 = result_doc
1892        .payload_sha256
1893        .clone()
1894        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1895    if payload_sha256.is_empty() {
1896        return Err(Error::ParseError(
1897            "result document payloadSha256 is empty".to_string(),
1898        ));
1899    }
1900    info!(
1901        restore = %restore.name_any(),
1902        backup_id = %expected_backup_id,
1903        manifest_key = %manifest_key,
1904        "safety backup result verified"
1905    );
1906    Ok(VerifiedSafetyBackup {
1907        manifest_key,
1908        payload_sha256,
1909    })
1910}
1911
1912struct VerifiedSourcePrepResult {
1913    #[allow(dead_code)]
1914    manifest_key: String,
1915    #[allow(dead_code)]
1916    payload_sha256: String,
1917}
1918
1919async fn read_source_prep_result(
1920    restore: &KanidmRestore,
1921    ctx: &RestoreContext,
1922    job_name: &str,
1923    expected_backup_id: &str,
1924) -> Result<VerifiedSourcePrepResult> {
1925    let ns = restore.namespace().unwrap();
1926    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1927    let pod_list = pods
1928        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1929        .await
1930        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1931    let pod = pod_list.items.first().ok_or_else(|| {
1932        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1933    })?;
1934    let pod_name = pod.name_any();
1935    let container_status = pod
1936        .status
1937        .as_ref()
1938        .and_then(|s| s.container_statuses.as_ref())
1939        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1940        .ok_or_else(|| {
1941            Error::MissingData(format!(
1942                "source-prep container status not found in pod {pod_name}"
1943            ))
1944        })?;
1945    let termination_message = container_status
1946        .state
1947        .as_ref()
1948        .and_then(|state| state.terminated.as_ref())
1949        .and_then(|t| t.message.as_ref())
1950        .ok_or_else(|| {
1951            Error::MissingData(format!(
1952                "termination message not found in pod {pod_name}; result transport failed"
1953            ))
1954        })?;
1955    if termination_message.is_empty() {
1956        return Err(Error::MissingData(format!(
1957            "termination message is empty in pod {pod_name}; result document was not written"
1958        )));
1959    }
1960    let result_doc = parse_result_document(termination_message).map_err(|e| {
1961        Error::ParseError(format!(
1962            "failed to parse result document from pod {pod_name}: {e}"
1963        ))
1964    })?;
1965    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1966        return Err(Error::ParseError(format!(
1967            "unsupported result document version: {}",
1968            result_doc.api_version
1969        )));
1970    }
1971    if result_doc.kind != "ResultDocument" {
1972        return Err(Error::ParseError(format!(
1973            "invalid result document kind: {}",
1974            result_doc.kind
1975        )));
1976    }
1977    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1978        return Err(Error::ParseError(format!(
1979            "result document operation mismatch: expected '{}', got '{}'",
1980            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1981        )));
1982    }
1983    if !result_doc.success {
1984        return Err(Error::ParseError(format!(
1985            "source prep reported failure with exit code {:?}",
1986            result_doc.exit_code
1987        )));
1988    }
1989    if result_doc.exit_code != ExitCode::Success {
1990        return Err(Error::ParseError(format!(
1991            "source prep exit code is not Success: {:?}",
1992            result_doc.exit_code
1993        )));
1994    }
1995    let doc_backup_id = result_doc
1996        .backup_id
1997        .as_deref()
1998        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1999    if doc_backup_id != expected_backup_id {
2000        return Err(Error::ParseError(format!(
2001            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2002        )));
2003    }
2004    let manifest_key = result_doc
2005        .manifest_key
2006        .clone()
2007        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2008    if manifest_key.is_empty() {
2009        return Err(Error::ParseError(
2010            "result document manifestKey is empty".to_string(),
2011        ));
2012    }
2013    let payload_sha256 = result_doc
2014        .payload_sha256
2015        .clone()
2016        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2017    if payload_sha256.is_empty() {
2018        return Err(Error::ParseError(
2019            "result document payloadSha256 is empty".to_string(),
2020        ));
2021    }
2022    info!(
2023        restore = %restore.name_any(),
2024        backup_id = %expected_backup_id,
2025        manifest_key = %manifest_key,
2026        payload_sha256 = %payload_sha256,
2027        "source prep result verified"
2028    );
2029    Ok(VerifiedSourcePrepResult {
2030        manifest_key,
2031        payload_sha256,
2032    })
2033}
2034
2035fn hardened_security_context() -> SecurityContext {
2036    SecurityContext {
2037        allow_privilege_escalation: Some(false),
2038        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2039            drop: Some(vec!["ALL".to_string()]),
2040            ..Default::default()
2041        }),
2042        read_only_root_filesystem: Some(true),
2043        run_as_non_root: Some(true),
2044        seccomp_profile: Some(SeccompProfile {
2045            type_: "RuntimeDefault".to_string(),
2046            ..Default::default()
2047        }),
2048        ..Default::default()
2049    }
2050}
2051
2052fn kanidm_job_security_context() -> SecurityContext {
2053    SecurityContext {
2054        allow_privilege_escalation: Some(false),
2055        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2056            drop: Some(vec!["ALL".to_string()]),
2057            ..Default::default()
2058        }),
2059        seccomp_profile: Some(SeccompProfile {
2060            type_: "RuntimeDefault".to_string(),
2061            ..Default::default()
2062        }),
2063        ..Default::default()
2064    }
2065}
2066
2067fn job_resource_requirements() -> ResourceRequirements {
2068    ResourceRequirements {
2069        requests: Some(std::collections::BTreeMap::from([
2070            ("cpu".to_string(), Quantity("100m".to_string())),
2071            ("memory".to_string(), Quantity("128Mi".to_string())),
2072        ])),
2073        limits: Some(std::collections::BTreeMap::from([
2074            ("cpu".to_string(), Quantity("2".to_string())),
2075            ("memory".to_string(), Quantity("2Gi".to_string())),
2076        ])),
2077        ..Default::default()
2078    }
2079}
2080
2081async fn ensure_safety_backup_job(
2082    restore: &KanidmRestore,
2083    target: &Kanidm,
2084    ctx: &RestoreContext,
2085    name: &str,
2086    backup_id: &str,
2087    manifest_key: &str,
2088) -> Result<()> {
2089    let ns = restore.namespace().unwrap();
2090    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2091    if api
2092        .get_opt(name)
2093        .await
2094        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2095        .is_some()
2096    {
2097        return Ok(());
2098    }
2099    let safety = restore
2100        .spec
2101        .safety_backup
2102        .as_ref()
2103        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2104    let repo_ref = safety
2105        .repository_ref
2106        .as_ref()
2107        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2108    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2109        .get(&repo_ref.name)
2110        .await
2111        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2112    let _namespace_uid = &ns;
2113    let _kanidm_uid = &restore.spec.target_ref.uid;
2114    let endpoint = &repo.spec.s3.endpoint;
2115    let region = &repo.spec.s3.region;
2116    let operation_doc = build_safety_upload_operation_doc(
2117        restore,
2118        target,
2119        backup_id,
2120        manifest_key,
2121        &repo.spec.s3.bucket,
2122        &repo.spec.s3.prefix,
2123        endpoint,
2124        region,
2125        repo.spec.s3.force_path_style,
2126        repo.spec.s3.insecure,
2127        repo.spec
2128            .s3
2129            .ca_bundle_ref
2130            .as_deref()
2131            .map(|_| ca_bundle_path())
2132            .as_deref(),
2133        repo.spec.encryption.as_ref(),
2134    )?;
2135    let operation_cm_name = format!("{name}-op");
2136    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2137    let auth_method = &repo.spec.authentication.writer;
2138    let data_mover_image = data_mover_image();
2139    let init_command = vec![
2140        "kanidmd".to_string(),
2141        "database".to_string(),
2142        "backup".to_string(),
2143        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2144    ];
2145    let upload_script = format!(
2146        r#"set -eu
2147/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2148RESULT_FILE="/run/kaniop-result/result.json"
2149TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2150if [ ! -f "$RESULT_FILE" ]; then
2151  echo "ERROR: result document not found at $RESULT_FILE"
2152  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2153  exit 1
2154fi
2155if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2156  echo "ERROR: safety backup upload reported failure"
2157  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2158  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2159  exit 1
2160fi
2161dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2162echo "safety backup upload completed successfully"
2163"#
2164    );
2165    let job = Job {
2166        metadata: ObjectMeta {
2167            name: Some(name.to_string()),
2168            namespace: Some(ns.clone()),
2169            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2170            ..Default::default()
2171        },
2172        spec: Some(JobSpec {
2173            backoff_limit: Some(0),
2174            template: PodTemplateSpec {
2175                metadata: Some(ObjectMeta::default()),
2176                spec: Some(PodSpec {
2177                    automount_service_account_token: Some(false),
2178                    restart_policy: Some("Never".to_string()),
2179                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2180                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2181                            type_: "RuntimeDefault".to_string(),
2182                            ..Default::default()
2183                        }),
2184                        ..Default::default()
2185                    }),
2186                    init_containers: Some(vec![Container {
2187                        name: "safety-backup".to_string(),
2188                        image: Some(restore.spec.restore_image.clone()),
2189                        command: Some(init_command),
2190                        security_context: Some(SecurityContext {
2191                            allow_privilege_escalation: Some(false),
2192                            capabilities: Some(Capabilities {
2193                                drop: Some(vec!["ALL".to_string()]),
2194                                ..Default::default()
2195                            }),
2196                            read_only_root_filesystem: Some(true),
2197                            seccomp_profile: Some(SeccompProfile {
2198                                type_: "RuntimeDefault".to_string(),
2199                                ..Default::default()
2200                            }),
2201                            ..Default::default()
2202                        }),
2203                        resources: Some(job_resource_requirements()),
2204                        env: Some(build_kanidm_db_env_vars(target)),
2205                        volume_mounts: Some(vec![
2206                            VolumeMount {
2207                                name: DATA_VOLUME.to_string(),
2208                                mount_path: DATA_PATH.to_string(),
2209                                read_only: Some(false),
2210                                ..Default::default()
2211                            },
2212                            VolumeMount {
2213                                name: TLS_VOLUME.to_string(),
2214                                mount_path: TLS_PATH.to_string(),
2215                                read_only: Some(true),
2216                                ..Default::default()
2217                            },
2218                            VolumeMount {
2219                                name: SHARED_VOLUME.to_string(),
2220                                mount_path: SHARED_VOL_PATH.to_string(),
2221                                ..Default::default()
2222                            },
2223                        ]),
2224                        ..Default::default()
2225                    }]),
2226                    containers: vec![Container {
2227                        name: "safety-upload".to_string(),
2228                        image: Some(data_mover_image),
2229                        command: Some(vec!["/bin/sh".to_string()]),
2230                        args: Some(vec!["-c".to_string(), upload_script]),
2231                        security_context: Some(hardened_security_context()),
2232                        resources: Some(job_resource_requirements()),
2233                        env: {
2234                            let mut env_vars =
2235                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2236                            env_vars.push(EnvVar {
2237                                name: "RUST_LOG".to_string(),
2238                                value: Some("info".to_string()),
2239                                ..Default::default()
2240                            });
2241                            if repo.spec.s3.ca_bundle_ref.is_some() {
2242                                env_vars.push(ca_bundle_env_var());
2243                            }
2244                            env_vars.extend(build_encryption_env_vars(
2245                                repo.spec
2246                                    .encryption
2247                                    .as_ref()
2248                                    .and_then(|e| e.key_ref.as_ref()),
2249                            ));
2250                            Some(env_vars)
2251                        },
2252                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2253                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2254                        volume_mounts: {
2255                            let mut mounts = vec![
2256                                VolumeMount {
2257                                    name: SHARED_VOLUME.to_string(),
2258                                    mount_path: SHARED_VOL_PATH.to_string(),
2259                                    read_only: Some(true),
2260                                    ..Default::default()
2261                                },
2262                                VolumeMount {
2263                                    name: "operation".to_string(),
2264                                    mount_path: "/run/kaniop".to_string(),
2265                                    read_only: Some(true),
2266                                    ..Default::default()
2267                                },
2268                                VolumeMount {
2269                                    name: "result".to_string(),
2270                                    mount_path: "/run/kaniop-result".to_string(),
2271                                    ..Default::default()
2272                                },
2273                            ];
2274                            mounts.extend(build_auth_volume_mounts(auth_method));
2275                            if repo.spec.s3.ca_bundle_ref.is_some() {
2276                                mounts.push(build_ca_bundle_volume_mount());
2277                            }
2278                            Some(mounts)
2279                        },
2280                        ..Default::default()
2281                    }],
2282                    volumes: {
2283                        let mut vols = vec![
2284                            Volume {
2285                                name: DATA_VOLUME.to_string(),
2286                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2287                                    claim_name: primary_pvc_name(target)?,
2288                                    read_only: Some(false),
2289                                }),
2290                                ..Default::default()
2291                            },
2292                            Volume {
2293                                name: TLS_VOLUME.to_string(),
2294                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2295                                    secret_name: Some(target.effective_tls_secret_name()),
2296                                    ..Default::default()
2297                                }),
2298                                ..Default::default()
2299                            },
2300                            Volume {
2301                                name: SHARED_VOLUME.to_string(),
2302                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2303                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2304                                    ..Default::default()
2305                                }),
2306                                ..Default::default()
2307                            },
2308                            Volume {
2309                                name: "operation".to_string(),
2310                                config_map: Some(ConfigMapVolumeSource {
2311                                    name: operation_cm_name,
2312                                    ..Default::default()
2313                                }),
2314                                ..Default::default()
2315                            },
2316                            Volume {
2317                                name: "result".to_string(),
2318                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2319                                    size_limit: Some(Quantity("16Mi".to_string())),
2320                                    ..Default::default()
2321                                }),
2322                                ..Default::default()
2323                            },
2324                        ];
2325                        vols.extend(build_auth_volumes(auth_method));
2326                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2327                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2328                        }
2329                        Some(vols)
2330                    },
2331                    ..Default::default()
2332                }),
2333            },
2334            ..Default::default()
2335        }),
2336        ..Default::default()
2337    };
2338    api.create(&PostParams::default(), &job)
2339        .await
2340        .map(|_| ())
2341        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2342}
2343
2344async fn ensure_source_prep_job(
2345    restore: &KanidmRestore,
2346    target: &Kanidm,
2347    ctx: &RestoreContext,
2348    name: &str,
2349) -> Result<()> {
2350    let ns = restore.namespace().unwrap();
2351    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2352    if api
2353        .get_opt(name)
2354        .await
2355        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2356        .is_some()
2357    {
2358        return Ok(());
2359    }
2360    let backup_name = restore
2361        .spec
2362        .source
2363        .backup_ref
2364        .as_ref()
2365        .map(|b| b.name.clone())
2366        .unwrap_or_default();
2367    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2368        .get(&backup_name)
2369        .await
2370        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2371    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2372        .get(&backup.spec.repository_ref.name)
2373        .await
2374        .map_err(|e| {
2375            Error::kube_error(
2376                "get",
2377                "KanidmBackupRepository",
2378                &ns,
2379                &backup.spec.repository_ref.name,
2380                e,
2381            )
2382        })?;
2383    let endpoint = &repo.spec.s3.endpoint;
2384    let region = &repo.spec.s3.region;
2385    let operation_doc = build_download_operation_doc(
2386        restore,
2387        target,
2388        &backup.spec.manifest_key,
2389        &backup.spec.backup_id,
2390        &repo.spec.s3.bucket,
2391        &repo.spec.s3.prefix,
2392        endpoint,
2393        region,
2394        repo.spec.s3.force_path_style,
2395        repo.spec.s3.insecure,
2396        repo.spec
2397            .s3
2398            .ca_bundle_ref
2399            .as_deref()
2400            .map(|_| ca_bundle_path())
2401            .as_deref(),
2402        repo.spec.encryption.as_ref(),
2403    );
2404    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2405    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2406    let auth_method = &repo.spec.authentication.reader;
2407    let data_mover_image = data_mover_image();
2408    let download_script = format!(
2409        r#"set -eu
2410/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2411RESULT_FILE="/run/kaniop-result/result.json"
2412TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2413if [ ! -f "$RESULT_FILE" ]; then
2414  echo "ERROR: result document not found at $RESULT_FILE"
2415  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2416  exit 1
2417fi
2418if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2419  echo "ERROR: source preparation download reported failure"
2420  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2421  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2422  exit 1
2423fi
2424dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2425echo "source preparation download completed successfully"
2426"#
2427    );
2428    let job = Job {
2429        metadata: ObjectMeta {
2430            name: Some(name.to_string()),
2431            namespace: Some(ns.clone()),
2432            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2433            ..Default::default()
2434        },
2435        spec: Some(JobSpec {
2436            backoff_limit: Some(0),
2437            template: PodTemplateSpec {
2438                metadata: Some(ObjectMeta::default()),
2439                spec: Some(PodSpec {
2440                    automount_service_account_token: Some(false),
2441                    restart_policy: Some("Never".to_string()),
2442                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2443                        run_as_non_root: Some(true),
2444                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2445                            type_: "RuntimeDefault".to_string(),
2446                            ..Default::default()
2447                        }),
2448                        ..Default::default()
2449                    }),
2450                    containers: vec![Container {
2451                        name: "source-prep".to_string(),
2452                        image: Some(data_mover_image),
2453                        command: Some(vec!["/bin/sh".to_string()]),
2454                        args: Some(vec!["-c".to_string(), download_script]),
2455                        security_context: Some(hardened_security_context()),
2456                        resources: Some(job_resource_requirements()),
2457                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2458                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2459                        env: {
2460                            let mut env_vars = build_auth_env_vars(
2461                                auth_method,
2462                                &backup.spec.repository_ref.name,
2463                                AuthRole::Reader,
2464                            );
2465                            env_vars.push(EnvVar {
2466                                name: "RUST_LOG".to_string(),
2467                                value: Some("info".to_string()),
2468                                ..Default::default()
2469                            });
2470                            if repo.spec.s3.ca_bundle_ref.is_some() {
2471                                env_vars.push(ca_bundle_env_var());
2472                            }
2473                            env_vars.extend(build_encryption_env_vars(
2474                                repo.spec
2475                                    .encryption
2476                                    .as_ref()
2477                                    .and_then(|e| e.key_ref.as_ref()),
2478                            ));
2479                            Some(env_vars)
2480                        },
2481                        volume_mounts: {
2482                            let mut mounts = vec![
2483                                VolumeMount {
2484                                    name: STAGING_VOLUME.to_string(),
2485                                    mount_path: STAGING_PATH.to_string(),
2486                                    ..Default::default()
2487                                },
2488                                VolumeMount {
2489                                    name: "operation".to_string(),
2490                                    mount_path: "/run/kaniop".to_string(),
2491                                    read_only: Some(true),
2492                                    ..Default::default()
2493                                },
2494                                VolumeMount {
2495                                    name: "result".to_string(),
2496                                    mount_path: "/run/kaniop-result".to_string(),
2497                                    ..Default::default()
2498                                },
2499                            ];
2500                            mounts.extend(build_auth_volume_mounts(auth_method));
2501                            if repo.spec.s3.ca_bundle_ref.is_some() {
2502                                mounts.push(build_ca_bundle_volume_mount());
2503                            }
2504                            Some(mounts)
2505                        },
2506                        ..Default::default()
2507                    }],
2508                    volumes: {
2509                        let mut vols = vec![
2510                            Volume {
2511                                name: STAGING_VOLUME.to_string(),
2512                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2513                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2514                                    ..Default::default()
2515                                }),
2516                                ..Default::default()
2517                            },
2518                            Volume {
2519                                name: "operation".to_string(),
2520                                config_map: Some(ConfigMapVolumeSource {
2521                                    name: operation_cm_name,
2522                                    ..Default::default()
2523                                }),
2524                                ..Default::default()
2525                            },
2526                            Volume {
2527                                name: "result".to_string(),
2528                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2529                                    size_limit: Some(Quantity("16Mi".to_string())),
2530                                    ..Default::default()
2531                                }),
2532                                ..Default::default()
2533                            },
2534                        ];
2535                        vols.extend(build_auth_volumes(auth_method));
2536                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2537                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2538                        }
2539                        Some(vols)
2540                    },
2541                    ..Default::default()
2542                }),
2543            },
2544            ..Default::default()
2545        }),
2546        ..Default::default()
2547    };
2548    api.create(&PostParams::default(), &job)
2549        .await
2550        .map(|_| ())
2551        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2552}
2553
2554#[allow(clippy::too_many_arguments)]
2555fn build_safety_upload_operation_doc(
2556    restore: &KanidmRestore,
2557    target: &Kanidm,
2558    backup_id: &str,
2559    manifest_key: &str,
2560    bucket: &str,
2561    prefix: &str,
2562    endpoint: &str,
2563    region: &str,
2564    force_path_style: bool,
2565    insecure: bool,
2566    ca_bundle_path: Option<&str>,
2567    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2568) -> Result<String> {
2569    let enc_mode =
2570        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2571    let enc_key_id = encryption
2572        .and_then(|e| e.key_id.as_ref())
2573        .map(|s| serde_json::Value::String(s.clone()));
2574    let op = UploadOperation {
2575        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2576        bucket: bucket.to_string(),
2577        prefix: prefix.to_string(),
2578        endpoint: endpoint.to_string(),
2579        region: region.to_string(),
2580        force_path_style,
2581        insecure,
2582        ca_bundle_path: ca_bundle_path.map(str::to_string),
2583        backup_id: backup_id.to_string(),
2584        namespace_uid: restore.namespace().unwrap_or_default(),
2585        kanidm_uid: restore.spec.target_ref.uid.clone(),
2586        kanidm_name: restore.spec.target_ref.name.clone(),
2587        domain: target.spec.domain.clone(),
2588        kanidm_version: target
2589            .status
2590            .as_ref()
2591            .and_then(|s| s.version.as_ref())
2592            .map(|v| v.image_tag.clone())
2593            .unwrap_or_default(),
2594        image_digest: None,
2595        consistency: "kanidm-offline".to_string(),
2596        reason: "restore-safety".to_string(),
2597        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2598        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2599        result_path: "/run/kaniop-result/result.json".to_string(),
2600        max_concurrent_parts: 4,
2601        max_retries: 3,
2602    };
2603    let _ = manifest_key;
2604    let doc = OperationDocument {
2605        api_version: OPERATION_DOC_VERSION.to_string(),
2606        kind: "OperationDocument".to_string(),
2607        spec: OperationSpec::Upload(op),
2608    };
2609    serde_json::to_string(&doc)
2610        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2611}
2612
2613async fn validate_backup_ref(
2614    restore: &KanidmRestore,
2615    target: &Kanidm,
2616    ctx: &RestoreContext,
2617) -> Result<()> {
2618    let backup_ref = restore
2619        .spec
2620        .source
2621        .backup_ref
2622        .as_ref()
2623        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2624    let ns = restore.namespace().unwrap();
2625    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2626        .get(&backup_ref.name)
2627        .await
2628        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2629    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2630        return Err(Error::MissingData(format!(
2631            "KanidmBackup '{}' is not Ready (phase: {:?})",
2632            backup_ref.name,
2633            backup.status.as_ref().map(|s| &s.phase)
2634        )));
2635    }
2636    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2637        return Err(Error::MissingData(format!(
2638            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2639            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2640        )));
2641    }
2642    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2643        return Err(Error::MissingData(format!(
2644            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2645            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2646        )));
2647    }
2648    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2649        .get(&backup.spec.repository_ref.name)
2650        .await
2651        .map_err(|e| {
2652            Error::kube_error(
2653                "get",
2654                "KanidmBackupRepository",
2655                &ns,
2656                &backup.spec.repository_ref.name,
2657                e,
2658            )
2659        })?;
2660    if !has_accepted_condition(
2661        &repo
2662            .status
2663            .as_ref()
2664            .map(|s| &s.conditions)
2665            .cloned()
2666            .unwrap_or_default(),
2667    ) {
2668        return Err(Error::MissingData(format!(
2669            "KanidmBackupRepository '{}' configuration has not been accepted",
2670            backup.spec.repository_ref.name
2671        )));
2672    }
2673    validate_backup_compatibility(restore, target, &backup)?;
2674    Ok(())
2675}
2676
2677fn has_accepted_condition(conditions: &[Condition]) -> bool {
2678    conditions
2679        .iter()
2680        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2681}
2682
2683fn validate_backup_compatibility(
2684    restore: &KanidmRestore,
2685    target: &Kanidm,
2686    backup: &KanidmBackup,
2687) -> Result<()> {
2688    let status = backup.status.as_ref();
2689    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2690        let target_version = target
2691            .status
2692            .as_ref()
2693            .and_then(|s| s.version.as_ref())
2694            .map(|v| v.image_tag.as_str());
2695        if let Some(tv) = target_version {
2696            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2697                return Err(Error::MissingData(format!(
2698                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2699                )));
2700            }
2701        }
2702    }
2703    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2704        if !backup_digest.is_empty()
2705            && restore.spec.restore_image.contains('@')
2706            && !restore.spec.restore_image.contains(backup_digest)
2707        {
2708            return Err(Error::MissingData(format!(
2709                "restore image digest does not match backup image digest '{backup_digest}'"
2710            )));
2711        }
2712    }
2713    Ok(())
2714}
2715
2716#[allow(clippy::too_many_arguments)]
2717fn build_download_operation_doc(
2718    restore: &KanidmRestore,
2719    target: &Kanidm,
2720    manifest_key: &str,
2721    expected_backup_id: &str,
2722    bucket: &str,
2723    prefix: &str,
2724    endpoint: &str,
2725    region: &str,
2726    force_path_style: bool,
2727    insecure: bool,
2728    ca_bundle_path: Option<&str>,
2729    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2730) -> String {
2731    let enc_mode =
2732        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2733    let enc_key_id = encryption
2734        .and_then(|e| e.key_id.as_ref())
2735        .map(|s| serde_json::Value::String(s.clone()));
2736    serde_json::json!({
2737        "apiVersion": "backup.kaniop.rs/v1alpha1",
2738        "kind": "OperationDocument",
2739        "operation": "download",
2740        "manifestKey": manifest_key,
2741        "bucket": bucket,
2742        "prefix": prefix,
2743        "endpoint": endpoint,
2744        "region": region,
2745        "forcePathStyle": force_path_style,
2746        "insecure": insecure,
2747        "caBundlePath": ca_bundle_path,
2748        "expectedBackupId": expected_backup_id,
2749        "expectedKanidmUid": restore.spec.target_ref.uid,
2750        "expectedDomain": target.spec.domain,
2751        "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2752        "resultPath": "/run/kaniop-result/result.json",
2753        "maxRetries": 3,
2754        "encryptionMode": enc_mode,
2755        "encryptionKeyId": enc_key_id,
2756    })
2757    .to_string()
2758}
2759
2760async fn ensure_operation_configmap(
2761    restore: &KanidmRestore,
2762    name: &str,
2763    data: &str,
2764    ns: &str,
2765    ctx: &RestoreContext,
2766) -> Result<()> {
2767    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2768    if api
2769        .get_opt(name)
2770        .await
2771        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2772        .is_some()
2773    {
2774        return Ok(());
2775    }
2776    let cm = ConfigMap {
2777        metadata: ObjectMeta {
2778            name: Some(name.to_string()),
2779            namespace: Some(ns.to_string()),
2780            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2781            ..Default::default()
2782        },
2783        data: Some(std::collections::BTreeMap::from([(
2784            "operation.json".to_string(),
2785            data.to_string(),
2786        )])),
2787        ..Default::default()
2788    };
2789    api.create(&PostParams::default(), &cm)
2790        .await
2791        .map(|_| ())
2792        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2793}
2794
2795#[cfg(test)]
2796mod tests {
2797    use super::{
2798        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2799        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2800        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2801        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2802        hardened_security_context, has_accepted_condition, is_remote_source,
2803        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2804        validate_safety_backup_config, validate_source,
2805    };
2806    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2807    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2808    use kube::api::ObjectMeta as ApiObjectMeta;
2809    use std::collections::BTreeMap;
2810
2811    fn make_restore(
2812        source: KanidmRestoreSource,
2813        safety: Option<SafetyBackupConfig>,
2814    ) -> KanidmRestore {
2815        KanidmRestore {
2816            metadata: ApiObjectMeta {
2817                name: Some("test-restore".to_string()),
2818                namespace: Some("default".to_string()),
2819                ..Default::default()
2820            },
2821            spec: KanidmRestoreSpec {
2822                target_ref: KanidmRestoreTargetRef {
2823                    name: "corp-idm".to_string(),
2824                    uid: "test-uid-123".to_string(),
2825                },
2826                source,
2827                restore_image: "kanidm/server@sha256:abc".to_string(),
2828                safety_backup: safety,
2829            },
2830            status: None,
2831        }
2832    }
2833
2834    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2835        super::super::crd::Kanidm {
2836            status: Some(super::super::crd::KanidmStatus {
2837                available_replicas: 1,
2838                replicas: 1,
2839                unavailable_replicas: 0,
2840                updated_replicas: 1,
2841                replica_statuses: vec![],
2842                replica_column: "1/1".to_string(),
2843                secret_name: None,
2844                version: Some(super::super::crd::KanidmVersionStatus {
2845                    image_tag: image_tag.to_string(),
2846                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2847                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2848                }),
2849                domain_appearance_image: None,
2850                mail_sender: None,
2851                conditions: None,
2852            }),
2853            ..Default::default()
2854        }
2855    }
2856
2857    #[test]
2858    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2859        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2860    }
2861
2862    #[test]
2863    fn completed_restore_sets_ready_condition() {
2864        let mut status = KanidmRestoreStatus {
2865            phase: KanidmRestorePhase::Completed,
2866            ..Default::default()
2867        };
2868        super::update_restore_conditions(&mut status, Some(7));
2869        assert!(status.conditions.iter().any(|condition| {
2870            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2871        }));
2872        assert!(!status.conditions.iter().any(|condition| {
2873            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2874        }));
2875    }
2876
2877    #[test]
2878    fn failed_restore_sets_failed_condition() {
2879        let mut status = KanidmRestoreStatus {
2880            phase: KanidmRestorePhase::Failed,
2881            message: Some("verification failed".to_string()),
2882            ..Default::default()
2883        };
2884        super::update_restore_conditions(&mut status, Some(8));
2885        assert!(status.conditions.iter().any(|condition| {
2886            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2887        }));
2888    }
2889
2890    #[test]
2891    fn restore_filename_is_confined_to_backup_directory() {
2892        assert!(safe_basename("backup.json.gz"));
2893        assert!(!safe_basename("../backup.json.gz"));
2894        assert!(!safe_basename("a/b"));
2895        assert!(!safe_basename(""));
2896    }
2897
2898    #[test]
2899    fn restore_image_rejects_latest_and_unpinned_names() {
2900        assert!(mutable_image("kanidm/server:latest"));
2901        assert!(mutable_image("kanidm/server"));
2902        assert!(!mutable_image("kanidm/server:1.10.0"));
2903        assert!(!mutable_image("kanidm/server@sha256:abc"));
2904    }
2905
2906    #[test]
2907    fn source_validation_rejects_both_local_and_backup_ref() {
2908        let source = KanidmRestoreSource {
2909            local: Some(super::KanidmRestoreLocalSource {
2910                file_name: "backup.json".to_string(),
2911            }),
2912            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2913                name: "some-backup".to_string(),
2914            }),
2915        };
2916        let restore = make_restore(source, None);
2917        assert!(validate_source(&restore).is_err());
2918    }
2919
2920    #[test]
2921    fn source_validation_rejects_empty_source() {
2922        let source = KanidmRestoreSource {
2923            local: None,
2924            backup_ref: None,
2925        };
2926        let restore = make_restore(source, None);
2927        assert!(validate_source(&restore).is_err());
2928    }
2929
2930    #[test]
2931    fn source_validation_accepts_local_source() {
2932        let source = KanidmRestoreSource {
2933            local: Some(super::KanidmRestoreLocalSource {
2934                file_name: "backup.json".to_string(),
2935            }),
2936            backup_ref: None,
2937        };
2938        let restore = make_restore(source, None);
2939        assert!(validate_source(&restore).is_ok());
2940    }
2941
2942    #[test]
2943    fn source_validation_accepts_backup_ref_source() {
2944        let source = KanidmRestoreSource {
2945            local: None,
2946            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2947                name: "corp-idm-019c7c76".to_string(),
2948            }),
2949        };
2950        let restore = make_restore(source, None);
2951        assert!(validate_source(&restore).is_ok());
2952    }
2953
2954    #[test]
2955    fn source_validation_rejects_empty_backup_ref_name() {
2956        let source = KanidmRestoreSource {
2957            local: None,
2958            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2959                name: "".to_string(),
2960            }),
2961        };
2962        let restore = make_restore(source, None);
2963        assert!(validate_source(&restore).is_err());
2964    }
2965
2966    #[test]
2967    fn is_remote_source_true_for_backup_ref() {
2968        let source = KanidmRestoreSource {
2969            local: None,
2970            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2971                name: "backup-1".to_string(),
2972            }),
2973        };
2974        let restore = make_restore(source, None);
2975        assert!(is_remote_source(&restore));
2976    }
2977
2978    #[test]
2979    fn is_remote_source_false_for_local() {
2980        let source = KanidmRestoreSource {
2981            local: Some(super::KanidmRestoreLocalSource {
2982                file_name: "backup.json".to_string(),
2983            }),
2984            backup_ref: None,
2985        };
2986        let restore = make_restore(source, None);
2987        assert!(!is_remote_source(&restore));
2988    }
2989
2990    #[test]
2991    fn requires_safety_backup_default_true() {
2992        let source = KanidmRestoreSource {
2993            local: None,
2994            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2995                name: "backup-1".to_string(),
2996            }),
2997        };
2998        let restore = make_restore(source, None);
2999        assert!(requires_safety_backup(&restore));
3000    }
3001
3002    #[test]
3003    fn requires_safety_backup_false_when_skip() {
3004        let source = KanidmRestoreSource {
3005            local: None,
3006            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3007                name: "backup-1".to_string(),
3008            }),
3009        };
3010        let safety = SafetyBackupConfig {
3011            repository_ref: Some(SafetyBackupRepositoryRef {
3012                name: "offsite".to_string(),
3013            }),
3014            skip: true,
3015        };
3016        let restore = make_restore(source, Some(safety));
3017        assert!(!requires_safety_backup(&restore));
3018    }
3019
3020    #[test]
3021    fn break_glass_validation_rejects_skip_without_reason() {
3022        let source = KanidmRestoreSource {
3023            local: None,
3024            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3025                name: "backup-1".to_string(),
3026            }),
3027        };
3028        let safety = SafetyBackupConfig {
3029            repository_ref: Some(SafetyBackupRepositoryRef {
3030                name: "offsite".to_string(),
3031            }),
3032            skip: true,
3033        };
3034        let mut restore = make_restore(source, Some(safety));
3035        restore.metadata.annotations = Some(BTreeMap::from([(
3036            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3037            "commander@example.com".to_string(),
3038        )]));
3039        assert!(validate_safety_backup_config(&restore).is_err());
3040    }
3041
3042    #[test]
3043    fn break_glass_validation_rejects_skip_without_approver() {
3044        let source = KanidmRestoreSource {
3045            local: None,
3046            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3047                name: "backup-1".to_string(),
3048            }),
3049        };
3050        let safety = SafetyBackupConfig {
3051            repository_ref: Some(SafetyBackupRepositoryRef {
3052                name: "offsite".to_string(),
3053            }),
3054            skip: true,
3055        };
3056        let mut restore = make_restore(source, Some(safety));
3057        restore.metadata.annotations = Some(BTreeMap::from([(
3058            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3059            "PVC is unreadable".to_string(),
3060        )]));
3061        assert!(validate_safety_backup_config(&restore).is_err());
3062    }
3063
3064    #[test]
3065    fn break_glass_validation_accepts_valid_annotations() {
3066        let source = KanidmRestoreSource {
3067            local: None,
3068            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3069                name: "backup-1".to_string(),
3070            }),
3071        };
3072        let safety = SafetyBackupConfig {
3073            repository_ref: Some(SafetyBackupRepositoryRef {
3074                name: "offsite".to_string(),
3075            }),
3076            skip: true,
3077        };
3078        let mut restore = make_restore(source, Some(safety));
3079        restore.metadata.annotations = Some(BTreeMap::from([
3080            (
3081                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3082                "PVC is unreadable".to_string(),
3083            ),
3084            (
3085                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3086                "commander@example.com".to_string(),
3087            ),
3088        ]));
3089        assert!(validate_safety_backup_config(&restore).is_ok());
3090    }
3091
3092    #[test]
3093    fn remote_source_requires_repository_ref_when_not_skipping() {
3094        let source = KanidmRestoreSource {
3095            local: None,
3096            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3097                name: "backup-1".to_string(),
3098            }),
3099        };
3100        let safety = SafetyBackupConfig {
3101            repository_ref: None,
3102            skip: false,
3103        };
3104        let restore = make_restore(source, Some(safety));
3105        assert!(validate_safety_backup_config(&restore).is_err());
3106    }
3107
3108    #[test]
3109    fn remote_source_accepts_repository_ref() {
3110        let source = KanidmRestoreSource {
3111            local: None,
3112            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3113                name: "backup-1".to_string(),
3114            }),
3115        };
3116        let safety = SafetyBackupConfig {
3117            repository_ref: Some(SafetyBackupRepositoryRef {
3118                name: "offsite".to_string(),
3119            }),
3120            skip: false,
3121        };
3122        let restore = make_restore(source, Some(safety));
3123        assert!(validate_safety_backup_config(&restore).is_ok());
3124    }
3125
3126    #[test]
3127    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3128        let phases = vec![
3129            KanidmRestorePhase::Pending,
3130            KanidmRestorePhase::Validating,
3131            KanidmRestorePhase::Quiescing,
3132            KanidmRestorePhase::SafetyBackup,
3133            KanidmRestorePhase::PreparingSource,
3134            KanidmRestorePhase::RestoringPrimary,
3135            KanidmRestorePhase::Verifying,
3136            KanidmRestorePhase::RebuildingReplicas,
3137            KanidmRestorePhase::Resuming,
3138            KanidmRestorePhase::Completed,
3139        ];
3140        assert_eq!(phases.len(), 10);
3141        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3142        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3143    }
3144
3145    #[test]
3146    fn status_default_has_empty_phase_timestamps() {
3147        let status = KanidmRestoreStatus::default();
3148        assert!(status.phase_timestamps.is_empty());
3149    }
3150
3151    #[test]
3152    fn status_default_has_empty_original_replicas() {
3153        let status = KanidmRestoreStatus::default();
3154        assert!(status.original_replicas.is_empty());
3155    }
3156
3157    #[test]
3158    fn hardened_security_context_drops_all_capabilities() {
3159        let ctx = hardened_security_context();
3160        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3161        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3162        assert_eq!(ctx.run_as_non_root, Some(true));
3163        let caps = ctx.capabilities.unwrap();
3164        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3165        assert_eq!(
3166            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3167            Some("RuntimeDefault")
3168        );
3169    }
3170
3171    #[test]
3172    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3173        let ctx = kanidm_job_security_context();
3174        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3175        assert!(ctx.run_as_non_root.is_none());
3176        assert!(ctx.read_only_root_filesystem.is_none());
3177        let caps = ctx.capabilities.unwrap();
3178        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3179        assert_eq!(
3180            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3181            Some("RuntimeDefault")
3182        );
3183    }
3184
3185    #[test]
3186    fn safety_backup_job_name_is_deterministic() {
3187        let source = KanidmRestoreSource {
3188            local: Some(super::KanidmRestoreLocalSource {
3189                file_name: "backup.json".to_string(),
3190            }),
3191            backup_ref: None,
3192        };
3193        let restore = make_restore(source, None);
3194        let name1 = super::safety_backup_job_name(&restore);
3195        let name2 = super::safety_backup_job_name(&restore);
3196        assert_eq!(name1, name2);
3197        assert_eq!(name1, "test-restore-safety-backup");
3198    }
3199
3200    #[test]
3201    fn source_prep_job_name_is_deterministic() {
3202        let source = KanidmRestoreSource {
3203            local: None,
3204            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3205                name: "backup-1".to_string(),
3206            }),
3207        };
3208        let restore = make_restore(source, None);
3209        let name = super::source_prep_job_name(&restore);
3210        assert_eq!(name, "test-restore-source-prep");
3211    }
3212
3213    #[test]
3214    fn break_glass_condition_is_recorded_in_status() {
3215        let mut status = KanidmRestoreStatus::default();
3216        let condition = super::restore_condition(
3217            &status.conditions,
3218            CONDITION_BREAK_GLASS,
3219            CONDITION_TRUE,
3220            "BreakGlassOverride",
3221            "Safety backup skipped",
3222            Some(1),
3223        );
3224        status.conditions.push(condition);
3225        assert!(status.conditions.iter().any(|c| {
3226            c.type_ == CONDITION_BREAK_GLASS
3227                && c.status == CONDITION_TRUE
3228                && c.reason == "BreakGlassOverride"
3229        }));
3230    }
3231
3232    #[test]
3233    fn replica_count_entry_serialization() {
3234        let entry = ReplicaCountEntry {
3235            group: "primary".to_string(),
3236            replicas: 3,
3237        };
3238        let json = serde_json::to_string(&entry).unwrap();
3239        assert!(json.contains("\"group\":\"primary\""));
3240        assert!(json.contains("\"replicas\":3"));
3241    }
3242
3243    #[test]
3244    fn local_source_does_not_require_repository_ref() {
3245        let source = KanidmRestoreSource {
3246            local: Some(super::KanidmRestoreLocalSource {
3247                file_name: "backup.json".to_string(),
3248            }),
3249            backup_ref: None,
3250        };
3251        let safety = SafetyBackupConfig {
3252            repository_ref: None,
3253            skip: false,
3254        };
3255        let restore = make_restore(source, Some(safety));
3256        assert!(validate_safety_backup_config(&restore).is_ok());
3257    }
3258
3259    #[test]
3260    fn skip_without_break_glass_annotations_is_rejected() {
3261        let source = KanidmRestoreSource {
3262            local: Some(super::KanidmRestoreLocalSource {
3263                file_name: "backup.json".to_string(),
3264            }),
3265            backup_ref: None,
3266        };
3267        let safety = SafetyBackupConfig {
3268            repository_ref: None,
3269            skip: true,
3270        };
3271        let restore = make_restore(source, Some(safety));
3272        assert!(validate_safety_backup_config(&restore).is_err());
3273    }
3274
3275    #[test]
3276    fn has_accepted_condition_true() {
3277        let conditions = vec![Condition {
3278            type_: "Ready".to_string(),
3279            status: "True".to_string(),
3280            reason: "Accepted".to_string(),
3281            message: String::new(),
3282            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3283                k8s_openapi::jiff::Timestamp::now(),
3284            ),
3285            observed_generation: None,
3286        }];
3287        assert!(has_accepted_condition(&conditions));
3288    }
3289
3290    #[test]
3291    fn has_accepted_condition_false_when_not_accepted_reason() {
3292        let conditions = vec![Condition {
3293            type_: "Ready".to_string(),
3294            status: "True".to_string(),
3295            reason: "SomeOtherReason".to_string(),
3296            message: String::new(),
3297            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3298                k8s_openapi::jiff::Timestamp::now(),
3299            ),
3300            observed_generation: None,
3301        }];
3302        assert!(!has_accepted_condition(&conditions));
3303    }
3304
3305    #[test]
3306    fn has_accepted_condition_false_when_status_false() {
3307        let conditions = vec![Condition {
3308            type_: "Ready".to_string(),
3309            status: "False".to_string(),
3310            reason: "Accepted".to_string(),
3311            message: String::new(),
3312            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3313                k8s_openapi::jiff::Timestamp::now(),
3314            ),
3315            observed_generation: None,
3316        }];
3317        assert!(!has_accepted_condition(&conditions));
3318    }
3319
3320    #[test]
3321    fn has_accepted_condition_empty() {
3322        assert!(!has_accepted_condition(&[]));
3323    }
3324
3325    #[test]
3326    fn safety_backup_upload_operation_doc_is_valid() {
3327        let source = KanidmRestoreSource {
3328            local: None,
3329            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3330                name: "backup-1".to_string(),
3331            }),
3332        };
3333        let mut restore = make_restore(source, None);
3334        restore.metadata.namespace = Some("test-ns".to_string());
3335        let target = super::super::crd::Kanidm::default();
3336        let doc_str = super::build_safety_upload_operation_doc(
3337            &restore,
3338            &target,
3339            "test-backup-id",
3340            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3341            "my-bucket",
3342            "prod",
3343            "https://s3.example.com",
3344            "us-east-1",
3345            false,
3346            false,
3347            None,
3348            None,
3349        )
3350        .unwrap();
3351        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3352        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3353        assert_eq!(parsed["kind"], "OperationDocument");
3354        assert_eq!(parsed["operation"], "upload");
3355        assert_eq!(parsed["bucket"], "my-bucket");
3356        assert_eq!(parsed["prefix"], "prod");
3357        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3358        assert_eq!(parsed["region"], "us-east-1");
3359        assert_eq!(parsed["backupId"], "test-backup-id");
3360        assert_eq!(parsed["consistency"], "kanidm-offline");
3361        assert_eq!(parsed["reason"], "restore-safety");
3362        assert_eq!(
3363            parsed["payloadPath"],
3364            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3365        );
3366        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3367    }
3368
3369    #[test]
3370    fn download_operation_doc_uses_real_repository_data() {
3371        let source = KanidmRestoreSource {
3372            local: None,
3373            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3374                name: "backup-1".to_string(),
3375            }),
3376        };
3377        let restore = make_restore(source, None);
3378        let target = super::super::crd::Kanidm::default();
3379        let doc_str = super::build_download_operation_doc(
3380            &restore,
3381            &target,
3382            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3383            "019c7c76-f423-7a12-8f41-2bea7588a303",
3384            "real-bucket",
3385            "real-prefix",
3386            "https://real-endpoint.com",
3387            "eu-west-1",
3388            true,
3389            false,
3390            None,
3391            None,
3392        );
3393        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3394        assert_eq!(parsed["bucket"], "real-bucket");
3395        assert_eq!(parsed["prefix"], "real-prefix");
3396        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3397        assert_eq!(parsed["region"], "eu-west-1");
3398        assert_eq!(parsed["forcePathStyle"], true);
3399        assert_eq!(
3400            parsed["manifestKey"],
3401            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3402        );
3403        assert_eq!(
3404            parsed["expectedBackupId"],
3405            "019c7c76-f423-7a12-8f41-2bea7588a303"
3406        );
3407    }
3408
3409    #[test]
3410    fn safety_backup_ref_is_set_after_verification() {
3411        let status = KanidmRestoreStatus {
3412            phase: KanidmRestorePhase::SafetyBackup,
3413            safety_backup_job_name: Some("test-safety-backup".to_string()),
3414            safety_backup_ref: Some("safety-test-restore".to_string()),
3415            safety_backup_expected_backup_id: Some(
3416                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3417            ),
3418            safety_backup_manifest_key: Some(
3419                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3420            ),
3421            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3422            ..Default::default()
3423        };
3424        assert!(status.safety_backup_ref.is_some());
3425        assert!(
3426            status
3427                .safety_backup_ref
3428                .as_ref()
3429                .unwrap()
3430                .starts_with("safety-")
3431        );
3432        assert!(status.safety_backup_expected_backup_id.is_some());
3433        assert!(status.safety_backup_manifest_key.is_some());
3434        assert!(status.safety_backup_payload_sha256.is_some());
3435    }
3436
3437    #[test]
3438    fn database_mutation_not_started_without_safety_backup_ref() {
3439        let status = KanidmRestoreStatus {
3440            phase: KanidmRestorePhase::PreparingSource,
3441            safety_backup_ref: None,
3442            database_mutation_started: false,
3443            ..Default::default()
3444        };
3445        assert!(!status.database_mutation_started);
3446        assert!(status.safety_backup_ref.is_none());
3447    }
3448
3449    #[test]
3450    fn pre_boundary_failure_does_not_set_mutation_started() {
3451        let status = KanidmRestoreStatus {
3452            phase: KanidmRestorePhase::Failed,
3453            message: Some(
3454                "safety backup job failed; target restored to original state".to_string(),
3455            ),
3456            database_mutation_started: false,
3457            ..Default::default()
3458        };
3459        assert!(!status.database_mutation_started);
3460    }
3461
3462    #[test]
3463    fn shared_volume_constants_are_distinct() {
3464        assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3465        assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3466    }
3467
3468    #[test]
3469    fn validate_backup_compatibility_matching_versions_passes() {
3470        let source = KanidmRestoreSource {
3471            local: None,
3472            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3473                name: "backup-1".to_string(),
3474            }),
3475        };
3476        let restore = make_restore(source, None);
3477        let target = kanidm_with_version("1.10.4");
3478        let backup = kaniop_backup_core::crd::KanidmBackup {
3479            metadata: kube::api::ObjectMeta::default(),
3480            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3481                backup_id: "id".to_string(),
3482                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3483                    name: "k".to_string(),
3484                    uid: "uid".to_string(),
3485                },
3486                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3487                    name: "repo".to_string(),
3488                },
3489                manifest_key: "key".to_string(),
3490            },
3491            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3492                kanidm_version: Some("1.10.4".to_string()),
3493                ..Default::default()
3494            }),
3495        };
3496        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3497    }
3498
3499    #[test]
3500    fn validate_backup_compatibility_version_mismatch_fails() {
3501        let source = KanidmRestoreSource {
3502            local: None,
3503            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3504                name: "backup-1".to_string(),
3505            }),
3506        };
3507        let restore = make_restore(source, None);
3508        let target = kanidm_with_version("1.11.0");
3509        let backup = kaniop_backup_core::crd::KanidmBackup {
3510            metadata: kube::api::ObjectMeta::default(),
3511            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3512                backup_id: "id".to_string(),
3513                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3514                    name: "k".to_string(),
3515                    uid: "uid".to_string(),
3516                },
3517                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3518                    name: "repo".to_string(),
3519                },
3520                manifest_key: "key".to_string(),
3521            },
3522            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3523                kanidm_version: Some("1.10.4".to_string()),
3524                ..Default::default()
3525            }),
3526        };
3527        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3528    }
3529
3530    #[test]
3531    fn validate_backup_compatibility_no_backup_version_passes() {
3532        let source = KanidmRestoreSource {
3533            local: None,
3534            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3535                name: "backup-1".to_string(),
3536            }),
3537        };
3538        let restore = make_restore(source, None);
3539        let target = super::super::crd::Kanidm::default();
3540        let backup = kaniop_backup_core::crd::KanidmBackup {
3541            metadata: kube::api::ObjectMeta::default(),
3542            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3543                backup_id: "id".to_string(),
3544                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3545                    name: "k".to_string(),
3546                    uid: "uid".to_string(),
3547                },
3548                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3549                    name: "repo".to_string(),
3550                },
3551                manifest_key: "key".to_string(),
3552            },
3553            status: None,
3554        };
3555        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3556    }
3557
3558    #[test]
3559    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3560        let source = KanidmRestoreSource {
3561            local: None,
3562            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3563                name: "backup-1".to_string(),
3564            }),
3565        };
3566        let mut restore = make_restore(source, None);
3567        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3568        let target = super::super::crd::Kanidm::default();
3569        let backup = kaniop_backup_core::crd::KanidmBackup {
3570            metadata: kube::api::ObjectMeta::default(),
3571            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3572                backup_id: "id".to_string(),
3573                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3574                    name: "k".to_string(),
3575                    uid: "uid".to_string(),
3576                },
3577                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3578                    name: "repo".to_string(),
3579                },
3580                manifest_key: "key".to_string(),
3581            },
3582            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3583                image_digest: Some("sha256:originaldigest".to_string()),
3584                ..Default::default()
3585            }),
3586        };
3587        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3588    }
3589
3590    #[test]
3591    fn validate_backup_compatibility_image_digest_match_passes() {
3592        let source = KanidmRestoreSource {
3593            local: None,
3594            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3595                name: "backup-1".to_string(),
3596            }),
3597        };
3598        let mut restore = make_restore(source, None);
3599        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3600        let target = super::super::crd::Kanidm::default();
3601        let backup = kaniop_backup_core::crd::KanidmBackup {
3602            metadata: kube::api::ObjectMeta::default(),
3603            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3604                backup_id: "id".to_string(),
3605                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3606                    name: "k".to_string(),
3607                    uid: "uid".to_string(),
3608                },
3609                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3610                    name: "repo".to_string(),
3611                },
3612                manifest_key: "key".to_string(),
3613            },
3614            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3615                image_digest: Some("sha256:abc123".to_string()),
3616                ..Default::default()
3617            }),
3618        };
3619        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3620    }
3621
3622    #[test]
3623    fn safety_backup_id_is_deterministic() {
3624        let source = KanidmRestoreSource {
3625            local: None,
3626            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3627                name: "backup-1".to_string(),
3628            }),
3629        };
3630        let restore = make_restore(source, None);
3631        let id1 = super::compute_safety_backup_id(&restore);
3632        let id2 = super::compute_safety_backup_id(&restore);
3633        assert_eq!(id1, id2);
3634        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3635    }
3636
3637    #[test]
3638    fn safety_backup_id_differs_for_different_restores() {
3639        let source1 = KanidmRestoreSource {
3640            local: None,
3641            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3642                name: "backup-1".to_string(),
3643            }),
3644        };
3645        let restore1 = make_restore(source1, None);
3646        let mut restore2 = make_restore(
3647            KanidmRestoreSource {
3648                local: None,
3649                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3650                    name: "backup-2".to_string(),
3651                }),
3652            },
3653            None,
3654        );
3655        restore2.metadata.name = Some("different-restore".to_string());
3656        let id1 = super::compute_safety_backup_id(&restore1);
3657        let id2 = super::compute_safety_backup_id(&restore2);
3658        assert_ne!(id1, id2);
3659    }
3660
3661    #[test]
3662    fn safety_manifest_key_is_deterministic() {
3663        let source = KanidmRestoreSource {
3664            local: None,
3665            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3666                name: "backup-1".to_string(),
3667            }),
3668        };
3669        let restore = make_restore(source, None);
3670        let key1 = super::compute_safety_manifest_key(&restore);
3671        let key2 = super::compute_safety_manifest_key(&restore);
3672        assert_eq!(key1, key2);
3673        assert!(key1.contains("manifest.json"));
3674        assert!(key1.starts_with("v1/tenants/"));
3675    }
3676
3677    #[test]
3678    fn safety_backup_id_survives_restart() {
3679        let source = KanidmRestoreSource {
3680            local: None,
3681            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3682                name: "backup-1".to_string(),
3683            }),
3684        };
3685        let mut restore = make_restore(source, None);
3686        let original_id = super::compute_safety_backup_id(&restore);
3687        restore.status = Some(KanidmRestoreStatus {
3688            phase: KanidmRestorePhase::SafetyBackup,
3689            safety_backup_expected_backup_id: Some(original_id.clone()),
3690            ..Default::default()
3691        });
3692        let recomputed_id = super::compute_safety_backup_id(&restore);
3693        assert_eq!(original_id, recomputed_id);
3694    }
3695
3696    #[test]
3697    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3698        let source = KanidmRestoreSource {
3699            local: None,
3700            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3701                name: "backup-1".to_string(),
3702            }),
3703        };
3704        let restore = make_restore(source, None);
3705        assert!(super::requires_safety_backup(&restore));
3706        let status_without_ref = KanidmRestoreStatus {
3707            phase: KanidmRestorePhase::RestoringPrimary,
3708            safety_backup_ref: None,
3709            database_mutation_started: false,
3710            ..Default::default()
3711        };
3712        assert!(status_without_ref.safety_backup_ref.is_none());
3713        assert!(!status_without_ref.database_mutation_started);
3714    }
3715
3716    #[test]
3717    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3718        let status_with_ref = KanidmRestoreStatus {
3719            phase: KanidmRestorePhase::RestoringPrimary,
3720            safety_backup_ref: Some("safety-test-restore".to_string()),
3721            safety_backup_manifest_key: Some(
3722                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3723            ),
3724            safety_backup_payload_sha256: Some("abc123".to_string()),
3725            database_mutation_started: false,
3726            ..Default::default()
3727        };
3728        assert!(status_with_ref.safety_backup_ref.is_some());
3729        assert!(!status_with_ref.database_mutation_started);
3730    }
3731
3732    #[test]
3733    fn result_document_verification_rejects_missing_backup_id() {
3734        let result_json = r#"{
3735            "apiVersion": "backup.kaniop.rs/v1alpha1",
3736            "kind": "ResultDocument",
3737            "operation": "upload",
3738            "success": true,
3739            "exitCode": "success",
3740            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3741            "payloadSha256": "abc123"
3742        }"#;
3743        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3744        assert!(result.backup_id.is_none());
3745    }
3746
3747    #[test]
3748    fn result_document_verification_rejects_missing_manifest_key() {
3749        let result_json = r#"{
3750            "apiVersion": "backup.kaniop.rs/v1alpha1",
3751            "kind": "ResultDocument",
3752            "operation": "upload",
3753            "success": true,
3754            "exitCode": "success",
3755            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3756            "payloadSha256": "abc123"
3757        }"#;
3758        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3759        assert!(result.manifest_key.is_none());
3760    }
3761
3762    #[test]
3763    fn result_document_verification_rejects_missing_payload_sha256() {
3764        let result_json = r#"{
3765            "apiVersion": "backup.kaniop.rs/v1alpha1",
3766            "kind": "ResultDocument",
3767            "operation": "upload",
3768            "success": true,
3769            "exitCode": "success",
3770            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3771            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3772        }"#;
3773        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3774        assert!(result.payload_sha256.is_none());
3775    }
3776
3777    #[test]
3778    fn result_document_verification_rejects_failure_exit_code() {
3779        let result_json = r#"{
3780            "apiVersion": "backup.kaniop.rs/v1alpha1",
3781            "kind": "ResultDocument",
3782            "operation": "upload",
3783            "success": false,
3784            "exitCode": "integrity",
3785            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3786            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3787            "payloadSha256": "abc123"
3788        }"#;
3789        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3790        assert!(!result.success);
3791        assert_eq!(
3792            result.exit_code,
3793            kaniop_backup_core::result::ExitCode::Integrity
3794        );
3795    }
3796
3797    #[test]
3798    fn result_document_verification_rejects_wrong_operation() {
3799        let result_json = r#"{
3800            "apiVersion": "backup.kaniop.rs/v1alpha1",
3801            "kind": "ResultDocument",
3802            "operation": "download",
3803            "success": true,
3804            "exitCode": "success",
3805            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3806            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3807            "payloadSha256": "abc123"
3808        }"#;
3809        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3810        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3811    }
3812
3813    #[test]
3814    fn result_document_verification_accepts_valid_document() {
3815        let result_json = r#"{
3816            "apiVersion": "backup.kaniop.rs/v1alpha1",
3817            "kind": "ResultDocument",
3818            "operation": "upload",
3819            "success": true,
3820            "exitCode": "success",
3821            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3822            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3823            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3824        }"#;
3825        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3826        assert!(result.success);
3827        assert_eq!(
3828            result.exit_code,
3829            kaniop_backup_core::result::ExitCode::Success
3830        );
3831        assert_eq!(result.operation, "upload");
3832        assert!(result.backup_id.is_some());
3833        assert!(result.manifest_key.is_some());
3834        assert!(result.payload_sha256.is_some());
3835    }
3836
3837    #[test]
3838    fn termination_message_path_constant_is_valid() {
3839        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3840        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3841    }
3842
3843    #[test]
3844    fn status_tracks_safety_backup_evidence() {
3845        let status = KanidmRestoreStatus {
3846            phase: KanidmRestorePhase::PreparingSource,
3847            safety_backup_ref: Some("safety-test-restore".to_string()),
3848            safety_backup_expected_backup_id: Some(
3849                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3850            ),
3851            safety_backup_manifest_key: Some(
3852                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3853            ),
3854            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3855            ..Default::default()
3856        };
3857        assert!(status.safety_backup_ref.is_some());
3858        assert!(status.safety_backup_expected_backup_id.is_some());
3859        assert!(status.safety_backup_manifest_key.is_some());
3860        assert!(status.safety_backup_payload_sha256.is_some());
3861    }
3862
3863    #[test]
3864    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3865        let invalid_json = "not valid json";
3866        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3867        assert!(result.is_err());
3868    }
3869
3870    #[test]
3871    fn fail_closed_semantics_preserved_on_oversized_result() {
3872        let oversized_json = format!(
3873            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3874            kaniop_backup_core::result::RESULT_DOC_VERSION,
3875            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3876        );
3877        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3878        assert!(result.is_err());
3879    }
3880
3881    #[test]
3882    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3883        let source = KanidmRestoreSource {
3884            local: None,
3885            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3886                name: "backup-1".to_string(),
3887            }),
3888        };
3889        let mut restore = make_restore(source, None);
3890        restore.metadata.namespace = Some("test-ns".to_string());
3891        let target = super::super::crd::Kanidm::default();
3892        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3893        let doc_str = super::build_safety_upload_operation_doc(
3894            &restore,
3895            &target,
3896            "test-backup-id",
3897            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3898            "my-bucket",
3899            "prod",
3900            "https://s3.example.com",
3901            "us-east-1",
3902            false,
3903            false,
3904            Some(&ca_path),
3905            None,
3906        )
3907        .unwrap();
3908        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3909        assert_eq!(parsed["caBundlePath"], ca_path);
3910    }
3911
3912    #[test]
3913    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3914        let source = KanidmRestoreSource {
3915            local: None,
3916            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3917                name: "backup-1".to_string(),
3918            }),
3919        };
3920        let mut restore = make_restore(source, None);
3921        restore.metadata.namespace = Some("test-ns".to_string());
3922        let target = super::super::crd::Kanidm::default();
3923        let doc_str = super::build_safety_upload_operation_doc(
3924            &restore,
3925            &target,
3926            "test-backup-id",
3927            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3928            "my-bucket",
3929            "prod",
3930            "https://s3.example.com",
3931            "us-east-1",
3932            false,
3933            false,
3934            None,
3935            None,
3936        )
3937        .unwrap();
3938        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3939        assert!(parsed["caBundlePath"].is_null());
3940    }
3941
3942    #[test]
3943    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3944        let source = KanidmRestoreSource {
3945            local: None,
3946            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3947                name: "backup-1".to_string(),
3948            }),
3949        };
3950        let restore = make_restore(source, None);
3951        let target = super::super::crd::Kanidm::default();
3952        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3953        let doc_str = super::build_download_operation_doc(
3954            &restore,
3955            &target,
3956            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3957            "019c7c76-f423-7a12-8f41-2bea7588a303",
3958            "real-bucket",
3959            "real-prefix",
3960            "https://real-endpoint.com",
3961            "eu-west-1",
3962            true,
3963            false,
3964            Some(&ca_path),
3965            None,
3966        );
3967        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3968        assert_eq!(parsed["caBundlePath"], ca_path);
3969    }
3970
3971    #[test]
3972    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3973        let source = KanidmRestoreSource {
3974            local: None,
3975            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3976                name: "backup-1".to_string(),
3977            }),
3978        };
3979        let restore = make_restore(source, None);
3980        let target = super::super::crd::Kanidm::default();
3981        let doc_str = super::build_download_operation_doc(
3982            &restore,
3983            &target,
3984            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3985            "019c7c76-f423-7a12-8f41-2bea7588a303",
3986            "real-bucket",
3987            "real-prefix",
3988            "https://real-endpoint.com",
3989            "eu-west-1",
3990            true,
3991            false,
3992            None,
3993            None,
3994        );
3995        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3996        assert!(parsed["caBundlePath"].is_null());
3997    }
3998
3999    #[test]
4000    fn source_prep_result_operation_is_download() {
4001        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4002        assert_ne!(
4003            super::SOURCE_PREP_RESULT_OPERATION,
4004            super::SAFETY_BACKUP_RESULT_OPERATION
4005        );
4006    }
4007
4008    #[test]
4009    fn download_result_document_verification_accepts_valid_document() {
4010        let result_json = r#"{
4011            "apiVersion": "backup.kaniop.rs/v1alpha1",
4012            "kind": "ResultDocument",
4013            "operation": "download",
4014            "success": true,
4015            "exitCode": "success",
4016            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4019        }"#;
4020        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021        assert!(result.success);
4022        assert_eq!(
4023            result.exit_code,
4024            kaniop_backup_core::result::ExitCode::Success
4025        );
4026        assert_eq!(result.operation, "download");
4027        assert_eq!(
4028            result.backup_id.as_deref(),
4029            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4030        );
4031        assert!(result.manifest_key.is_some());
4032        assert!(result.payload_sha256.is_some());
4033    }
4034
4035    #[test]
4036    fn download_result_document_rejects_wrong_operation() {
4037        let result_json = r#"{
4038            "apiVersion": "backup.kaniop.rs/v1alpha1",
4039            "kind": "ResultDocument",
4040            "operation": "upload",
4041            "success": true,
4042            "exitCode": "success",
4043            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4044            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4045            "payloadSha256": "abc123"
4046        }"#;
4047        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4048        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4049    }
4050
4051    #[test]
4052    fn download_result_document_rejects_failure_success_field() {
4053        let result_json = r#"{
4054            "apiVersion": "backup.kaniop.rs/v1alpha1",
4055            "kind": "ResultDocument",
4056            "operation": "download",
4057            "success": false,
4058            "exitCode": "retryable",
4059            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4060            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4061            "payloadSha256": "abc123"
4062        }"#;
4063        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4064        assert!(!result.success);
4065        assert_ne!(
4066            result.exit_code,
4067            kaniop_backup_core::result::ExitCode::Success
4068        );
4069    }
4070
4071    #[test]
4072    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4073        let status = KanidmRestoreStatus {
4074            phase: KanidmRestorePhase::PreparingSource,
4075            database_mutation_started: false,
4076            ..Default::default()
4077        };
4078        assert!(!status.database_mutation_started);
4079        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4080    }
4081
4082    #[test]
4083    fn backup_job_volume_size_default_is_10gi() {
4084        let qty = crate::controller::backup_job_volume_size();
4085        assert_eq!(qty, Quantity("10Gi".to_string()));
4086    }
4087
4088    #[test]
4089    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4090        use kaniop_backup_core::auth::build_encryption_env_vars;
4091        use kaniop_backup_core::crd::SecretRef;
4092
4093        let key_ref = SecretRef {
4094            name: "kek-secret".to_string(),
4095        };
4096        let env_vars = build_encryption_env_vars(Some(&key_ref));
4097        assert_eq!(env_vars.len(), 1);
4098        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4099        let secret_ref = env_vars[0]
4100            .value_from
4101            .as_ref()
4102            .unwrap()
4103            .secret_key_ref
4104            .as_ref()
4105            .unwrap();
4106        assert_eq!(secret_ref.name, "kek-secret");
4107        assert_eq!(secret_ref.key, "encryption-key");
4108    }
4109
4110    #[test]
4111    fn encryption_env_vars_absent_when_no_key_ref() {
4112        use kaniop_backup_core::auth::build_encryption_env_vars;
4113
4114        let env_vars = build_encryption_env_vars(None);
4115        assert!(
4116            env_vars.is_empty(),
4117            "encryption env vars must be empty when key_ref is None"
4118        );
4119    }
4120
4121    #[test]
4122    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4123        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4124
4125        let source = KanidmRestoreSource {
4126            local: None,
4127            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4128                name: "backup-1".to_string(),
4129            }),
4130        };
4131        let mut restore = make_restore(source, None);
4132        restore.metadata.namespace = Some("test-ns".to_string());
4133        let target = super::super::crd::Kanidm::default();
4134        let encryption = RepositoryEncryption {
4135            mode: EncryptionMode::ClientSide,
4136            key_id: None,
4137            key_ref: Some(SecretRef {
4138                name: "kek-secret".to_string(),
4139            }),
4140        };
4141        let doc_str = super::build_safety_upload_operation_doc(
4142            &restore,
4143            &target,
4144            "test-backup-id",
4145            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4146            "my-bucket",
4147            "prod",
4148            "https://s3.example.com",
4149            "us-east-1",
4150            false,
4151            false,
4152            None,
4153            Some(&encryption),
4154        )
4155        .unwrap();
4156        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4157        assert_eq!(parsed["encryptionMode"], "clientSide");
4158        assert!(parsed["encryptionKeyId"].is_null());
4159    }
4160
4161    #[test]
4162    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4163        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4164
4165        let source = KanidmRestoreSource {
4166            local: None,
4167            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4168                name: "backup-1".to_string(),
4169            }),
4170        };
4171        let restore = make_restore(source, None);
4172        let target = super::super::crd::Kanidm::default();
4173        let encryption = RepositoryEncryption {
4174            mode: EncryptionMode::ProviderKms,
4175            key_id: Some("alias/kaniop-backups".to_string()),
4176            key_ref: None,
4177        };
4178        let doc_str = super::build_download_operation_doc(
4179            &restore,
4180            &target,
4181            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4182            "019c7c76-f423-7a12-8f41-2bea7588a303",
4183            "real-bucket",
4184            "real-prefix",
4185            "https://real-endpoint.com",
4186            "eu-west-1",
4187            true,
4188            false,
4189            None,
4190            Some(&encryption),
4191        );
4192        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4193        assert_eq!(parsed["encryptionMode"], "providerKms");
4194        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4195    }
4196}