Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76    feature = "schemars",
77    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80    category = "kaniop",
81    group = "kaniop.rs",
82    version = "v1beta1",
83    kind = "KanidmRestore",
84    plural = "kanidmrestores",
85    singular = "kanidmrestore",
86    shortname = "idmrestore",
87    namespaced,
88    status = "KanidmRestoreStatus",
89    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95    pub target_ref: KanidmRestoreTargetRef,
96    pub source: KanidmRestoreSource,
97    pub restore_image: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106    pub name: String,
107    pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113    feature = "schemars",
114    schemars(extend("x-kubernetes-validations" = [
115        {
116            "message": "exactly one of local or backupRef must be set",
117            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118        }
119    ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123    /// Local file source. Mutually exclusive with `backupRef`.
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub local: Option<KanidmRestoreLocalSource>,
126    /// Remote cataloged backup reference. Mutually exclusive with `local`.
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
136    pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
144    pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub repository_ref: Option<SafetyBackupRepositoryRef>,
154    /// Skip the mandatory safety backup. Requires break-glass annotations.
155    #[serde(default)]
156    pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163    pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169    #[default]
170    Pending,
171    Validating,
172    Quiescing,
173    SafetyBackup,
174    PreparingSource,
175    RestoringPrimary,
176    Verifying,
177    RebuildingReplicas,
178    Resuming,
179    Completed,
180    Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub observed_generation: Option<i64>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_target_uid: Option<String>,
191    #[serde(default)]
192    pub phase: KanidmRestorePhase,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub restore_job_name: Option<String>,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub verify_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub safety_backup_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub source_prep_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub safety_backup_ref: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_expected_backup_id: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_manifest_key: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_payload_sha256: Option<String>,
209    #[serde(default)]
210    pub replicas_cleared: bool,
211    #[serde(default)]
212    pub certificates_cleared: bool,
213    #[serde(default)]
214    pub database_mutation_started: bool,
215    #[serde(default, skip_serializing_if = "Vec::is_empty")]
216    pub original_replicas: Vec<ReplicaCountEntry>,
217    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218    pub phase_timestamps: std::collections::BTreeMap<String, String>,
219    #[serde(default, skip_serializing_if = "Vec::is_empty")]
220    #[cfg_attr(
221        feature = "schemars",
222        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223    )]
224    pub conditions: Vec<Condition>,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233    pub group: String,
234    pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239    attempts: Counter<u64>,
240    outcomes: Counter<u64>,
241    duration_seconds: Histogram<f64>,
242    break_glass_total: Counter<u64>,
243    safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247    fn new() -> Self {
248        let meter = global::meter("kaniop");
249        Self {
250            attempts: meter
251                .u64_counter("kaniop_restore_attempts_total")
252                .with_description("Number of Kanidm restore attempts started")
253                .build(),
254            outcomes: meter
255                .u64_counter("kaniop_restore_outcomes_total")
256                .with_description("Number of terminal Kanidm restore outcomes")
257                .build(),
258            duration_seconds: meter
259                .f64_histogram("kaniop_restore_duration_seconds")
260                .with_description("Kanidm restore duration from object creation to terminal phase")
261                .with_unit("s")
262                .build(),
263            break_glass_total: meter
264                .u64_counter("kaniop_restore_break_glass_total")
265                .with_description("Number of break-glass safety backup overrides")
266                .build(),
267            safety_backup_duration_seconds: meter
268                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269                .with_description("Duration of pre-restore safety backup creation")
270                .with_unit("s")
271                .build(),
272        }
273    }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278    client: Client,
279    recorder: Recorder,
280    metrics: RestoreMetrics,
281}
282
283async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
284    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
285    match phase {
286        KanidmRestorePhase::Pending => {
287            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
288            Ok(Action::requeue(REQUEUE))
289        }
290        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
291            Ok(target) => {
292                mark_restoring(&restore, &target, &ctx).await?;
293                let mut status = restore.status.clone().unwrap_or_default();
294                status.observed_target_uid = target.uid();
295                status.original_replicas = capture_original_replicas(&target);
296                status.phase = KanidmRestorePhase::Quiescing;
297                status.message = None;
298                patch_status(&restore, &ctx, status).await?;
299                Ok(Action::requeue(REQUEUE))
300            }
301            Err(error) => {
302                set_phase(
303                    &restore,
304                    &ctx,
305                    KanidmRestorePhase::Failed,
306                    Some(error.to_string()),
307                )
308                .await?;
309                Ok(Action::requeue(Duration::from_secs(300)))
310            }
311        },
312        KanidmRestorePhase::Quiescing => {
313            let target = get_target(&restore, &ctx).await?;
314            scale_all(&target, &ctx, 0).await?;
315            if target_pods_stopped(&target, &ctx).await?
316                && target_volumes_detached(&target, &ctx).await?
317            {
318                let next_phase = if requires_safety_backup(&restore) {
319                    KanidmRestorePhase::SafetyBackup
320                } else {
321                    record_break_glass(&restore, &ctx).await?;
322                    KanidmRestorePhase::PreparingSource
323                };
324                if next_phase == KanidmRestorePhase::PreparingSource {
325                    let refreshed = Api::<KanidmRestore>::namespaced(
326                        ctx.client.clone(),
327                        restore.namespace().as_deref().unwrap_or_default(),
328                    )
329                    .get(&restore.name_any())
330                    .await
331                    .map_err(|error| {
332                        Error::kube_error(
333                            "get",
334                            "KanidmRestore",
335                            restore.namespace().as_deref().unwrap_or_default(),
336                            restore.name_any(),
337                            error,
338                        )
339                    })?;
340                    set_phase(&refreshed, &ctx, next_phase, None).await?;
341                } else {
342                    set_phase(&restore, &ctx, next_phase, None).await?;
343                }
344            }
345            Ok(Action::requeue(REQUEUE))
346        }
347        KanidmRestorePhase::SafetyBackup => {
348            let target = get_target(&restore, &ctx).await?;
349            let name = safety_backup_job_name(&restore);
350            let expected_backup_id = compute_safety_backup_id(&restore);
351            let expected_manifest_key = compute_safety_manifest_key(&restore);
352            ensure_safety_backup_job(
353                &restore,
354                &target,
355                &ctx,
356                &name,
357                &expected_backup_id,
358                &expected_manifest_key,
359            )
360            .await?;
361            let mut status = restore.status.clone().unwrap_or_default();
362            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
363            match job_state(&restore, &ctx, &name).await? {
364                JobState::Complete => {
365                    let result =
366                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
367                    match result {
368                        Ok(verified) => {
369                            status.safety_backup_job_name = Some(name);
370                            status.safety_backup_ref =
371                                Some(format!("safety-{}", restore.name_any()));
372                            status.safety_backup_manifest_key = Some(verified.manifest_key);
373                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
374                            record_safety_backup_duration(&restore, &ctx).await;
375                            status.phase = KanidmRestorePhase::PreparingSource;
376                            status.message = None;
377                            patch_status(&restore, &ctx, status).await?;
378                        }
379                        Err(error) => {
380                            resume_before_mutation(&restore, &ctx).await?;
381                            set_phase(
382                                &restore,
383                                &ctx,
384                                KanidmRestorePhase::Failed,
385                                Some(format!("safety backup result verification failed: {error}")),
386                            )
387                            .await?;
388                        }
389                    }
390                }
391                JobState::Failed => {
392                    resume_before_mutation(&restore, &ctx).await?;
393                    set_phase(
394                        &restore,
395                        &ctx,
396                        KanidmRestorePhase::Failed,
397                        Some(
398                            "safety backup job failed; target restored to original state"
399                                .to_string(),
400                        ),
401                    )
402                    .await?;
403                }
404                JobState::Running => {
405                    patch_status(&restore, &ctx, status).await?;
406                }
407            }
408            Ok(Action::requeue(REQUEUE))
409        }
410        KanidmRestorePhase::PreparingSource => {
411            if is_remote_source(&restore) {
412                let target = get_target(&restore, &ctx).await?;
413                let name = source_prep_job_name(&restore);
414                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
415                match job_state(&restore, &ctx, &name).await? {
416                    JobState::Complete => {
417                        let backup_id = restore
418                            .spec
419                            .source
420                            .backup_ref
421                            .as_ref()
422                            .map(|b| b.name.clone())
423                            .unwrap_or_default();
424                        let backup = {
425                            let ns = restore.namespace().unwrap();
426                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
427                                .get(&backup_id)
428                                .await
429                                .map_err(|e| {
430                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
431                                })?
432                        };
433                        let expected_backup_id = &backup.spec.backup_id;
434                        let result =
435                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
436                                .await;
437                        match result {
438                            Ok(verified) => {
439                                if let Some(expected_sha256) = backup
440                                    .status
441                                    .as_ref()
442                                    .and_then(|s| s.payload_sha256.as_deref())
443                                {
444                                    if verified.payload_sha256 != expected_sha256 {
445                                        resume_before_mutation(&restore, &ctx).await?;
446                                        set_phase(
447                                            &restore,
448                                            &ctx,
449                                            KanidmRestorePhase::Failed,
450                                            Some(format!(
451                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
452                                                verified.payload_sha256
453                                            )),
454                                        )
455                                        .await?;
456                                        return Ok(Action::requeue(REQUEUE));
457                                    }
458                                }
459                                let mut status = restore.status.clone().unwrap_or_default();
460                                status.source_prep_job_name = Some(name);
461                                status.phase = KanidmRestorePhase::RestoringPrimary;
462                                status.message = None;
463                                patch_status(&restore, &ctx, status).await?;
464                            }
465                            Err(error) => {
466                                resume_before_mutation(&restore, &ctx).await?;
467                                set_phase(
468                                    &restore,
469                                    &ctx,
470                                    KanidmRestorePhase::Failed,
471                                    Some(format!(
472                                        "source preparation result verification failed: {error}"
473                                    )),
474                                )
475                                .await?;
476                            }
477                        }
478                    }
479                    JobState::Failed => {
480                        resume_before_mutation(&restore, &ctx).await?;
481                        set_phase(
482                            &restore,
483                            &ctx,
484                            KanidmRestorePhase::Failed,
485                            Some(
486                                "source preparation job failed; target restored to original state"
487                                    .to_string(),
488                            ),
489                        )
490                        .await?;
491                    }
492                    JobState::Running => {}
493                }
494            } else {
495                let target = get_target(&restore, &ctx).await?;
496                let name = source_check_job_name(&restore);
497                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
498                match job_state(&restore, &ctx, &name).await? {
499                    JobState::Complete => {
500                        let mut status = restore.status.clone().unwrap_or_default();
501                        status.source_prep_job_name = Some(name);
502                        status.phase = KanidmRestorePhase::RestoringPrimary;
503                        status.message = None;
504                        patch_status(&restore, &ctx, status).await?;
505                    }
506                    JobState::Failed => {
507                        resume_before_mutation(&restore, &ctx).await?;
508                        set_phase(
509                            &restore,
510                            &ctx,
511                            KanidmRestorePhase::Failed,
512                            Some(
513                                "local backup file check failed; target restored to original state"
514                                    .to_string(),
515                            ),
516                        )
517                        .await?;
518                    }
519                    JobState::Running => {}
520                }
521            }
522            Ok(Action::requeue(REQUEUE))
523        }
524        KanidmRestorePhase::RestoringPrimary => {
525            let target = get_target(&restore, &ctx).await?;
526            let status = restore.status.as_ref().cloned().unwrap_or_default();
527            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
528                return Err(Error::MissingData(
529                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
530                ));
531            }
532            if !status.database_mutation_started {
533                let mut status = status;
534                status.database_mutation_started = true;
535                status.message = Some(
536                    "database mutation boundary persisted; starting primary restore".to_string(),
537                );
538                patch_status(&restore, &ctx, status).await?;
539                return Ok(Action::requeue(REQUEUE));
540            }
541            let name = restore_job_name(&restore);
542            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
543            match job_state(&restore, &ctx, &name).await? {
544                JobState::Complete => {
545                    let mut status = restore.status.clone().unwrap_or_default();
546                    status.restore_job_name = Some(name);
547                    status.phase = KanidmRestorePhase::Verifying;
548                    status.message = None;
549                    patch_status(&restore, &ctx, status).await?;
550                }
551                JobState::Failed => {
552                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
553                }
554                JobState::Running => {}
555            }
556            Ok(Action::requeue(REQUEUE))
557        }
558        KanidmRestorePhase::Verifying => {
559            let target = get_target(&restore, &ctx).await?;
560            let name = verify_job_name(&restore);
561            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
562            match job_state(&restore, &ctx, &name).await? {
563                JobState::Complete => {
564                    let mut status = restore.status.clone().unwrap_or_default();
565                    status.verify_job_name = Some(name);
566                    status.phase = KanidmRestorePhase::RebuildingReplicas;
567                    status.message = None;
568                    patch_status(&restore, &ctx, status).await?;
569                }
570                JobState::Failed => {
571                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
572                }
573                JobState::Running => {}
574            }
575            Ok(Action::requeue(REQUEUE))
576        }
577        KanidmRestorePhase::RebuildingReplicas => {
578            let target = get_target(&restore, &ctx).await?;
579            let mut status = restore.status.clone().unwrap_or_default();
580            if !status.replicas_cleared {
581                if delete_secondary_pvcs(&target, &ctx).await? {
582                    status.replicas_cleared = true;
583                    status.message = Some("secondary database state cleared".to_string());
584                    patch_status(&restore, &ctx, status).await?;
585                }
586                return Ok(Action::requeue(REQUEUE));
587            }
588
589            scale_primary(&target, &ctx, 1).await?;
590            if !primary_ready(&target, &ctx).await? {
591                return Ok(Action::requeue(REQUEUE));
592            }
593            if !status.certificates_cleared {
594                delete_replica_cert_secrets(&target, &ctx).await?;
595                delete_admin_secret(&target, &ctx).await?;
596                status.certificates_cleared = true;
597                status.message = Some(
598                    "replica certificates and admin secret cleared for regeneration".to_string(),
599                );
600                patch_status(&restore, &ctx, status).await?;
601                return Ok(Action::requeue(REQUEUE));
602            }
603            status.phase = KanidmRestorePhase::Resuming;
604            status.message = None;
605            patch_status(&restore, &ctx, status).await?;
606            Ok(Action::requeue(REQUEUE))
607        }
608        KanidmRestorePhase::Resuming => {
609            let target = get_target(&restore, &ctx).await?;
610            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
611            clear_restoring(&restore, &target, &ctx).await?;
612            Ok(Action::requeue(Duration::from_secs(3600)))
613        }
614        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
615            Ok(Action::requeue(Duration::from_secs(3600)))
616        }
617    }
618}
619
620async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
621    let target = get_target(restore, ctx).await?;
622    let actual_uid = target
623        .uid()
624        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
625    if actual_uid != restore.spec.target_ref.uid {
626        return Err(Error::MissingData(format!(
627            "target UID mismatch: expected {}, got {}",
628            restore.spec.target_ref.uid, actual_uid
629        )));
630    }
631    validate_source(restore)?;
632    if is_remote_source(restore) {
633        validate_backup_ref(restore, &target, ctx).await?;
634    }
635    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
636    {
637        return Err(Error::MissingData(format!(
638            "restoreImage must be the target's pinned Kanidm image (target image is {})",
639            target.spec.image
640        )));
641    }
642    let storage = target
643        .spec
644        .storage
645        .as_ref()
646        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
647    if storage.empty_dir.is_some()
648        || storage.ephemeral.is_some()
649        || storage.volume_claim_template.is_none()
650    {
651        return Err(Error::MissingData(
652            "restore requires PVC-backed Kanidm storage".to_string(),
653        ));
654    }
655    let primaries = target
656        .spec
657        .replica_groups
658        .iter()
659        .filter(|rg| rg.primary_node)
660        .count();
661    if primaries != 1 {
662        return Err(Error::MissingData(
663            "backup/restore requires exactly one primary replica group".to_string(),
664        ));
665    }
666    let ns = restore.namespace().unwrap();
667    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
668        .list(&ListParams::default())
669        .await
670        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
671    if restores.items.iter().any(|other| {
672        other.name_any() != restore.name_any()
673            && other.spec.target_ref.name == restore.spec.target_ref.name
674            && !matches!(
675                other.status.as_ref().map(|s| s.phase),
676                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
677            )
678    }) {
679        return Err(Error::MissingData(
680            "another active restore targets this Kanidm".to_string(),
681        ));
682    }
683    validate_safety_backup_config(restore)?;
684    Ok(target)
685}
686
687fn safe_basename(name: &str) -> bool {
688    !name.is_empty()
689        && name != "."
690        && name != ".."
691        && !name.contains('/')
692        && !name.contains('\\')
693        && !name.contains('\0')
694}
695
696fn mutable_image(image: &str) -> bool {
697    image == "kanidm/server:latest"
698        || image.ends_with(":latest")
699        || (!image.contains('@')
700            && !image
701                .rsplit('/')
702                .next()
703                .is_some_and(|part| part.contains(':')))
704}
705
706async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
707    let ns = restore
708        .namespace()
709        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
710    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
711        .get(&restore.spec.target_ref.name)
712        .await
713        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
714}
715
716async fn mark_restoring(
717    restore: &KanidmRestore,
718    target: &Kanidm,
719    ctx: &RestoreContext,
720) -> Result<()> {
721    let ns = target.namespace().unwrap();
722    let uid = restore
723        .uid()
724        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
725    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
726        .patch(
727            &target.name_any(),
728            &PatchParams::default(),
729            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
730        )
731        .await
732        .map(|_| ())
733        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
734}
735
736async fn clear_restoring(
737    restore: &KanidmRestore,
738    target: &Kanidm,
739    ctx: &RestoreContext,
740) -> Result<()> {
741    let ns = target.namespace().unwrap();
742    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
743        return Ok(());
744    }
745    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
746        .patch(
747            &target.name_any(),
748            &PatchParams::default(),
749            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
750        )
751        .await
752        .map(|_| ())
753        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
754}
755
756async fn patch_status(
757    restore: &KanidmRestore,
758    ctx: &RestoreContext,
759    mut status: KanidmRestoreStatus,
760) -> Result<()> {
761    let ns = restore.namespace().unwrap();
762    let previous_phase = restore.status.as_ref().map(|current| current.phase);
763    let next_phase = status.phase;
764    status.observed_generation = restore.metadata.generation;
765    update_restore_conditions(&mut status, restore.metadata.generation);
766
767    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
768        .patch_status(
769            &restore.name_any(),
770            &PatchParams::default(),
771            &Patch::Merge(json!({"status": &status})),
772        )
773        .await
774        .map_err(|e| {
775            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
776        })?;
777
778    if previous_phase != Some(next_phase) {
779        record_restore_transition(
780            restore,
781            ctx,
782            previous_phase,
783            next_phase,
784            status.message.as_deref(),
785        )
786        .await;
787    }
788    Ok(())
789}
790
791fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
792    let previous = status.conditions.clone();
793    let phase = status.phase;
794    let phase_reason = format!("{phase:?}");
795    let message = status
796        .message
797        .clone()
798        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
799    let terminal_success = phase == KanidmRestorePhase::Completed;
800    let terminal_failure = phase == KanidmRestorePhase::Failed;
801    let progressing = !terminal_success && !terminal_failure;
802
803    let break_glass = previous
804        .iter()
805        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
806        .cloned();
807    status.conditions = vec![
808        restore_condition(
809            &previous,
810            CONDITION_PROGRESSING,
811            if progressing {
812                CONDITION_TRUE
813            } else {
814                CONDITION_FALSE
815            },
816            &phase_reason,
817            &message,
818            generation,
819        ),
820        restore_condition(
821            &previous,
822            CONDITION_READY,
823            if terminal_success {
824                CONDITION_TRUE
825            } else {
826                CONDITION_FALSE
827            },
828            if terminal_success {
829                "RestoreCompleted"
830            } else {
831                "RestoreNotCompleted"
832            },
833            if terminal_success {
834                "Kanidm restore completed successfully."
835            } else {
836                "Kanidm restore has not completed successfully."
837            },
838            generation,
839        ),
840        restore_condition(
841            &previous,
842            CONDITION_FAILED,
843            if terminal_failure {
844                CONDITION_TRUE
845            } else {
846                CONDITION_FALSE
847            },
848            if terminal_failure {
849                "RestoreFailed"
850            } else {
851                "NoRestoreFailure"
852            },
853            if terminal_failure {
854                status
855                    .message
856                    .as_deref()
857                    .unwrap_or("Kanidm restore failed.")
858            } else {
859                "No terminal restore failure has been recorded."
860            },
861            generation,
862        ),
863    ];
864    if let Some(condition) = break_glass {
865        status.conditions.push(condition);
866    }
867}
868
869fn restore_condition(
870    previous: &[Condition],
871    condition_type: &str,
872    condition_status: &str,
873    reason: &str,
874    message: &str,
875    generation: Option<i64>,
876) -> Condition {
877    let last_transition_time = previous
878        .iter()
879        .find(|condition| {
880            condition.type_ == condition_type
881                && condition.status == condition_status
882                && condition.reason == reason
883        })
884        .map(|condition| condition.last_transition_time.clone())
885        .unwrap_or_else(|| Time(Timestamp::now()));
886    Condition {
887        type_: condition_type.to_string(),
888        status: condition_status.to_string(),
889        reason: reason.to_string(),
890        message: message.to_string(),
891        last_transition_time,
892        observed_generation: generation,
893    }
894}
895
896async fn record_restore_transition(
897    restore: &KanidmRestore,
898    ctx: &RestoreContext,
899    previous_phase: Option<KanidmRestorePhase>,
900    phase: KanidmRestorePhase,
901    message: Option<&str>,
902) {
903    if phase == KanidmRestorePhase::Validating {
904        ctx.metrics.attempts.add(1, &[]);
905    }
906
907    let result = match phase {
908        KanidmRestorePhase::Completed => Some("success"),
909        KanidmRestorePhase::Failed => Some("failure"),
910        _ => None,
911    };
912    if let Some(result) = result {
913        let attributes = [KeyValue::new("result", result)];
914        ctx.metrics.outcomes.add(1, &attributes);
915        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
916            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
917            ctx.metrics.duration_seconds.record(elapsed, &attributes);
918        }
919    }
920
921    let reason = if phase == KanidmRestorePhase::Failed {
922        "RestoreFailed"
923    } else {
924        "RestorePhaseChanged"
925    };
926    let note = message.map(str::to_string).or_else(|| {
927        Some(format!(
928            "Kanidm restore phase changed from {} to {phase:?}.",
929            previous_phase
930                .map(|previous| format!("{previous:?}"))
931                .unwrap_or_else(|| "None".to_string())
932        ))
933    });
934    if let Err(error) = ctx
935        .recorder
936        .publish(
937            &Event {
938                type_: if phase == KanidmRestorePhase::Failed {
939                    EventType::Warning
940                } else {
941                    EventType::Normal
942                },
943                reason: reason.to_string(),
944                note,
945                action: "Restore".to_string(),
946                secondary: None,
947            },
948            &restore.object_ref(&()),
949        )
950        .await
951    {
952        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
953    }
954}
955
956async fn set_phase(
957    restore: &KanidmRestore,
958    ctx: &RestoreContext,
959    phase: KanidmRestorePhase,
960    message: Option<String>,
961) -> Result<()> {
962    let mut status = restore.status.clone().unwrap_or_default();
963    status.phase = phase;
964    status.message = message;
965    patch_status(restore, ctx, status).await
966}
967
968async fn fail_after_mutation(
969    restore: &KanidmRestore,
970    ctx: &RestoreContext,
971    message: &str,
972) -> Result<()> {
973    set_phase(
974        restore,
975        ctx,
976        KanidmRestorePhase::Failed,
977        Some(message.to_string()),
978    )
979    .await
980}
981
982fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
983    target
984        .spec
985        .replica_groups
986        .iter()
987        .find(|rg| rg.primary_node)
988        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
989}
990
991async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
992    let ns = target.namespace().unwrap();
993    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
994    for rg in &target.spec.replica_groups {
995        let name = target.statefulset_name(&rg.name);
996        api.patch(
997            &name,
998            &PatchParams::default(),
999            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1000        )
1001        .await
1002        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1003    }
1004    Ok(())
1005}
1006
1007async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1008    let ns = target.namespace().unwrap();
1009    let rg = primary_group(target)?;
1010    let name = target.statefulset_name(&rg.name);
1011    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1012        .patch(
1013            &name,
1014            &PatchParams::default(),
1015            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1016        )
1017        .await
1018        .map(|_| ())
1019        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1020}
1021
1022async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1023    let ns = target.namespace().unwrap();
1024    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1025    for rg in &target.spec.replica_groups {
1026        let name = target.statefulset_name(&rg.name);
1027        api.patch(
1028            &name,
1029            &PatchParams::default(),
1030            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1031        )
1032        .await
1033        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1034    }
1035    Ok(())
1036}
1037
1038async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1039    let ns = target.namespace().unwrap();
1040    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1041        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1042        .await
1043        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1044    Ok(pods.items.is_empty())
1045}
1046
1047async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1048    let ns = target.namespace().unwrap();
1049    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1050    let mut pv_names = BTreeSet::new();
1051    let primary_name = primary_pvc_name(target)?;
1052    let primary_pvc = pvc_api
1053        .get(&primary_name)
1054        .await
1055        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1056    let primary_pv = primary_pvc
1057        .spec
1058        .as_ref()
1059        .and_then(|spec| spec.volume_name.clone())
1060        .ok_or_else(|| {
1061            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1062        })?;
1063    pv_names.insert(primary_pv);
1064
1065    for rg in &target.spec.replica_groups {
1066        let sts = target.statefulset_name(&rg.name);
1067        for ordinal in 0..rg.replicas {
1068            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1069            if name == primary_name {
1070                continue;
1071            }
1072            if let Some(pvc) = pvc_api
1073                .get_opt(&name)
1074                .await
1075                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1076                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1077            {
1078                pv_names.insert(pv_name);
1079            }
1080        }
1081    }
1082
1083    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1084        .list(&ListParams::default())
1085        .await
1086        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1087    Ok(!attachments.items.iter().any(|attachment| {
1088        attachment
1089            .spec
1090            .source
1091            .persistent_volume_name
1092            .as_ref()
1093            .is_some_and(|pv_name| pv_names.contains(pv_name))
1094    }))
1095}
1096
1097async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098    let ns = target.namespace().unwrap();
1099    let name = target.statefulset_name(&primary_group(target)?.name);
1100    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1101        .get(&name)
1102        .await
1103        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1104    Ok(sts
1105        .status
1106        .as_ref()
1107        .and_then(|s| s.ready_replicas)
1108        .unwrap_or(0)
1109        >= 1)
1110}
1111
1112fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1113    let rg = primary_group(target)?;
1114    Ok(format!(
1115        "{DATA_VOLUME}-{}-0",
1116        target.statefulset_name(&rg.name)
1117    ))
1118}
1119
1120async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1121    let ns = target.namespace().unwrap();
1122    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1123    let mut all_absent = true;
1124    for rg in &target.spec.replica_groups {
1125        let sts = target.statefulset_name(&rg.name);
1126        for ordinal in 0..rg.replicas {
1127            if rg.primary_node && ordinal == 0 {
1128                continue;
1129            }
1130            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1131            match api
1132                .get_opt(&name)
1133                .await
1134                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135            {
1136                None => {}
1137                Some(pvc) => {
1138                    all_absent = false;
1139                    if pvc.metadata.deletion_timestamp.is_none() {
1140                        match api.delete(&name, &DeleteParams::default()).await {
1141                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1142                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1143                            Err(error) => {
1144                                return Err(Error::kube_error(
1145                                    "delete",
1146                                    "PersistentVolumeClaim",
1147                                    &ns,
1148                                    &name,
1149                                    error,
1150                                ));
1151                            }
1152                        }
1153                    }
1154                }
1155            }
1156        }
1157    }
1158    Ok(all_absent)
1159}
1160
1161async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1162    let ns = target.namespace().unwrap();
1163    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1164    let label_selector = format!(
1165        "{}={},{}={}",
1166        CLUSTER_LABEL,
1167        target.name_any(),
1168        SECRET_TYPE_LABEL,
1169        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1170    );
1171    let secrets = api
1172        .list(&ListParams::default().labels(&label_selector))
1173        .await
1174        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1175    let mut all_absent = secrets.items.is_empty();
1176    for secret in secrets.items {
1177        let name = secret.name_any();
1178        if secret.metadata.deletion_timestamp.is_none() {
1179            match api.delete(&name, &DeleteParams::default()).await {
1180                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1181                Err(kube::Error::Api(status)) if status.code == 404 => {}
1182                Err(error) => {
1183                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1184                }
1185            }
1186            all_absent = false;
1187        }
1188    }
1189    Ok(all_absent)
1190}
1191
1192async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1193    let ns = target.namespace().unwrap();
1194    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1195    let name = format!("{}-admin-passwords", target.name_any());
1196    match api
1197        .get_opt(&name)
1198        .await
1199        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1200    {
1201        None => Ok(true),
1202        Some(secret) => {
1203            if secret.metadata.deletion_timestamp.is_some() {
1204                return Ok(false);
1205            }
1206            match api.delete(&name, &DeleteParams::default()).await {
1207                Ok(_) => {
1208                    debug!(secret = %name, "deleting stale admin secret");
1209                    Ok(false)
1210                }
1211                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1212                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1213            }
1214        }
1215    }
1216}
1217
1218fn restore_job_name(restore: &KanidmRestore) -> String {
1219    format!("{}-restore", restore.name_any())
1220}
1221fn verify_job_name(restore: &KanidmRestore) -> String {
1222    format!("{}-verify", restore.name_any())
1223}
1224
1225fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1226    let origin = target
1227        .spec
1228        .origin
1229        .clone()
1230        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1231    vec![
1232        EnvVar {
1233            name: "KANIDM_DB_PATH".to_string(),
1234            value: Some("/data/kanidm.db".to_string()),
1235            ..Default::default()
1236        },
1237        EnvVar {
1238            name: "KANIDM_DOMAIN".to_string(),
1239            value: Some(target.spec.domain.clone()),
1240            ..Default::default()
1241        },
1242        EnvVar {
1243            name: "KANIDM_ORIGIN".to_string(),
1244            value: Some(origin),
1245            ..Default::default()
1246        },
1247        EnvVar {
1248            name: "KANIDM_TLS_CHAIN".to_string(),
1249            value: Some(format!("{TLS_PATH}/tls.crt")),
1250            ..Default::default()
1251        },
1252        EnvVar {
1253            name: "KANIDM_TLS_KEY".to_string(),
1254            value: Some(format!("{TLS_PATH}/tls.key")),
1255            ..Default::default()
1256        },
1257    ]
1258}
1259
1260async fn ensure_database_job(
1261    restore: &KanidmRestore,
1262    target: &Kanidm,
1263    ctx: &RestoreContext,
1264    name: &str,
1265    verify: bool,
1266) -> Result<()> {
1267    let ns = restore.namespace().unwrap();
1268    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1269    if api
1270        .get_opt(name)
1271        .await
1272        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1273        .is_some()
1274    {
1275        return Ok(());
1276    }
1277    let mut command = vec![
1278        "kanidmd".to_string(),
1279        "database".to_string(),
1280        if verify { "backup" } else { "restore" }.to_string(),
1281    ];
1282    if verify {
1283        command.push("/verify/verification.json.gz".to_string());
1284    } else {
1285        if is_remote_source(restore) {
1286            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1287        } else {
1288            let file_name = restore
1289                .spec
1290                .source
1291                .local
1292                .as_ref()
1293                .map(|l| l.file_name.clone())
1294                .unwrap_or_default();
1295            command.push(format!("{BACKUP_PATH}/{file_name}"));
1296        }
1297    }
1298    let job = Job {
1299        metadata: ObjectMeta {
1300            name: Some(name.to_string()),
1301            namespace: Some(ns.clone()),
1302            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1303            ..Default::default()
1304        },
1305        spec: Some(JobSpec {
1306            backoff_limit: Some(0),
1307            template: PodTemplateSpec {
1308                metadata: Some(ObjectMeta::default()),
1309                spec: Some(PodSpec {
1310                    automount_service_account_token: Some(false),
1311                    restart_policy: Some("Never".to_string()),
1312                    security_context: Some(source_job_pod_security_context(target)),
1313                    containers: vec![Container {
1314                        name: if verify { "verify" } else { "restore" }.to_string(),
1315                        image: Some(restore.spec.restore_image.clone()),
1316                        command: Some(command),
1317                        env: Some(build_kanidm_db_env_vars(target)),
1318                        security_context: Some(kanidm_job_security_context()),
1319                        resources: Some(job_resource_requirements()),
1320                        volume_mounts: Some(vec![
1321                            VolumeMount {
1322                                name: DATA_VOLUME.to_string(),
1323                                mount_path: DATA_PATH.to_string(),
1324                                ..Default::default()
1325                            },
1326                            VolumeMount {
1327                                name: TLS_VOLUME.to_string(),
1328                                mount_path: TLS_PATH.to_string(),
1329                                read_only: Some(true),
1330                                ..Default::default()
1331                            },
1332                            VolumeMount {
1333                                name: "verification-output".to_string(),
1334                                mount_path: "/verify".to_string(),
1335                                ..Default::default()
1336                            },
1337                        ]),
1338                        ..Default::default()
1339                    }],
1340                    volumes: Some(vec![
1341                        Volume {
1342                            name: DATA_VOLUME.to_string(),
1343                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1344                                claim_name: primary_pvc_name(target)?,
1345                                read_only: Some(false),
1346                            }),
1347                            ..Default::default()
1348                        },
1349                        Volume {
1350                            name: TLS_VOLUME.to_string(),
1351                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1352                                secret_name: Some(target.effective_tls_secret_name()),
1353                                ..Default::default()
1354                            }),
1355                            ..Default::default()
1356                        },
1357                        Volume {
1358                            name: "verification-output".to_string(),
1359                            empty_dir: Some(Default::default()),
1360                            ..Default::default()
1361                        },
1362                    ]),
1363                    ..Default::default()
1364                }),
1365            },
1366            ..Default::default()
1367        }),
1368        ..Default::default()
1369    };
1370    api.create(&PostParams::default(), &job)
1371        .await
1372        .map(|_| ())
1373        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1374}
1375
1376enum JobState {
1377    Running,
1378    Complete,
1379    Failed,
1380}
1381
1382async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1383    let ns = restore.namespace().unwrap();
1384    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1385        .get(name)
1386        .await
1387        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1388    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1389        return Ok(JobState::Failed);
1390    }
1391    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1392        return Ok(JobState::Complete);
1393    }
1394    Ok(JobState::Running)
1395}
1396
1397fn is_remote_source(restore: &KanidmRestore) -> bool {
1398    restore.spec.source.backup_ref.is_some()
1399}
1400
1401fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1402    let safety = restore.spec.safety_backup.as_ref();
1403    let skip = safety.is_some_and(|s| s.skip);
1404    !skip
1405}
1406
1407fn validate_source(restore: &KanidmRestore) -> Result<()> {
1408    let has_local = restore.spec.source.local.is_some();
1409    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1410    if has_local && has_backup_ref {
1411        return Err(Error::MissingData(
1412            "source.local and source.backupRef are mutually exclusive".to_string(),
1413        ));
1414    }
1415    if !has_local && !has_backup_ref {
1416        return Err(Error::MissingData(
1417            "source must specify either local or backupRef".to_string(),
1418        ));
1419    }
1420    if let Some(local) = &restore.spec.source.local {
1421        if !safe_basename(&local.file_name) {
1422            return Err(Error::MissingData(
1423                "restore source fileName must be a safe basename".to_string(),
1424            ));
1425        }
1426    }
1427    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1428        if backup_ref.name.is_empty() {
1429            return Err(Error::MissingData(
1430                "source.backupRef.name must not be empty".to_string(),
1431            ));
1432        }
1433    }
1434    Ok(())
1435}
1436
1437fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1438    let safety = restore.spec.safety_backup.as_ref();
1439    let skip = safety.is_some_and(|s| s.skip);
1440    if skip {
1441        let annotations = restore.metadata.annotations.as_ref();
1442        let reason = annotations
1443            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1444            .map(|s| s.trim())
1445            .unwrap_or("");
1446        let approver = annotations
1447            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1448            .map(|s| s.trim())
1449            .unwrap_or("");
1450        if reason.is_empty() {
1451            return Err(Error::MissingData(format!(
1452                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1453            )));
1454        }
1455        if approver.is_empty() {
1456            return Err(Error::MissingData(format!(
1457                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1458            )));
1459        }
1460    }
1461    if is_remote_source(restore) && !skip {
1462        let has_repo = safety
1463            .and_then(|s| s.repository_ref.as_ref())
1464            .is_some_and(|r| !r.name.is_empty());
1465        if !has_repo {
1466            return Err(Error::MissingData(
1467                "remote restore requires safetyBackup.repositoryRef".to_string(),
1468            ));
1469        }
1470    }
1471    Ok(())
1472}
1473
1474fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1475    target
1476        .spec
1477        .replica_groups
1478        .iter()
1479        .map(|rg| ReplicaCountEntry {
1480            group: rg.name.clone(),
1481            replicas: rg.replicas,
1482        })
1483        .collect()
1484}
1485
1486async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1487    if let Ok(target) = get_target(restore, ctx).await {
1488        let owns_maintenance =
1489            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1490        if owns_maintenance {
1491            scale_desired(&target, ctx).await?;
1492            clear_restoring(restore, &target, ctx).await?;
1493        }
1494    }
1495    Ok(())
1496}
1497
1498async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1499    ctx.metrics.break_glass_total.add(1, &[]);
1500    let reason = restore
1501        .metadata
1502        .annotations
1503        .as_ref()
1504        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1505        .cloned()
1506        .unwrap_or_default();
1507    let approver = restore
1508        .metadata
1509        .annotations
1510        .as_ref()
1511        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1512        .cloned()
1513        .unwrap_or_default();
1514    let mut status = restore.status.clone().unwrap_or_default();
1515    let break_glass_condition = restore_condition(
1516        &status.conditions,
1517        CONDITION_BREAK_GLASS,
1518        CONDITION_TRUE,
1519        "BreakGlassOverride",
1520        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1521        restore.metadata.generation,
1522    );
1523    status
1524        .conditions
1525        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1526    status.conditions.push(break_glass_condition);
1527    patch_status(restore, ctx, status).await?;
1528    if let Err(error) = ctx
1529        .recorder
1530        .publish(
1531            &Event {
1532                type_: EventType::Warning,
1533                reason: "SafetyBackupSkipped".to_string(),
1534                note: Some(format!(
1535                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1536                )),
1537                action: "Restore".to_string(),
1538                secondary: None,
1539            },
1540            &restore.object_ref(&()),
1541        )
1542        .await
1543    {
1544        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1545    }
1546    warn!(
1547        restore = %restore.name_any(),
1548        reason = %reason,
1549        approver = %approver,
1550        "break-glass safety backup override activated"
1551    );
1552    Ok(())
1553}
1554
1555async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1556    let status = restore.status.as_ref().cloned().unwrap_or_default();
1557    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1558        if let Ok(start_ts) = start.parse::<Timestamp>() {
1559            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1560            ctx.metrics
1561                .safety_backup_duration_seconds
1562                .record(elapsed, &[]);
1563        }
1564    }
1565}
1566
1567fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1568    format!("{}-safety-backup", restore.name_any())
1569}
1570
1571fn source_prep_job_name(restore: &KanidmRestore) -> String {
1572    format!("{}-source-prep", restore.name_any())
1573}
1574
1575fn source_check_job_name(restore: &KanidmRestore) -> String {
1576    format!("{}-source-check", restore.name_any())
1577}
1578
1579async fn ensure_source_check_job(
1580    restore: &KanidmRestore,
1581    target: &Kanidm,
1582    ctx: &RestoreContext,
1583    name: &str,
1584) -> Result<()> {
1585    let ns = restore.namespace().unwrap();
1586    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1587    if api
1588        .get_opt(name)
1589        .await
1590        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1591        .is_some()
1592    {
1593        return Ok(());
1594    }
1595    let file_name = restore
1596        .spec
1597        .source
1598        .local
1599        .as_ref()
1600        .map(|l| l.file_name.clone())
1601        .unwrap_or_default();
1602    let operation_doc = serde_json::json!({
1603        "apiVersion": "backup.kaniop.rs/v1alpha1",
1604        "kind": "OperationDocument",
1605        "operation": "check",
1606        "path": format!("{BACKUP_PATH}/{file_name}"),
1607        "resultPath": "/run/kaniop-result/result.json",
1608        "format": "kanidmJsonGzip",
1609    })
1610    .to_string();
1611    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1612    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1613    let data_mover_image = data_mover_image();
1614    let job = Job {
1615        metadata: ObjectMeta {
1616            name: Some(name.to_string()),
1617            namespace: Some(ns.clone()),
1618            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1619            ..Default::default()
1620        },
1621        spec: Some(JobSpec {
1622            backoff_limit: Some(0),
1623            template: PodTemplateSpec {
1624                metadata: Some(ObjectMeta::default()),
1625                spec: Some(PodSpec {
1626                    automount_service_account_token: Some(false),
1627                    restart_policy: Some("Never".to_string()),
1628                    security_context: Some(source_job_pod_security_context(target)),
1629                    containers: vec![Container {
1630                        name: "source-check".to_string(),
1631                        image: Some(data_mover_image),
1632                        command: Some(vec![
1633                            "/bin/kaniop-data-mover".to_string(),
1634                            "check".to_string(),
1635                            "--operation-doc".to_string(),
1636                            "/run/kaniop/operation.json".to_string(),
1637                        ]),
1638                        security_context: Some(hardened_security_context()),
1639                        resources: Some(job_resource_requirements()),
1640                        env: Some(vec![EnvVar {
1641                            name: "RUST_LOG".to_string(),
1642                            value: Some("info".to_string()),
1643                            ..Default::default()
1644                        }]),
1645                        volume_mounts: Some(vec![
1646                            VolumeMount {
1647                                name: DATA_VOLUME.to_string(),
1648                                mount_path: DATA_PATH.to_string(),
1649                                read_only: Some(true),
1650                                ..Default::default()
1651                            },
1652                            VolumeMount {
1653                                name: "operation".to_string(),
1654                                mount_path: "/run/kaniop".to_string(),
1655                                read_only: Some(true),
1656                                ..Default::default()
1657                            },
1658                            VolumeMount {
1659                                name: "result".to_string(),
1660                                mount_path: "/run/kaniop-result".to_string(),
1661                                ..Default::default()
1662                            },
1663                        ]),
1664                        ..Default::default()
1665                    }],
1666                    volumes: Some(vec![
1667                        Volume {
1668                            name: DATA_VOLUME.to_string(),
1669                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1670                                claim_name: primary_pvc_name(target)?,
1671                                read_only: Some(true),
1672                            }),
1673                            ..Default::default()
1674                        },
1675                        Volume {
1676                            name: "operation".to_string(),
1677                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1678                                name: operation_cm_name,
1679                                ..Default::default()
1680                            }),
1681                            ..Default::default()
1682                        },
1683                        Volume {
1684                            name: "result".to_string(),
1685                            empty_dir: Some(Default::default()),
1686                            ..Default::default()
1687                        },
1688                    ]),
1689                    ..Default::default()
1690                }),
1691            },
1692            ..Default::default()
1693        }),
1694        ..Default::default()
1695    };
1696    api.create(&PostParams::default(), &job)
1697        .await
1698        .map(|_| ())
1699        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1700}
1701
1702const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1703
1704fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1705    let ns = restore.namespace().unwrap_or_default();
1706    let name = restore.name_any();
1707    let namespace_uuid =
1708        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1709    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1710    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1711}
1712
1713fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1714    let ns = restore.namespace().unwrap_or_default();
1715    let kanidm_uid = &restore.spec.target_ref.uid;
1716    let backup_id = compute_safety_backup_id(restore);
1717    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1718}
1719
1720struct VerifiedSafetyBackup {
1721    manifest_key: String,
1722    payload_sha256: String,
1723}
1724
1725async fn read_safety_backup_result(
1726    restore: &KanidmRestore,
1727    ctx: &RestoreContext,
1728    job_name: &str,
1729    expected_backup_id: &str,
1730) -> Result<VerifiedSafetyBackup> {
1731    let ns = restore.namespace().unwrap();
1732    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1733    let pod_list = pods
1734        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1735        .await
1736        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1737    let pod = pod_list.items.first().ok_or_else(|| {
1738        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1739    })?;
1740    let pod_name = pod.name_any();
1741    let container_status = pod
1742        .status
1743        .as_ref()
1744        .and_then(|s| s.container_statuses.as_ref())
1745        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1746        .ok_or_else(|| {
1747            Error::MissingData(format!(
1748                "safety-upload container status not found in pod {pod_name}"
1749            ))
1750        })?;
1751    let termination_message = container_status
1752        .state
1753        .as_ref()
1754        .and_then(|state| state.terminated.as_ref())
1755        .and_then(|t| t.message.as_ref())
1756        .ok_or_else(|| {
1757            Error::MissingData(format!(
1758                "termination message not found in pod {pod_name}; result transport failed"
1759            ))
1760        })?;
1761    if termination_message.is_empty() {
1762        return Err(Error::MissingData(format!(
1763            "termination message is empty in pod {pod_name}; result document was not written"
1764        )));
1765    }
1766    let result_doc = parse_result_document(termination_message).map_err(|e| {
1767        Error::ParseError(format!(
1768            "failed to parse result document from pod {pod_name}: {e}"
1769        ))
1770    })?;
1771    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1772        return Err(Error::ParseError(format!(
1773            "unsupported result document version: {}",
1774            result_doc.api_version
1775        )));
1776    }
1777    if result_doc.kind != "ResultDocument" {
1778        return Err(Error::ParseError(format!(
1779            "invalid result document kind: {}",
1780            result_doc.kind
1781        )));
1782    }
1783    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1784        return Err(Error::ParseError(format!(
1785            "result document operation mismatch: expected '{}', got '{}'",
1786            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1787        )));
1788    }
1789    if !result_doc.success {
1790        return Err(Error::ParseError(format!(
1791            "safety backup reported failure with exit code {:?}",
1792            result_doc.exit_code
1793        )));
1794    }
1795    if result_doc.exit_code != ExitCode::Success {
1796        return Err(Error::ParseError(format!(
1797            "safety backup exit code is not Success: {:?}",
1798            result_doc.exit_code
1799        )));
1800    }
1801    let doc_backup_id = result_doc
1802        .backup_id
1803        .as_deref()
1804        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1805    if doc_backup_id != expected_backup_id {
1806        return Err(Error::ParseError(format!(
1807            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1808        )));
1809    }
1810    let manifest_key = result_doc
1811        .manifest_key
1812        .clone()
1813        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1814    if manifest_key.is_empty() {
1815        return Err(Error::ParseError(
1816            "result document manifestKey is empty".to_string(),
1817        ));
1818    }
1819    let payload_sha256 = result_doc
1820        .payload_sha256
1821        .clone()
1822        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1823    if payload_sha256.is_empty() {
1824        return Err(Error::ParseError(
1825            "result document payloadSha256 is empty".to_string(),
1826        ));
1827    }
1828    info!(
1829        restore = %restore.name_any(),
1830        backup_id = %expected_backup_id,
1831        manifest_key = %manifest_key,
1832        "safety backup result verified"
1833    );
1834    Ok(VerifiedSafetyBackup {
1835        manifest_key,
1836        payload_sha256,
1837    })
1838}
1839
1840struct VerifiedSourcePrepResult {
1841    #[allow(dead_code)]
1842    manifest_key: String,
1843    payload_sha256: String,
1844}
1845
1846async fn read_source_prep_result(
1847    restore: &KanidmRestore,
1848    ctx: &RestoreContext,
1849    job_name: &str,
1850    expected_backup_id: &str,
1851) -> Result<VerifiedSourcePrepResult> {
1852    let ns = restore.namespace().unwrap();
1853    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1854    let pod_list = pods
1855        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1856        .await
1857        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1858    let pod = pod_list.items.first().ok_or_else(|| {
1859        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1860    })?;
1861    let pod_name = pod.name_any();
1862    let container_status = pod
1863        .status
1864        .as_ref()
1865        .and_then(|s| s.container_statuses.as_ref())
1866        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1867        .ok_or_else(|| {
1868            Error::MissingData(format!(
1869                "source-prep container status not found in pod {pod_name}"
1870            ))
1871        })?;
1872    let termination_message = container_status
1873        .state
1874        .as_ref()
1875        .and_then(|state| state.terminated.as_ref())
1876        .and_then(|t| t.message.as_ref())
1877        .ok_or_else(|| {
1878            Error::MissingData(format!(
1879                "termination message not found in pod {pod_name}; result transport failed"
1880            ))
1881        })?;
1882    if termination_message.is_empty() {
1883        return Err(Error::MissingData(format!(
1884            "termination message is empty in pod {pod_name}; result document was not written"
1885        )));
1886    }
1887    let result_doc = parse_result_document(termination_message).map_err(|e| {
1888        Error::ParseError(format!(
1889            "failed to parse result document from pod {pod_name}: {e}"
1890        ))
1891    })?;
1892    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1893        return Err(Error::ParseError(format!(
1894            "unsupported result document version: {}",
1895            result_doc.api_version
1896        )));
1897    }
1898    if result_doc.kind != "ResultDocument" {
1899        return Err(Error::ParseError(format!(
1900            "invalid result document kind: {}",
1901            result_doc.kind
1902        )));
1903    }
1904    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1905        return Err(Error::ParseError(format!(
1906            "result document operation mismatch: expected '{}', got '{}'",
1907            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1908        )));
1909    }
1910    if !result_doc.success {
1911        return Err(Error::ParseError(format!(
1912            "source prep reported failure with exit code {:?}",
1913            result_doc.exit_code
1914        )));
1915    }
1916    if result_doc.exit_code != ExitCode::Success {
1917        return Err(Error::ParseError(format!(
1918            "source prep exit code is not Success: {:?}",
1919            result_doc.exit_code
1920        )));
1921    }
1922    let doc_backup_id = result_doc
1923        .backup_id
1924        .as_deref()
1925        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1926    if doc_backup_id != expected_backup_id {
1927        return Err(Error::ParseError(format!(
1928            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1929        )));
1930    }
1931    let manifest_key = result_doc
1932        .manifest_key
1933        .clone()
1934        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1935    if manifest_key.is_empty() {
1936        return Err(Error::ParseError(
1937            "result document manifestKey is empty".to_string(),
1938        ));
1939    }
1940    let payload_sha256 = result_doc
1941        .payload_sha256
1942        .clone()
1943        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1944    if payload_sha256.is_empty() {
1945        return Err(Error::ParseError(
1946            "result document payloadSha256 is empty".to_string(),
1947        ));
1948    }
1949    info!(
1950        restore = %restore.name_any(),
1951        backup_id = %expected_backup_id,
1952        manifest_key = %manifest_key,
1953        payload_sha256 = %payload_sha256,
1954        "source prep result verified"
1955    );
1956    Ok(VerifiedSourcePrepResult {
1957        manifest_key,
1958        payload_sha256,
1959    })
1960}
1961
1962fn source_job_pod_security_context(
1963    target: &Kanidm,
1964) -> k8s_openapi::api::core::v1::PodSecurityContext {
1965    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
1966    security_context.seccomp_profile = Some(SeccompProfile {
1967        type_: "RuntimeDefault".to_string(),
1968        ..Default::default()
1969    });
1970    security_context
1971}
1972
1973fn hardened_security_context() -> SecurityContext {
1974    SecurityContext {
1975        allow_privilege_escalation: Some(false),
1976        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
1977            drop: Some(vec!["ALL".to_string()]),
1978            ..Default::default()
1979        }),
1980        read_only_root_filesystem: Some(true),
1981        run_as_non_root: Some(true),
1982        seccomp_profile: Some(SeccompProfile {
1983            type_: "RuntimeDefault".to_string(),
1984            ..Default::default()
1985        }),
1986        ..Default::default()
1987    }
1988}
1989
1990fn kanidm_job_security_context() -> SecurityContext {
1991    SecurityContext {
1992        allow_privilege_escalation: Some(false),
1993        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
1994            drop: Some(vec!["ALL".to_string()]),
1995            ..Default::default()
1996        }),
1997        seccomp_profile: Some(SeccompProfile {
1998            type_: "RuntimeDefault".to_string(),
1999            ..Default::default()
2000        }),
2001        ..Default::default()
2002    }
2003}
2004
2005fn job_resource_requirements() -> ResourceRequirements {
2006    ResourceRequirements {
2007        requests: Some(std::collections::BTreeMap::from([
2008            ("cpu".to_string(), Quantity("100m".to_string())),
2009            ("memory".to_string(), Quantity("128Mi".to_string())),
2010        ])),
2011        limits: Some(std::collections::BTreeMap::from([
2012            ("cpu".to_string(), Quantity("2".to_string())),
2013            ("memory".to_string(), Quantity("2Gi".to_string())),
2014        ])),
2015        ..Default::default()
2016    }
2017}
2018
2019async fn ensure_safety_backup_job(
2020    restore: &KanidmRestore,
2021    target: &Kanidm,
2022    ctx: &RestoreContext,
2023    name: &str,
2024    backup_id: &str,
2025    manifest_key: &str,
2026) -> Result<()> {
2027    let ns = restore.namespace().unwrap();
2028    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2029    if api
2030        .get_opt(name)
2031        .await
2032        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2033        .is_some()
2034    {
2035        return Ok(());
2036    }
2037    let safety = restore
2038        .spec
2039        .safety_backup
2040        .as_ref()
2041        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2042    let repo_ref = safety
2043        .repository_ref
2044        .as_ref()
2045        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2046    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2047        .get(&repo_ref.name)
2048        .await
2049        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2050    let _namespace_uid = &ns;
2051    let _kanidm_uid = &restore.spec.target_ref.uid;
2052    let endpoint = &repo.spec.s3.endpoint;
2053    let region = &repo.spec.s3.region;
2054    let operation_doc = build_safety_upload_operation_doc(
2055        restore,
2056        target,
2057        backup_id,
2058        manifest_key,
2059        &repo.spec.s3.bucket,
2060        &repo.spec.s3.prefix,
2061        endpoint,
2062        region,
2063        repo.spec.s3.force_path_style,
2064        repo.spec.s3.insecure,
2065        repo.spec
2066            .s3
2067            .ca_bundle_ref
2068            .as_deref()
2069            .map(|_| ca_bundle_path())
2070            .as_deref(),
2071        repo.spec.encryption.as_ref(),
2072    )?;
2073    let operation_cm_name = format!("{name}-op");
2074    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2075    let auth_method = &repo.spec.authentication.writer;
2076    let data_mover_image = data_mover_image();
2077    let init_command = vec![
2078        "kanidmd".to_string(),
2079        "database".to_string(),
2080        "backup".to_string(),
2081        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2082    ];
2083    let upload_script = format!(
2084        r#"set -eu
2085/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2086RESULT_FILE="/run/kaniop-result/result.json"
2087TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2088if [ ! -f "$RESULT_FILE" ]; then
2089  echo "ERROR: result document not found at $RESULT_FILE"
2090  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2091  exit 1
2092fi
2093if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2094  echo "ERROR: safety backup upload reported failure"
2095  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2096  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2097  exit 1
2098fi
2099dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2100echo "safety backup upload completed successfully"
2101"#
2102    );
2103    let job = Job {
2104        metadata: ObjectMeta {
2105            name: Some(name.to_string()),
2106            namespace: Some(ns.clone()),
2107            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2108            ..Default::default()
2109        },
2110        spec: Some(JobSpec {
2111            backoff_limit: Some(0),
2112            template: PodTemplateSpec {
2113                metadata: Some(ObjectMeta::default()),
2114                spec: Some(PodSpec {
2115                    automount_service_account_token: Some(false),
2116                    restart_policy: Some("Never".to_string()),
2117                    security_context: Some(source_job_pod_security_context(target)),
2118                    init_containers: Some(vec![Container {
2119                        name: "safety-backup".to_string(),
2120                        image: Some(restore.spec.restore_image.clone()),
2121                        command: Some(init_command),
2122                        security_context: Some(SecurityContext {
2123                            allow_privilege_escalation: Some(false),
2124                            capabilities: Some(Capabilities {
2125                                drop: Some(vec!["ALL".to_string()]),
2126                                ..Default::default()
2127                            }),
2128                            read_only_root_filesystem: Some(true),
2129                            seccomp_profile: Some(SeccompProfile {
2130                                type_: "RuntimeDefault".to_string(),
2131                                ..Default::default()
2132                            }),
2133                            ..Default::default()
2134                        }),
2135                        resources: Some(job_resource_requirements()),
2136                        env: Some(build_kanidm_db_env_vars(target)),
2137                        volume_mounts: Some(vec![
2138                            VolumeMount {
2139                                name: DATA_VOLUME.to_string(),
2140                                mount_path: DATA_PATH.to_string(),
2141                                read_only: Some(false),
2142                                ..Default::default()
2143                            },
2144                            VolumeMount {
2145                                name: TLS_VOLUME.to_string(),
2146                                mount_path: TLS_PATH.to_string(),
2147                                read_only: Some(true),
2148                                ..Default::default()
2149                            },
2150                            VolumeMount {
2151                                name: SHARED_VOLUME.to_string(),
2152                                mount_path: SHARED_VOL_PATH.to_string(),
2153                                ..Default::default()
2154                            },
2155                        ]),
2156                        ..Default::default()
2157                    }]),
2158                    containers: vec![Container {
2159                        name: "safety-upload".to_string(),
2160                        image: Some(data_mover_image),
2161                        command: Some(vec!["/bin/sh".to_string()]),
2162                        args: Some(vec!["-c".to_string(), upload_script]),
2163                        security_context: Some(hardened_security_context()),
2164                        resources: Some(job_resource_requirements()),
2165                        env: {
2166                            let mut env_vars =
2167                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2168                            env_vars.push(EnvVar {
2169                                name: "RUST_LOG".to_string(),
2170                                value: Some("info".to_string()),
2171                                ..Default::default()
2172                            });
2173                            if repo.spec.s3.ca_bundle_ref.is_some() {
2174                                env_vars.push(ca_bundle_env_var());
2175                            }
2176                            env_vars.extend(build_encryption_env_vars(
2177                                repo.spec
2178                                    .encryption
2179                                    .as_ref()
2180                                    .and_then(|e| e.key_ref.as_ref()),
2181                            ));
2182                            Some(env_vars)
2183                        },
2184                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2185                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2186                        volume_mounts: {
2187                            let mut mounts = vec![
2188                                VolumeMount {
2189                                    name: SHARED_VOLUME.to_string(),
2190                                    mount_path: SHARED_VOL_PATH.to_string(),
2191                                    read_only: Some(true),
2192                                    ..Default::default()
2193                                },
2194                                VolumeMount {
2195                                    name: "operation".to_string(),
2196                                    mount_path: "/run/kaniop".to_string(),
2197                                    read_only: Some(true),
2198                                    ..Default::default()
2199                                },
2200                                VolumeMount {
2201                                    name: "result".to_string(),
2202                                    mount_path: "/run/kaniop-result".to_string(),
2203                                    ..Default::default()
2204                                },
2205                            ];
2206                            mounts.extend(build_auth_volume_mounts(auth_method));
2207                            if repo.spec.s3.ca_bundle_ref.is_some() {
2208                                mounts.push(build_ca_bundle_volume_mount());
2209                            }
2210                            Some(mounts)
2211                        },
2212                        ..Default::default()
2213                    }],
2214                    volumes: {
2215                        let mut vols = vec![
2216                            Volume {
2217                                name: DATA_VOLUME.to_string(),
2218                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2219                                    claim_name: primary_pvc_name(target)?,
2220                                    read_only: Some(false),
2221                                }),
2222                                ..Default::default()
2223                            },
2224                            Volume {
2225                                name: TLS_VOLUME.to_string(),
2226                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2227                                    secret_name: Some(target.effective_tls_secret_name()),
2228                                    ..Default::default()
2229                                }),
2230                                ..Default::default()
2231                            },
2232                            Volume {
2233                                name: SHARED_VOLUME.to_string(),
2234                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2235                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2236                                    ..Default::default()
2237                                }),
2238                                ..Default::default()
2239                            },
2240                            Volume {
2241                                name: "operation".to_string(),
2242                                config_map: Some(ConfigMapVolumeSource {
2243                                    name: operation_cm_name,
2244                                    ..Default::default()
2245                                }),
2246                                ..Default::default()
2247                            },
2248                            Volume {
2249                                name: "result".to_string(),
2250                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2251                                    size_limit: Some(Quantity("16Mi".to_string())),
2252                                    ..Default::default()
2253                                }),
2254                                ..Default::default()
2255                            },
2256                        ];
2257                        vols.extend(build_auth_volumes(auth_method));
2258                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2259                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2260                        }
2261                        Some(vols)
2262                    },
2263                    ..Default::default()
2264                }),
2265            },
2266            ..Default::default()
2267        }),
2268        ..Default::default()
2269    };
2270    api.create(&PostParams::default(), &job)
2271        .await
2272        .map(|_| ())
2273        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2274}
2275
2276async fn ensure_source_prep_job(
2277    restore: &KanidmRestore,
2278    target: &Kanidm,
2279    ctx: &RestoreContext,
2280    name: &str,
2281) -> Result<()> {
2282    let ns = restore.namespace().unwrap();
2283    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2284    if api
2285        .get_opt(name)
2286        .await
2287        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2288        .is_some()
2289    {
2290        return Ok(());
2291    }
2292    let backup_name = restore
2293        .spec
2294        .source
2295        .backup_ref
2296        .as_ref()
2297        .map(|b| b.name.clone())
2298        .unwrap_or_default();
2299    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2300        .get(&backup_name)
2301        .await
2302        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2303    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2304        .get(&backup.spec.repository_ref.name)
2305        .await
2306        .map_err(|e| {
2307            Error::kube_error(
2308                "get",
2309                "KanidmBackupRepository",
2310                &ns,
2311                &backup.spec.repository_ref.name,
2312                e,
2313            )
2314        })?;
2315    let endpoint = &repo.spec.s3.endpoint;
2316    let region = &repo.spec.s3.region;
2317    let operation_doc = build_download_operation_doc(
2318        restore,
2319        target,
2320        &backup.spec.manifest_key,
2321        &backup.spec.backup_id,
2322        &repo.spec.s3.bucket,
2323        &repo.spec.s3.prefix,
2324        endpoint,
2325        region,
2326        repo.spec.s3.force_path_style,
2327        repo.spec.s3.insecure,
2328        repo.spec
2329            .s3
2330            .ca_bundle_ref
2331            .as_deref()
2332            .map(|_| ca_bundle_path())
2333            .as_deref(),
2334        repo.spec.encryption.as_ref(),
2335    );
2336    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2337    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2338    let auth_method = &repo.spec.authentication.reader;
2339    let data_mover_image = data_mover_image();
2340    let download_script = format!(
2341        r#"set -eu
2342/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2343RESULT_FILE="/run/kaniop-result/result.json"
2344TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2345if [ ! -f "$RESULT_FILE" ]; then
2346  echo "ERROR: result document not found at $RESULT_FILE"
2347  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2348  exit 1
2349fi
2350if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2351  echo "ERROR: source preparation download reported failure"
2352  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2353  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2354  exit 1
2355fi
2356dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2357echo "source preparation download completed successfully"
2358"#
2359    );
2360    let job = Job {
2361        metadata: ObjectMeta {
2362            name: Some(name.to_string()),
2363            namespace: Some(ns.clone()),
2364            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2365            ..Default::default()
2366        },
2367        spec: Some(JobSpec {
2368            backoff_limit: Some(0),
2369            template: PodTemplateSpec {
2370                metadata: Some(ObjectMeta::default()),
2371                spec: Some(PodSpec {
2372                    automount_service_account_token: Some(false),
2373                    restart_policy: Some("Never".to_string()),
2374                    security_context: Some(source_job_pod_security_context(target)),
2375                    containers: vec![Container {
2376                        name: "source-prep".to_string(),
2377                        image: Some(data_mover_image),
2378                        command: Some(vec!["/bin/sh".to_string()]),
2379                        args: Some(vec!["-c".to_string(), download_script]),
2380                        security_context: Some(hardened_security_context()),
2381                        resources: Some(job_resource_requirements()),
2382                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2383                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2384                        env: {
2385                            let mut env_vars = build_auth_env_vars(
2386                                auth_method,
2387                                &backup.spec.repository_ref.name,
2388                                AuthRole::Reader,
2389                            );
2390                            env_vars.push(EnvVar {
2391                                name: "RUST_LOG".to_string(),
2392                                value: Some("info".to_string()),
2393                                ..Default::default()
2394                            });
2395                            if repo.spec.s3.ca_bundle_ref.is_some() {
2396                                env_vars.push(ca_bundle_env_var());
2397                            }
2398                            env_vars.extend(build_encryption_env_vars(
2399                                repo.spec
2400                                    .encryption
2401                                    .as_ref()
2402                                    .and_then(|e| e.key_ref.as_ref()),
2403                            ));
2404                            Some(env_vars)
2405                        },
2406                        volume_mounts: {
2407                            let mut mounts = vec![
2408                                VolumeMount {
2409                                    name: DATA_VOLUME.to_string(),
2410                                    mount_path: DATA_PATH.to_string(),
2411                                    ..Default::default()
2412                                },
2413                                VolumeMount {
2414                                    name: "operation".to_string(),
2415                                    mount_path: "/run/kaniop".to_string(),
2416                                    read_only: Some(true),
2417                                    ..Default::default()
2418                                },
2419                                VolumeMount {
2420                                    name: "result".to_string(),
2421                                    mount_path: "/run/kaniop-result".to_string(),
2422                                    ..Default::default()
2423                                },
2424                            ];
2425                            mounts.extend(build_auth_volume_mounts(auth_method));
2426                            if repo.spec.s3.ca_bundle_ref.is_some() {
2427                                mounts.push(build_ca_bundle_volume_mount());
2428                            }
2429                            Some(mounts)
2430                        },
2431                        ..Default::default()
2432                    }],
2433                    volumes: {
2434                        let mut vols = vec![
2435                            Volume {
2436                                name: DATA_VOLUME.to_string(),
2437                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2438                                    claim_name: primary_pvc_name(target)?,
2439                                    read_only: Some(false),
2440                                }),
2441                                ..Default::default()
2442                            },
2443                            Volume {
2444                                name: "operation".to_string(),
2445                                config_map: Some(ConfigMapVolumeSource {
2446                                    name: operation_cm_name,
2447                                    ..Default::default()
2448                                }),
2449                                ..Default::default()
2450                            },
2451                            Volume {
2452                                name: "result".to_string(),
2453                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2454                                    size_limit: Some(Quantity("16Mi".to_string())),
2455                                    ..Default::default()
2456                                }),
2457                                ..Default::default()
2458                            },
2459                        ];
2460                        vols.extend(build_auth_volumes(auth_method));
2461                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2462                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2463                        }
2464                        Some(vols)
2465                    },
2466                    ..Default::default()
2467                }),
2468            },
2469            ..Default::default()
2470        }),
2471        ..Default::default()
2472    };
2473    api.create(&PostParams::default(), &job)
2474        .await
2475        .map(|_| ())
2476        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2477}
2478
2479#[allow(clippy::too_many_arguments)]
2480fn build_safety_upload_operation_doc(
2481    restore: &KanidmRestore,
2482    target: &Kanidm,
2483    backup_id: &str,
2484    manifest_key: &str,
2485    bucket: &str,
2486    prefix: &str,
2487    endpoint: &str,
2488    region: &str,
2489    force_path_style: bool,
2490    insecure: bool,
2491    ca_bundle_path: Option<&str>,
2492    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2493) -> Result<String> {
2494    let enc_mode =
2495        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2496    let enc_key_id = encryption
2497        .and_then(|e| e.key_id.as_ref())
2498        .map(|s| serde_json::Value::String(s.clone()));
2499    let op = UploadOperation {
2500        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2501        bucket: bucket.to_string(),
2502        prefix: prefix.to_string(),
2503        endpoint: endpoint.to_string(),
2504        region: region.to_string(),
2505        force_path_style,
2506        insecure,
2507        ca_bundle_path: ca_bundle_path.map(str::to_string),
2508        backup_id: backup_id.to_string(),
2509        namespace_uid: restore.namespace().unwrap_or_default(),
2510        kanidm_uid: restore.spec.target_ref.uid.clone(),
2511        kanidm_name: restore.spec.target_ref.name.clone(),
2512        domain: target.spec.domain.clone(),
2513        kanidm_version: target
2514            .status
2515            .as_ref()
2516            .and_then(|s| s.version.as_ref())
2517            .map(|v| v.image_tag.clone())
2518            .unwrap_or_default(),
2519        image_digest: None,
2520        consistency: "kanidm-offline".to_string(),
2521        reason: "restore-safety".to_string(),
2522        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2523        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2524        result_path: "/run/kaniop-result/result.json".to_string(),
2525        max_concurrent_parts: 4,
2526        max_retries: 3,
2527    };
2528    let _ = manifest_key;
2529    let doc = OperationDocument {
2530        api_version: OPERATION_DOC_VERSION.to_string(),
2531        kind: "OperationDocument".to_string(),
2532        spec: OperationSpec::Upload(op),
2533    };
2534    serde_json::to_string(&doc)
2535        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2536}
2537
2538async fn validate_backup_ref(
2539    restore: &KanidmRestore,
2540    target: &Kanidm,
2541    ctx: &RestoreContext,
2542) -> Result<()> {
2543    let backup_ref = restore
2544        .spec
2545        .source
2546        .backup_ref
2547        .as_ref()
2548        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2549    let ns = restore.namespace().unwrap();
2550    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2551        .get(&backup_ref.name)
2552        .await
2553        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2554    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2555        return Err(Error::MissingData(format!(
2556            "KanidmBackup '{}' is not Ready (phase: {:?})",
2557            backup_ref.name,
2558            backup.status.as_ref().map(|s| &s.phase)
2559        )));
2560    }
2561    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2562        return Err(Error::MissingData(format!(
2563            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2564            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2565        )));
2566    }
2567    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2568        return Err(Error::MissingData(format!(
2569            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2570            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2571        )));
2572    }
2573    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2574        .get(&backup.spec.repository_ref.name)
2575        .await
2576        .map_err(|e| {
2577            Error::kube_error(
2578                "get",
2579                "KanidmBackupRepository",
2580                &ns,
2581                &backup.spec.repository_ref.name,
2582                e,
2583            )
2584        })?;
2585    if !has_accepted_condition(
2586        &repo
2587            .status
2588            .as_ref()
2589            .map(|s| &s.conditions)
2590            .cloned()
2591            .unwrap_or_default(),
2592    ) {
2593        return Err(Error::MissingData(format!(
2594            "KanidmBackupRepository '{}' configuration has not been accepted",
2595            backup.spec.repository_ref.name
2596        )));
2597    }
2598    validate_backup_compatibility(restore, target, &backup)?;
2599    Ok(())
2600}
2601
2602fn has_accepted_condition(conditions: &[Condition]) -> bool {
2603    conditions
2604        .iter()
2605        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2606}
2607
2608fn validate_backup_compatibility(
2609    restore: &KanidmRestore,
2610    target: &Kanidm,
2611    backup: &KanidmBackup,
2612) -> Result<()> {
2613    let status = backup.status.as_ref();
2614    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2615        let target_version = target
2616            .status
2617            .as_ref()
2618            .and_then(|s| s.version.as_ref())
2619            .map(|v| v.image_tag.as_str());
2620        if let Some(tv) = target_version {
2621            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2622                return Err(Error::MissingData(format!(
2623                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2624                )));
2625            }
2626        }
2627    }
2628    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2629        if !backup_digest.is_empty()
2630            && restore.spec.restore_image.contains('@')
2631            && !restore.spec.restore_image.contains(backup_digest)
2632        {
2633            return Err(Error::MissingData(format!(
2634                "restore image digest does not match backup image digest '{backup_digest}'"
2635            )));
2636        }
2637    }
2638    Ok(())
2639}
2640
2641#[allow(clippy::too_many_arguments)]
2642fn build_download_operation_doc(
2643    restore: &KanidmRestore,
2644    target: &Kanidm,
2645    manifest_key: &str,
2646    expected_backup_id: &str,
2647    bucket: &str,
2648    prefix: &str,
2649    endpoint: &str,
2650    region: &str,
2651    force_path_style: bool,
2652    insecure: bool,
2653    ca_bundle_path: Option<&str>,
2654    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2655) -> String {
2656    let enc_mode =
2657        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2658    let enc_key_id = encryption
2659        .and_then(|e| e.key_id.as_ref())
2660        .map(|s| serde_json::Value::String(s.clone()));
2661    serde_json::json!({
2662        "apiVersion": "backup.kaniop.rs/v1alpha1",
2663        "kind": "OperationDocument",
2664        "operation": "download",
2665        "manifestKey": manifest_key,
2666        "bucket": bucket,
2667        "prefix": prefix,
2668        "endpoint": endpoint,
2669        "region": region,
2670        "forcePathStyle": force_path_style,
2671        "insecure": insecure,
2672        "caBundlePath": ca_bundle_path,
2673        "expectedBackupId": expected_backup_id,
2674        "expectedKanidmUid": restore.spec.target_ref.uid,
2675        "expectedDomain": target.spec.domain,
2676        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2677        "resultPath": "/run/kaniop-result/result.json",
2678        "maxRetries": 3,
2679        "encryptionMode": enc_mode,
2680        "encryptionKeyId": enc_key_id,
2681    })
2682    .to_string()
2683}
2684
2685async fn ensure_operation_configmap(
2686    restore: &KanidmRestore,
2687    name: &str,
2688    data: &str,
2689    ns: &str,
2690    ctx: &RestoreContext,
2691) -> Result<()> {
2692    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2693    if api
2694        .get_opt(name)
2695        .await
2696        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2697        .is_some()
2698    {
2699        return Ok(());
2700    }
2701    let cm = ConfigMap {
2702        metadata: ObjectMeta {
2703            name: Some(name.to_string()),
2704            namespace: Some(ns.to_string()),
2705            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2706            ..Default::default()
2707        },
2708        data: Some(std::collections::BTreeMap::from([(
2709            "operation.json".to_string(),
2710            data.to_string(),
2711        )])),
2712        ..Default::default()
2713    };
2714    api.create(&PostParams::default(), &cm)
2715        .await
2716        .map(|_| ())
2717        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2718}
2719
2720#[cfg(test)]
2721mod tests {
2722    use super::{
2723        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2724        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2725        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2726        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2727        hardened_security_context, has_accepted_condition, is_remote_source,
2728        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2729        validate_safety_backup_config, validate_source,
2730    };
2731    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2732    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2733    use kube::api::ObjectMeta as ApiObjectMeta;
2734    use std::collections::BTreeMap;
2735
2736    fn make_restore(
2737        source: KanidmRestoreSource,
2738        safety: Option<SafetyBackupConfig>,
2739    ) -> KanidmRestore {
2740        KanidmRestore {
2741            metadata: ApiObjectMeta {
2742                name: Some("test-restore".to_string()),
2743                namespace: Some("default".to_string()),
2744                ..Default::default()
2745            },
2746            spec: KanidmRestoreSpec {
2747                target_ref: KanidmRestoreTargetRef {
2748                    name: "corp-idm".to_string(),
2749                    uid: "test-uid-123".to_string(),
2750                },
2751                source,
2752                restore_image: "kanidm/server@sha256:abc".to_string(),
2753                safety_backup: safety,
2754            },
2755            status: None,
2756        }
2757    }
2758
2759    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2760        super::super::crd::Kanidm {
2761            status: Some(super::super::crd::KanidmStatus {
2762                available_replicas: 1,
2763                replicas: 1,
2764                unavailable_replicas: 0,
2765                updated_replicas: 1,
2766                replica_statuses: vec![],
2767                replica_column: "1/1".to_string(),
2768                secret_name: None,
2769                version: Some(super::super::crd::KanidmVersionStatus {
2770                    image_tag: image_tag.to_string(),
2771                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2772                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2773                }),
2774                domain_appearance_image: None,
2775                mail_sender: None,
2776                conditions: None,
2777            }),
2778            ..Default::default()
2779        }
2780    }
2781
2782    #[test]
2783    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2784        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2785    }
2786
2787    #[test]
2788    fn completed_restore_sets_ready_condition() {
2789        let mut status = KanidmRestoreStatus {
2790            phase: KanidmRestorePhase::Completed,
2791            ..Default::default()
2792        };
2793        super::update_restore_conditions(&mut status, Some(7));
2794        assert!(status.conditions.iter().any(|condition| {
2795            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2796        }));
2797        assert!(!status.conditions.iter().any(|condition| {
2798            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2799        }));
2800    }
2801
2802    #[test]
2803    fn failed_restore_sets_failed_condition() {
2804        let mut status = KanidmRestoreStatus {
2805            phase: KanidmRestorePhase::Failed,
2806            message: Some("verification failed".to_string()),
2807            ..Default::default()
2808        };
2809        super::update_restore_conditions(&mut status, Some(8));
2810        assert!(status.conditions.iter().any(|condition| {
2811            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2812        }));
2813    }
2814
2815    #[test]
2816    fn restore_filename_is_confined_to_backup_directory() {
2817        assert!(safe_basename("backup.json.gz"));
2818        assert!(!safe_basename("../backup.json.gz"));
2819        assert!(!safe_basename("a/b"));
2820        assert!(!safe_basename(""));
2821    }
2822
2823    #[test]
2824    fn restore_image_rejects_latest_and_unpinned_names() {
2825        assert!(mutable_image("kanidm/server:latest"));
2826        assert!(mutable_image("kanidm/server"));
2827        assert!(!mutable_image("kanidm/server:1.10.0"));
2828        assert!(!mutable_image("kanidm/server@sha256:abc"));
2829    }
2830
2831    #[test]
2832    fn source_validation_rejects_both_local_and_backup_ref() {
2833        let source = KanidmRestoreSource {
2834            local: Some(super::KanidmRestoreLocalSource {
2835                file_name: "backup.json".to_string(),
2836            }),
2837            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2838                name: "some-backup".to_string(),
2839            }),
2840        };
2841        let restore = make_restore(source, None);
2842        assert!(validate_source(&restore).is_err());
2843    }
2844
2845    #[test]
2846    fn source_validation_rejects_empty_source() {
2847        let source = KanidmRestoreSource {
2848            local: None,
2849            backup_ref: None,
2850        };
2851        let restore = make_restore(source, None);
2852        assert!(validate_source(&restore).is_err());
2853    }
2854
2855    #[test]
2856    fn source_validation_accepts_local_source() {
2857        let source = KanidmRestoreSource {
2858            local: Some(super::KanidmRestoreLocalSource {
2859                file_name: "backup.json".to_string(),
2860            }),
2861            backup_ref: None,
2862        };
2863        let restore = make_restore(source, None);
2864        assert!(validate_source(&restore).is_ok());
2865    }
2866
2867    #[test]
2868    fn source_validation_accepts_backup_ref_source() {
2869        let source = KanidmRestoreSource {
2870            local: None,
2871            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2872                name: "corp-idm-019c7c76".to_string(),
2873            }),
2874        };
2875        let restore = make_restore(source, None);
2876        assert!(validate_source(&restore).is_ok());
2877    }
2878
2879    #[test]
2880    fn source_validation_rejects_empty_backup_ref_name() {
2881        let source = KanidmRestoreSource {
2882            local: None,
2883            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2884                name: "".to_string(),
2885            }),
2886        };
2887        let restore = make_restore(source, None);
2888        assert!(validate_source(&restore).is_err());
2889    }
2890
2891    #[test]
2892    fn is_remote_source_true_for_backup_ref() {
2893        let source = KanidmRestoreSource {
2894            local: None,
2895            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2896                name: "backup-1".to_string(),
2897            }),
2898        };
2899        let restore = make_restore(source, None);
2900        assert!(is_remote_source(&restore));
2901    }
2902
2903    #[test]
2904    fn is_remote_source_false_for_local() {
2905        let source = KanidmRestoreSource {
2906            local: Some(super::KanidmRestoreLocalSource {
2907                file_name: "backup.json".to_string(),
2908            }),
2909            backup_ref: None,
2910        };
2911        let restore = make_restore(source, None);
2912        assert!(!is_remote_source(&restore));
2913    }
2914
2915    #[test]
2916    fn requires_safety_backup_default_true() {
2917        let source = KanidmRestoreSource {
2918            local: None,
2919            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2920                name: "backup-1".to_string(),
2921            }),
2922        };
2923        let restore = make_restore(source, None);
2924        assert!(requires_safety_backup(&restore));
2925    }
2926
2927    #[test]
2928    fn requires_safety_backup_false_when_skip() {
2929        let source = KanidmRestoreSource {
2930            local: None,
2931            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2932                name: "backup-1".to_string(),
2933            }),
2934        };
2935        let safety = SafetyBackupConfig {
2936            repository_ref: Some(SafetyBackupRepositoryRef {
2937                name: "offsite".to_string(),
2938            }),
2939            skip: true,
2940        };
2941        let restore = make_restore(source, Some(safety));
2942        assert!(!requires_safety_backup(&restore));
2943    }
2944
2945    #[test]
2946    fn break_glass_validation_rejects_skip_without_reason() {
2947        let source = KanidmRestoreSource {
2948            local: None,
2949            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2950                name: "backup-1".to_string(),
2951            }),
2952        };
2953        let safety = SafetyBackupConfig {
2954            repository_ref: Some(SafetyBackupRepositoryRef {
2955                name: "offsite".to_string(),
2956            }),
2957            skip: true,
2958        };
2959        let mut restore = make_restore(source, Some(safety));
2960        restore.metadata.annotations = Some(BTreeMap::from([(
2961            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
2962            "commander@example.com".to_string(),
2963        )]));
2964        assert!(validate_safety_backup_config(&restore).is_err());
2965    }
2966
2967    #[test]
2968    fn break_glass_validation_rejects_skip_without_approver() {
2969        let source = KanidmRestoreSource {
2970            local: None,
2971            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2972                name: "backup-1".to_string(),
2973            }),
2974        };
2975        let safety = SafetyBackupConfig {
2976            repository_ref: Some(SafetyBackupRepositoryRef {
2977                name: "offsite".to_string(),
2978            }),
2979            skip: true,
2980        };
2981        let mut restore = make_restore(source, Some(safety));
2982        restore.metadata.annotations = Some(BTreeMap::from([(
2983            BREAK_GLASS_REASON_ANNOTATION.to_string(),
2984            "PVC is unreadable".to_string(),
2985        )]));
2986        assert!(validate_safety_backup_config(&restore).is_err());
2987    }
2988
2989    #[test]
2990    fn break_glass_validation_accepts_valid_annotations() {
2991        let source = KanidmRestoreSource {
2992            local: None,
2993            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2994                name: "backup-1".to_string(),
2995            }),
2996        };
2997        let safety = SafetyBackupConfig {
2998            repository_ref: Some(SafetyBackupRepositoryRef {
2999                name: "offsite".to_string(),
3000            }),
3001            skip: true,
3002        };
3003        let mut restore = make_restore(source, Some(safety));
3004        restore.metadata.annotations = Some(BTreeMap::from([
3005            (
3006                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3007                "PVC is unreadable".to_string(),
3008            ),
3009            (
3010                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3011                "commander@example.com".to_string(),
3012            ),
3013        ]));
3014        assert!(validate_safety_backup_config(&restore).is_ok());
3015    }
3016
3017    #[test]
3018    fn remote_source_requires_repository_ref_when_not_skipping() {
3019        let source = KanidmRestoreSource {
3020            local: None,
3021            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3022                name: "backup-1".to_string(),
3023            }),
3024        };
3025        let safety = SafetyBackupConfig {
3026            repository_ref: None,
3027            skip: false,
3028        };
3029        let restore = make_restore(source, Some(safety));
3030        assert!(validate_safety_backup_config(&restore).is_err());
3031    }
3032
3033    #[test]
3034    fn remote_source_accepts_repository_ref() {
3035        let source = KanidmRestoreSource {
3036            local: None,
3037            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3038                name: "backup-1".to_string(),
3039            }),
3040        };
3041        let safety = SafetyBackupConfig {
3042            repository_ref: Some(SafetyBackupRepositoryRef {
3043                name: "offsite".to_string(),
3044            }),
3045            skip: false,
3046        };
3047        let restore = make_restore(source, Some(safety));
3048        assert!(validate_safety_backup_config(&restore).is_ok());
3049    }
3050
3051    #[test]
3052    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3053        let phases = vec![
3054            KanidmRestorePhase::Pending,
3055            KanidmRestorePhase::Validating,
3056            KanidmRestorePhase::Quiescing,
3057            KanidmRestorePhase::SafetyBackup,
3058            KanidmRestorePhase::PreparingSource,
3059            KanidmRestorePhase::RestoringPrimary,
3060            KanidmRestorePhase::Verifying,
3061            KanidmRestorePhase::RebuildingReplicas,
3062            KanidmRestorePhase::Resuming,
3063            KanidmRestorePhase::Completed,
3064        ];
3065        assert_eq!(phases.len(), 10);
3066        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3067        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3068    }
3069
3070    #[test]
3071    fn status_default_has_empty_phase_timestamps() {
3072        let status = KanidmRestoreStatus::default();
3073        assert!(status.phase_timestamps.is_empty());
3074    }
3075
3076    #[test]
3077    fn status_default_has_empty_original_replicas() {
3078        let status = KanidmRestoreStatus::default();
3079        assert!(status.original_replicas.is_empty());
3080    }
3081
3082    #[test]
3083    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3084        let mut target = super::super::crd::Kanidm::default();
3085        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3086            run_as_non_root: Some(false),
3087            run_as_user: Some(389),
3088            run_as_group: Some(389),
3089            fs_group: Some(389),
3090            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3091            supplemental_groups: Some(vec![390, 391]),
3092            ..Default::default()
3093        });
3094
3095        let ctx = super::source_job_pod_security_context(&target);
3096
3097        assert_eq!(ctx.run_as_non_root, Some(false));
3098        assert_eq!(ctx.run_as_user, Some(389));
3099        assert_eq!(ctx.run_as_group, Some(389));
3100        assert_eq!(ctx.fs_group, Some(389));
3101        assert_eq!(
3102            ctx.fs_group_change_policy.as_deref(),
3103            Some("OnRootMismatch")
3104        );
3105        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3106        assert_eq!(
3107            ctx.seccomp_profile
3108                .as_ref()
3109                .map(|profile| profile.type_.as_str()),
3110            Some("RuntimeDefault")
3111        );
3112    }
3113
3114    #[test]
3115    fn hardened_security_context_drops_all_capabilities() {
3116        let ctx = hardened_security_context();
3117        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3118        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3119        assert_eq!(ctx.run_as_non_root, Some(true));
3120        let caps = ctx.capabilities.unwrap();
3121        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3122        assert_eq!(
3123            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3124            Some("RuntimeDefault")
3125        );
3126    }
3127
3128    #[test]
3129    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3130        let ctx = kanidm_job_security_context();
3131        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3132        assert!(ctx.run_as_non_root.is_none());
3133        assert!(ctx.read_only_root_filesystem.is_none());
3134        let caps = ctx.capabilities.unwrap();
3135        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3136        assert_eq!(
3137            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3138            Some("RuntimeDefault")
3139        );
3140    }
3141
3142    #[test]
3143    fn safety_backup_job_name_is_deterministic() {
3144        let source = KanidmRestoreSource {
3145            local: Some(super::KanidmRestoreLocalSource {
3146                file_name: "backup.json".to_string(),
3147            }),
3148            backup_ref: None,
3149        };
3150        let restore = make_restore(source, None);
3151        let name1 = super::safety_backup_job_name(&restore);
3152        let name2 = super::safety_backup_job_name(&restore);
3153        assert_eq!(name1, name2);
3154        assert_eq!(name1, "test-restore-safety-backup");
3155    }
3156
3157    #[test]
3158    fn source_prep_job_name_is_deterministic() {
3159        let source = KanidmRestoreSource {
3160            local: None,
3161            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3162                name: "backup-1".to_string(),
3163            }),
3164        };
3165        let restore = make_restore(source, None);
3166        let name = super::source_prep_job_name(&restore);
3167        assert_eq!(name, "test-restore-source-prep");
3168    }
3169
3170    #[test]
3171    fn break_glass_condition_is_recorded_in_status() {
3172        let mut status = KanidmRestoreStatus::default();
3173        let condition = super::restore_condition(
3174            &status.conditions,
3175            CONDITION_BREAK_GLASS,
3176            CONDITION_TRUE,
3177            "BreakGlassOverride",
3178            "Safety backup skipped",
3179            Some(1),
3180        );
3181        status.conditions.push(condition);
3182        assert!(status.conditions.iter().any(|c| {
3183            c.type_ == CONDITION_BREAK_GLASS
3184                && c.status == CONDITION_TRUE
3185                && c.reason == "BreakGlassOverride"
3186        }));
3187    }
3188
3189    #[test]
3190    fn replica_count_entry_serialization() {
3191        let entry = ReplicaCountEntry {
3192            group: "primary".to_string(),
3193            replicas: 3,
3194        };
3195        let json = serde_json::to_string(&entry).unwrap();
3196        assert!(json.contains("\"group\":\"primary\""));
3197        assert!(json.contains("\"replicas\":3"));
3198    }
3199
3200    #[test]
3201    fn local_source_does_not_require_repository_ref() {
3202        let source = KanidmRestoreSource {
3203            local: Some(super::KanidmRestoreLocalSource {
3204                file_name: "backup.json".to_string(),
3205            }),
3206            backup_ref: None,
3207        };
3208        let safety = SafetyBackupConfig {
3209            repository_ref: None,
3210            skip: false,
3211        };
3212        let restore = make_restore(source, Some(safety));
3213        assert!(validate_safety_backup_config(&restore).is_ok());
3214    }
3215
3216    #[test]
3217    fn skip_without_break_glass_annotations_is_rejected() {
3218        let source = KanidmRestoreSource {
3219            local: Some(super::KanidmRestoreLocalSource {
3220                file_name: "backup.json".to_string(),
3221            }),
3222            backup_ref: None,
3223        };
3224        let safety = SafetyBackupConfig {
3225            repository_ref: None,
3226            skip: true,
3227        };
3228        let restore = make_restore(source, Some(safety));
3229        assert!(validate_safety_backup_config(&restore).is_err());
3230    }
3231
3232    #[test]
3233    fn has_accepted_condition_true() {
3234        let conditions = vec![Condition {
3235            type_: "Ready".to_string(),
3236            status: "True".to_string(),
3237            reason: "Accepted".to_string(),
3238            message: String::new(),
3239            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3240                k8s_openapi::jiff::Timestamp::now(),
3241            ),
3242            observed_generation: None,
3243        }];
3244        assert!(has_accepted_condition(&conditions));
3245    }
3246
3247    #[test]
3248    fn has_accepted_condition_false_when_not_accepted_reason() {
3249        let conditions = vec![Condition {
3250            type_: "Ready".to_string(),
3251            status: "True".to_string(),
3252            reason: "SomeOtherReason".to_string(),
3253            message: String::new(),
3254            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3255                k8s_openapi::jiff::Timestamp::now(),
3256            ),
3257            observed_generation: None,
3258        }];
3259        assert!(!has_accepted_condition(&conditions));
3260    }
3261
3262    #[test]
3263    fn has_accepted_condition_false_when_status_false() {
3264        let conditions = vec![Condition {
3265            type_: "Ready".to_string(),
3266            status: "False".to_string(),
3267            reason: "Accepted".to_string(),
3268            message: String::new(),
3269            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3270                k8s_openapi::jiff::Timestamp::now(),
3271            ),
3272            observed_generation: None,
3273        }];
3274        assert!(!has_accepted_condition(&conditions));
3275    }
3276
3277    #[test]
3278    fn has_accepted_condition_empty() {
3279        assert!(!has_accepted_condition(&[]));
3280    }
3281
3282    #[test]
3283    fn safety_backup_upload_operation_doc_is_valid() {
3284        let source = KanidmRestoreSource {
3285            local: None,
3286            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3287                name: "backup-1".to_string(),
3288            }),
3289        };
3290        let mut restore = make_restore(source, None);
3291        restore.metadata.namespace = Some("test-ns".to_string());
3292        let target = super::super::crd::Kanidm::default();
3293        let doc_str = super::build_safety_upload_operation_doc(
3294            &restore,
3295            &target,
3296            "test-backup-id",
3297            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3298            "my-bucket",
3299            "prod",
3300            "https://s3.example.com",
3301            "us-east-1",
3302            false,
3303            false,
3304            None,
3305            None,
3306        )
3307        .unwrap();
3308        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3309        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3310        assert_eq!(parsed["kind"], "OperationDocument");
3311        assert_eq!(parsed["operation"], "upload");
3312        assert_eq!(parsed["bucket"], "my-bucket");
3313        assert_eq!(parsed["prefix"], "prod");
3314        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3315        assert_eq!(parsed["region"], "us-east-1");
3316        assert_eq!(parsed["backupId"], "test-backup-id");
3317        assert_eq!(parsed["consistency"], "kanidm-offline");
3318        assert_eq!(parsed["reason"], "restore-safety");
3319        assert_eq!(
3320            parsed["payloadPath"],
3321            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3322        );
3323        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3324    }
3325
3326    #[test]
3327    fn download_operation_doc_uses_real_repository_data() {
3328        let source = KanidmRestoreSource {
3329            local: None,
3330            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3331                name: "backup-1".to_string(),
3332            }),
3333        };
3334        let restore = make_restore(source, None);
3335        let target = super::super::crd::Kanidm::default();
3336        let doc_str = super::build_download_operation_doc(
3337            &restore,
3338            &target,
3339            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3340            "019c7c76-f423-7a12-8f41-2bea7588a303",
3341            "real-bucket",
3342            "real-prefix",
3343            "https://real-endpoint.com",
3344            "eu-west-1",
3345            true,
3346            false,
3347            None,
3348            None,
3349        );
3350        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3351        assert_eq!(parsed["bucket"], "real-bucket");
3352        assert_eq!(parsed["prefix"], "real-prefix");
3353        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3354        assert_eq!(parsed["region"], "eu-west-1");
3355        assert_eq!(parsed["forcePathStyle"], true);
3356        assert_eq!(
3357            parsed["manifestKey"],
3358            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3359        );
3360        assert_eq!(
3361            parsed["expectedBackupId"],
3362            "019c7c76-f423-7a12-8f41-2bea7588a303"
3363        );
3364    }
3365
3366    #[test]
3367    fn safety_backup_ref_is_set_after_verification() {
3368        let status = KanidmRestoreStatus {
3369            phase: KanidmRestorePhase::SafetyBackup,
3370            safety_backup_job_name: Some("test-safety-backup".to_string()),
3371            safety_backup_ref: Some("safety-test-restore".to_string()),
3372            safety_backup_expected_backup_id: Some(
3373                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3374            ),
3375            safety_backup_manifest_key: Some(
3376                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3377            ),
3378            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3379            ..Default::default()
3380        };
3381        assert!(status.safety_backup_ref.is_some());
3382        assert!(
3383            status
3384                .safety_backup_ref
3385                .as_ref()
3386                .unwrap()
3387                .starts_with("safety-")
3388        );
3389        assert!(status.safety_backup_expected_backup_id.is_some());
3390        assert!(status.safety_backup_manifest_key.is_some());
3391        assert!(status.safety_backup_payload_sha256.is_some());
3392    }
3393
3394    #[test]
3395    fn database_mutation_not_started_without_safety_backup_ref() {
3396        let status = KanidmRestoreStatus {
3397            phase: KanidmRestorePhase::PreparingSource,
3398            safety_backup_ref: None,
3399            database_mutation_started: false,
3400            ..Default::default()
3401        };
3402        assert!(!status.database_mutation_started);
3403        assert!(status.safety_backup_ref.is_none());
3404    }
3405
3406    #[test]
3407    fn pre_boundary_failure_does_not_set_mutation_started() {
3408        let status = KanidmRestoreStatus {
3409            phase: KanidmRestorePhase::Failed,
3410            message: Some(
3411                "safety backup job failed; target restored to original state".to_string(),
3412            ),
3413            database_mutation_started: false,
3414            ..Default::default()
3415        };
3416        assert!(!status.database_mutation_started);
3417    }
3418
3419    #[test]
3420    fn shared_volume_constants_are_distinct() {
3421        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3422        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3423    }
3424
3425    #[test]
3426    fn validate_backup_compatibility_matching_versions_passes() {
3427        let source = KanidmRestoreSource {
3428            local: None,
3429            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3430                name: "backup-1".to_string(),
3431            }),
3432        };
3433        let restore = make_restore(source, None);
3434        let target = kanidm_with_version("1.10.4");
3435        let backup = kaniop_backup_core::crd::KanidmBackup {
3436            metadata: kube::api::ObjectMeta::default(),
3437            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3438                backup_id: "id".to_string(),
3439                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3440                    name: "k".to_string(),
3441                    uid: "uid".to_string(),
3442                },
3443                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3444                    name: "repo".to_string(),
3445                },
3446                manifest_key: "key".to_string(),
3447            },
3448            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3449                kanidm_version: Some("1.10.4".to_string()),
3450                ..Default::default()
3451            }),
3452        };
3453        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3454    }
3455
3456    #[test]
3457    fn validate_backup_compatibility_version_mismatch_fails() {
3458        let source = KanidmRestoreSource {
3459            local: None,
3460            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3461                name: "backup-1".to_string(),
3462            }),
3463        };
3464        let restore = make_restore(source, None);
3465        let target = kanidm_with_version("1.11.0");
3466        let backup = kaniop_backup_core::crd::KanidmBackup {
3467            metadata: kube::api::ObjectMeta::default(),
3468            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3469                backup_id: "id".to_string(),
3470                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3471                    name: "k".to_string(),
3472                    uid: "uid".to_string(),
3473                },
3474                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3475                    name: "repo".to_string(),
3476                },
3477                manifest_key: "key".to_string(),
3478            },
3479            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3480                kanidm_version: Some("1.10.4".to_string()),
3481                ..Default::default()
3482            }),
3483        };
3484        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3485    }
3486
3487    #[test]
3488    fn validate_backup_compatibility_no_backup_version_passes() {
3489        let source = KanidmRestoreSource {
3490            local: None,
3491            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3492                name: "backup-1".to_string(),
3493            }),
3494        };
3495        let restore = make_restore(source, None);
3496        let target = super::super::crd::Kanidm::default();
3497        let backup = kaniop_backup_core::crd::KanidmBackup {
3498            metadata: kube::api::ObjectMeta::default(),
3499            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3500                backup_id: "id".to_string(),
3501                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3502                    name: "k".to_string(),
3503                    uid: "uid".to_string(),
3504                },
3505                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3506                    name: "repo".to_string(),
3507                },
3508                manifest_key: "key".to_string(),
3509            },
3510            status: None,
3511        };
3512        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3513    }
3514
3515    #[test]
3516    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3517        let source = KanidmRestoreSource {
3518            local: None,
3519            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3520                name: "backup-1".to_string(),
3521            }),
3522        };
3523        let mut restore = make_restore(source, None);
3524        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3525        let target = super::super::crd::Kanidm::default();
3526        let backup = kaniop_backup_core::crd::KanidmBackup {
3527            metadata: kube::api::ObjectMeta::default(),
3528            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3529                backup_id: "id".to_string(),
3530                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3531                    name: "k".to_string(),
3532                    uid: "uid".to_string(),
3533                },
3534                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3535                    name: "repo".to_string(),
3536                },
3537                manifest_key: "key".to_string(),
3538            },
3539            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3540                image_digest: Some("sha256:originaldigest".to_string()),
3541                ..Default::default()
3542            }),
3543        };
3544        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3545    }
3546
3547    #[test]
3548    fn validate_backup_compatibility_image_digest_match_passes() {
3549        let source = KanidmRestoreSource {
3550            local: None,
3551            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3552                name: "backup-1".to_string(),
3553            }),
3554        };
3555        let mut restore = make_restore(source, None);
3556        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3557        let target = super::super::crd::Kanidm::default();
3558        let backup = kaniop_backup_core::crd::KanidmBackup {
3559            metadata: kube::api::ObjectMeta::default(),
3560            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3561                backup_id: "id".to_string(),
3562                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3563                    name: "k".to_string(),
3564                    uid: "uid".to_string(),
3565                },
3566                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3567                    name: "repo".to_string(),
3568                },
3569                manifest_key: "key".to_string(),
3570            },
3571            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3572                image_digest: Some("sha256:abc123".to_string()),
3573                ..Default::default()
3574            }),
3575        };
3576        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3577    }
3578
3579    #[test]
3580    fn safety_backup_id_is_deterministic() {
3581        let source = KanidmRestoreSource {
3582            local: None,
3583            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3584                name: "backup-1".to_string(),
3585            }),
3586        };
3587        let restore = make_restore(source, None);
3588        let id1 = super::compute_safety_backup_id(&restore);
3589        let id2 = super::compute_safety_backup_id(&restore);
3590        assert_eq!(id1, id2);
3591        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3592    }
3593
3594    #[test]
3595    fn safety_backup_id_differs_for_different_restores() {
3596        let source1 = KanidmRestoreSource {
3597            local: None,
3598            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3599                name: "backup-1".to_string(),
3600            }),
3601        };
3602        let restore1 = make_restore(source1, None);
3603        let mut restore2 = make_restore(
3604            KanidmRestoreSource {
3605                local: None,
3606                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3607                    name: "backup-2".to_string(),
3608                }),
3609            },
3610            None,
3611        );
3612        restore2.metadata.name = Some("different-restore".to_string());
3613        let id1 = super::compute_safety_backup_id(&restore1);
3614        let id2 = super::compute_safety_backup_id(&restore2);
3615        assert_ne!(id1, id2);
3616    }
3617
3618    #[test]
3619    fn safety_manifest_key_is_deterministic() {
3620        let source = KanidmRestoreSource {
3621            local: None,
3622            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3623                name: "backup-1".to_string(),
3624            }),
3625        };
3626        let restore = make_restore(source, None);
3627        let key1 = super::compute_safety_manifest_key(&restore);
3628        let key2 = super::compute_safety_manifest_key(&restore);
3629        assert_eq!(key1, key2);
3630        assert!(key1.contains("manifest.json"));
3631        assert!(key1.starts_with("v1/tenants/"));
3632    }
3633
3634    #[test]
3635    fn safety_backup_id_survives_restart() {
3636        let source = KanidmRestoreSource {
3637            local: None,
3638            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3639                name: "backup-1".to_string(),
3640            }),
3641        };
3642        let mut restore = make_restore(source, None);
3643        let original_id = super::compute_safety_backup_id(&restore);
3644        restore.status = Some(KanidmRestoreStatus {
3645            phase: KanidmRestorePhase::SafetyBackup,
3646            safety_backup_expected_backup_id: Some(original_id.clone()),
3647            ..Default::default()
3648        });
3649        let recomputed_id = super::compute_safety_backup_id(&restore);
3650        assert_eq!(original_id, recomputed_id);
3651    }
3652
3653    #[test]
3654    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3655        let source = KanidmRestoreSource {
3656            local: None,
3657            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3658                name: "backup-1".to_string(),
3659            }),
3660        };
3661        let restore = make_restore(source, None);
3662        assert!(super::requires_safety_backup(&restore));
3663        let status_without_ref = KanidmRestoreStatus {
3664            phase: KanidmRestorePhase::RestoringPrimary,
3665            safety_backup_ref: None,
3666            database_mutation_started: false,
3667            ..Default::default()
3668        };
3669        assert!(status_without_ref.safety_backup_ref.is_none());
3670        assert!(!status_without_ref.database_mutation_started);
3671    }
3672
3673    #[test]
3674    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3675        let status_with_ref = KanidmRestoreStatus {
3676            phase: KanidmRestorePhase::RestoringPrimary,
3677            safety_backup_ref: Some("safety-test-restore".to_string()),
3678            safety_backup_manifest_key: Some(
3679                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3680            ),
3681            safety_backup_payload_sha256: Some("abc123".to_string()),
3682            database_mutation_started: false,
3683            ..Default::default()
3684        };
3685        assert!(status_with_ref.safety_backup_ref.is_some());
3686        assert!(!status_with_ref.database_mutation_started);
3687    }
3688
3689    #[test]
3690    fn result_document_verification_rejects_missing_backup_id() {
3691        let result_json = r#"{
3692            "apiVersion": "backup.kaniop.rs/v1alpha1",
3693            "kind": "ResultDocument",
3694            "operation": "upload",
3695            "success": true,
3696            "exitCode": "success",
3697            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3698            "payloadSha256": "abc123"
3699        }"#;
3700        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3701        assert!(result.backup_id.is_none());
3702    }
3703
3704    #[test]
3705    fn result_document_verification_rejects_missing_manifest_key() {
3706        let result_json = r#"{
3707            "apiVersion": "backup.kaniop.rs/v1alpha1",
3708            "kind": "ResultDocument",
3709            "operation": "upload",
3710            "success": true,
3711            "exitCode": "success",
3712            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3713            "payloadSha256": "abc123"
3714        }"#;
3715        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3716        assert!(result.manifest_key.is_none());
3717    }
3718
3719    #[test]
3720    fn result_document_verification_rejects_missing_payload_sha256() {
3721        let result_json = r#"{
3722            "apiVersion": "backup.kaniop.rs/v1alpha1",
3723            "kind": "ResultDocument",
3724            "operation": "upload",
3725            "success": true,
3726            "exitCode": "success",
3727            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3728            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3729        }"#;
3730        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3731        assert!(result.payload_sha256.is_none());
3732    }
3733
3734    #[test]
3735    fn result_document_verification_rejects_failure_exit_code() {
3736        let result_json = r#"{
3737            "apiVersion": "backup.kaniop.rs/v1alpha1",
3738            "kind": "ResultDocument",
3739            "operation": "upload",
3740            "success": false,
3741            "exitCode": "integrity",
3742            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3743            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3744            "payloadSha256": "abc123"
3745        }"#;
3746        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3747        assert!(!result.success);
3748        assert_eq!(
3749            result.exit_code,
3750            kaniop_backup_core::result::ExitCode::Integrity
3751        );
3752    }
3753
3754    #[test]
3755    fn result_document_verification_rejects_wrong_operation() {
3756        let result_json = r#"{
3757            "apiVersion": "backup.kaniop.rs/v1alpha1",
3758            "kind": "ResultDocument",
3759            "operation": "download",
3760            "success": true,
3761            "exitCode": "success",
3762            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3763            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3764            "payloadSha256": "abc123"
3765        }"#;
3766        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3767        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3768    }
3769
3770    #[test]
3771    fn result_document_verification_accepts_valid_document() {
3772        let result_json = r#"{
3773            "apiVersion": "backup.kaniop.rs/v1alpha1",
3774            "kind": "ResultDocument",
3775            "operation": "upload",
3776            "success": true,
3777            "exitCode": "success",
3778            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3779            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3780            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3781        }"#;
3782        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3783        assert!(result.success);
3784        assert_eq!(
3785            result.exit_code,
3786            kaniop_backup_core::result::ExitCode::Success
3787        );
3788        assert_eq!(result.operation, "upload");
3789        assert!(result.backup_id.is_some());
3790        assert!(result.manifest_key.is_some());
3791        assert!(result.payload_sha256.is_some());
3792    }
3793
3794    #[test]
3795    fn termination_message_path_constant_is_valid() {
3796        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3797        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3798    }
3799
3800    #[test]
3801    fn status_tracks_safety_backup_evidence() {
3802        let status = KanidmRestoreStatus {
3803            phase: KanidmRestorePhase::PreparingSource,
3804            safety_backup_ref: Some("safety-test-restore".to_string()),
3805            safety_backup_expected_backup_id: Some(
3806                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3807            ),
3808            safety_backup_manifest_key: Some(
3809                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3810            ),
3811            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3812            ..Default::default()
3813        };
3814        assert!(status.safety_backup_ref.is_some());
3815        assert!(status.safety_backup_expected_backup_id.is_some());
3816        assert!(status.safety_backup_manifest_key.is_some());
3817        assert!(status.safety_backup_payload_sha256.is_some());
3818    }
3819
3820    #[test]
3821    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3822        let invalid_json = "not valid json";
3823        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3824        assert!(result.is_err());
3825    }
3826
3827    #[test]
3828    fn fail_closed_semantics_preserved_on_oversized_result() {
3829        let oversized_json = format!(
3830            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3831            kaniop_backup_core::result::RESULT_DOC_VERSION,
3832            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3833        );
3834        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3835        assert!(result.is_err());
3836    }
3837
3838    #[test]
3839    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3840        let source = KanidmRestoreSource {
3841            local: None,
3842            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3843                name: "backup-1".to_string(),
3844            }),
3845        };
3846        let mut restore = make_restore(source, None);
3847        restore.metadata.namespace = Some("test-ns".to_string());
3848        let target = super::super::crd::Kanidm::default();
3849        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3850        let doc_str = super::build_safety_upload_operation_doc(
3851            &restore,
3852            &target,
3853            "test-backup-id",
3854            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3855            "my-bucket",
3856            "prod",
3857            "https://s3.example.com",
3858            "us-east-1",
3859            false,
3860            false,
3861            Some(&ca_path),
3862            None,
3863        )
3864        .unwrap();
3865        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3866        assert_eq!(parsed["caBundlePath"], ca_path);
3867    }
3868
3869    #[test]
3870    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3871        let source = KanidmRestoreSource {
3872            local: None,
3873            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3874                name: "backup-1".to_string(),
3875            }),
3876        };
3877        let mut restore = make_restore(source, None);
3878        restore.metadata.namespace = Some("test-ns".to_string());
3879        let target = super::super::crd::Kanidm::default();
3880        let doc_str = super::build_safety_upload_operation_doc(
3881            &restore,
3882            &target,
3883            "test-backup-id",
3884            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3885            "my-bucket",
3886            "prod",
3887            "https://s3.example.com",
3888            "us-east-1",
3889            false,
3890            false,
3891            None,
3892            None,
3893        )
3894        .unwrap();
3895        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3896        assert!(parsed["caBundlePath"].is_null());
3897    }
3898
3899    #[test]
3900    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3901        let source = KanidmRestoreSource {
3902            local: None,
3903            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3904                name: "backup-1".to_string(),
3905            }),
3906        };
3907        let restore = make_restore(source, None);
3908        let target = super::super::crd::Kanidm::default();
3909        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3910        let doc_str = super::build_download_operation_doc(
3911            &restore,
3912            &target,
3913            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3914            "019c7c76-f423-7a12-8f41-2bea7588a303",
3915            "real-bucket",
3916            "real-prefix",
3917            "https://real-endpoint.com",
3918            "eu-west-1",
3919            true,
3920            false,
3921            Some(&ca_path),
3922            None,
3923        );
3924        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3925        assert_eq!(parsed["caBundlePath"], ca_path);
3926    }
3927
3928    #[test]
3929    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3930        let source = KanidmRestoreSource {
3931            local: None,
3932            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3933                name: "backup-1".to_string(),
3934            }),
3935        };
3936        let restore = make_restore(source, None);
3937        let target = super::super::crd::Kanidm::default();
3938        let doc_str = super::build_download_operation_doc(
3939            &restore,
3940            &target,
3941            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3942            "019c7c76-f423-7a12-8f41-2bea7588a303",
3943            "real-bucket",
3944            "real-prefix",
3945            "https://real-endpoint.com",
3946            "eu-west-1",
3947            true,
3948            false,
3949            None,
3950            None,
3951        );
3952        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3953        assert!(parsed["caBundlePath"].is_null());
3954    }
3955
3956    #[test]
3957    fn source_prep_result_operation_is_download() {
3958        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3959        assert_ne!(
3960            super::SOURCE_PREP_RESULT_OPERATION,
3961            super::SAFETY_BACKUP_RESULT_OPERATION
3962        );
3963    }
3964
3965    #[test]
3966    fn download_result_document_verification_accepts_valid_document() {
3967        let result_json = r#"{
3968            "apiVersion": "backup.kaniop.rs/v1alpha1",
3969            "kind": "ResultDocument",
3970            "operation": "download",
3971            "success": true,
3972            "exitCode": "success",
3973            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3974            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3975            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3976        }"#;
3977        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3978        assert!(result.success);
3979        assert_eq!(
3980            result.exit_code,
3981            kaniop_backup_core::result::ExitCode::Success
3982        );
3983        assert_eq!(result.operation, "download");
3984        assert_eq!(
3985            result.backup_id.as_deref(),
3986            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
3987        );
3988        assert!(result.manifest_key.is_some());
3989        assert!(result.payload_sha256.is_some());
3990    }
3991
3992    #[test]
3993    fn download_result_document_rejects_wrong_operation() {
3994        let result_json = r#"{
3995            "apiVersion": "backup.kaniop.rs/v1alpha1",
3996            "kind": "ResultDocument",
3997            "operation": "upload",
3998            "success": true,
3999            "exitCode": "success",
4000            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4001            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4002            "payloadSha256": "abc123"
4003        }"#;
4004        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4005        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4006    }
4007
4008    #[test]
4009    fn download_result_document_rejects_failure_success_field() {
4010        let result_json = r#"{
4011            "apiVersion": "backup.kaniop.rs/v1alpha1",
4012            "kind": "ResultDocument",
4013            "operation": "download",
4014            "success": false,
4015            "exitCode": "retryable",
4016            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018            "payloadSha256": "abc123"
4019        }"#;
4020        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021        assert!(!result.success);
4022        assert_ne!(
4023            result.exit_code,
4024            kaniop_backup_core::result::ExitCode::Success
4025        );
4026    }
4027
4028    #[test]
4029    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4030        let status = KanidmRestoreStatus {
4031            phase: KanidmRestorePhase::PreparingSource,
4032            database_mutation_started: false,
4033            ..Default::default()
4034        };
4035        assert!(!status.database_mutation_started);
4036        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4037    }
4038
4039    #[test]
4040    fn backup_job_volume_size_default_is_10gi() {
4041        let qty = crate::controller::backup_job_volume_size();
4042        assert_eq!(qty, Quantity("10Gi".to_string()));
4043    }
4044
4045    #[test]
4046    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4047        use kaniop_backup_core::auth::build_encryption_env_vars;
4048        use kaniop_backup_core::crd::SecretRef;
4049
4050        let key_ref = SecretRef {
4051            name: "kek-secret".to_string(),
4052        };
4053        let env_vars = build_encryption_env_vars(Some(&key_ref));
4054        assert_eq!(env_vars.len(), 1);
4055        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4056        let secret_ref = env_vars[0]
4057            .value_from
4058            .as_ref()
4059            .unwrap()
4060            .secret_key_ref
4061            .as_ref()
4062            .unwrap();
4063        assert_eq!(secret_ref.name, "kek-secret");
4064        assert_eq!(secret_ref.key, "encryption-key");
4065    }
4066
4067    #[test]
4068    fn encryption_env_vars_absent_when_no_key_ref() {
4069        use kaniop_backup_core::auth::build_encryption_env_vars;
4070
4071        let env_vars = build_encryption_env_vars(None);
4072        assert!(
4073            env_vars.is_empty(),
4074            "encryption env vars must be empty when key_ref is None"
4075        );
4076    }
4077
4078    #[test]
4079    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4080        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4081
4082        let source = KanidmRestoreSource {
4083            local: None,
4084            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4085                name: "backup-1".to_string(),
4086            }),
4087        };
4088        let mut restore = make_restore(source, None);
4089        restore.metadata.namespace = Some("test-ns".to_string());
4090        let target = super::super::crd::Kanidm::default();
4091        let encryption = RepositoryEncryption {
4092            mode: EncryptionMode::ClientSide,
4093            key_id: None,
4094            key_ref: Some(SecretRef {
4095                name: "kek-secret".to_string(),
4096            }),
4097        };
4098        let doc_str = super::build_safety_upload_operation_doc(
4099            &restore,
4100            &target,
4101            "test-backup-id",
4102            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4103            "my-bucket",
4104            "prod",
4105            "https://s3.example.com",
4106            "us-east-1",
4107            false,
4108            false,
4109            None,
4110            Some(&encryption),
4111        )
4112        .unwrap();
4113        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4114        assert_eq!(parsed["encryptionMode"], "clientSide");
4115        assert!(parsed["encryptionKeyId"].is_null());
4116    }
4117
4118    #[test]
4119    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4120        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4121
4122        let source = KanidmRestoreSource {
4123            local: None,
4124            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4125                name: "backup-1".to_string(),
4126            }),
4127        };
4128        let restore = make_restore(source, None);
4129        let target = super::super::crd::Kanidm::default();
4130        let encryption = RepositoryEncryption {
4131            mode: EncryptionMode::ProviderKms,
4132            key_id: Some("alias/kaniop-backups".to_string()),
4133            key_ref: None,
4134        };
4135        let doc_str = super::build_download_operation_doc(
4136            &restore,
4137            &target,
4138            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4139            "019c7c76-f423-7a12-8f41-2bea7588a303",
4140            "real-bucket",
4141            "real-prefix",
4142            "https://real-endpoint.com",
4143            "eu-west-1",
4144            true,
4145            false,
4146            None,
4147            Some(&encryption),
4148        );
4149        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4150        assert_eq!(parsed["encryptionMode"], "providerKms");
4151        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4152    }
4153}