Skip to main content

kaniop_operator/kanidm/restore/
legacy.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76    feature = "schemars",
77    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80    category = "kaniop",
81    group = "kaniop.rs",
82    version = "v1beta1",
83    kind = "KanidmRestore",
84    plural = "kanidmrestores",
85    singular = "kanidmrestore",
86    shortname = "idmrestore",
87    namespaced,
88    status = "KanidmRestoreStatus",
89    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95    pub target_ref: KanidmRestoreTargetRef,
96    pub source: KanidmRestoreSource,
97    pub restore_image: String,
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106    pub name: String,
107    pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113    feature = "schemars",
114    schemars(extend("x-kubernetes-validations" = [
115        {
116            "message": "exactly one of local or backupRef must be set",
117            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118        }
119    ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123    /// Local file source. Mutually exclusive with `backupRef`.
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub local: Option<KanidmRestoreLocalSource>,
126    /// Remote cataloged backup reference. Mutually exclusive with `local`.
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135    /// Basename of a backup below /data. Paths and traversal are rejected.
136    pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
144    pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub repository_ref: Option<SafetyBackupRepositoryRef>,
154    /// Skip the mandatory safety backup. Requires break-glass annotations.
155    #[serde(default)]
156    pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163    pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169    #[default]
170    Pending,
171    Validating,
172    Quiescing,
173    SafetyBackup,
174    PreparingSource,
175    RestoringPrimary,
176    Verifying,
177    RebuildingReplicas,
178    Resuming,
179    Completed,
180    Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187    #[serde(skip_serializing_if = "Option::is_none")]
188    pub observed_generation: Option<i64>,
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_target_uid: Option<String>,
191    #[serde(default)]
192    pub phase: KanidmRestorePhase,
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub restore_job_name: Option<String>,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub verify_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub safety_backup_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub source_prep_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub safety_backup_ref: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_expected_backup_id: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_manifest_key: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_payload_sha256: Option<String>,
209    #[serde(default)]
210    pub replicas_cleared: bool,
211    #[serde(default)]
212    pub certificates_cleared: bool,
213    #[serde(default)]
214    pub database_mutation_started: bool,
215    #[serde(default, skip_serializing_if = "Vec::is_empty")]
216    pub original_replicas: Vec<ReplicaCountEntry>,
217    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218    pub phase_timestamps: std::collections::BTreeMap<String, String>,
219    #[serde(default, skip_serializing_if = "Vec::is_empty")]
220    #[cfg_attr(
221        feature = "schemars",
222        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223    )]
224    pub conditions: Vec<Condition>,
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233    pub group: String,
234    pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239    attempts: Counter<u64>,
240    outcomes: Counter<u64>,
241    duration_seconds: Histogram<f64>,
242    break_glass_total: Counter<u64>,
243    safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247    fn new() -> Self {
248        let meter = global::meter("kaniop");
249        Self {
250            attempts: meter
251                .u64_counter("kaniop_restore_attempts_total")
252                .with_description("Number of Kanidm restore attempts started")
253                .build(),
254            outcomes: meter
255                .u64_counter("kaniop_restore_outcomes_total")
256                .with_description("Number of terminal Kanidm restore outcomes")
257                .build(),
258            duration_seconds: meter
259                .f64_histogram("kaniop_restore_duration_seconds")
260                .with_description("Kanidm restore duration from object creation to terminal phase")
261                .with_unit("s")
262                .build(),
263            break_glass_total: meter
264                .u64_counter("kaniop_restore_break_glass_total")
265                .with_description("Number of break-glass safety backup overrides")
266                .build(),
267            safety_backup_duration_seconds: meter
268                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269                .with_description("Duration of pre-restore safety backup creation")
270                .with_unit("s")
271                .build(),
272        }
273    }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278    client: Client,
279    recorder: Recorder,
280    metrics: RestoreMetrics,
281}
282
283async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
284    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
285    match phase {
286        KanidmRestorePhase::Pending => {
287            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
288            Ok(Action::requeue(REQUEUE))
289        }
290        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
291            Ok(target) => {
292                mark_restoring(&restore, &target, &ctx).await?;
293                let mut status = restore.status.clone().unwrap_or_default();
294                status.observed_target_uid = target.uid();
295                status.original_replicas = capture_original_replicas(&target);
296                status.phase = KanidmRestorePhase::Quiescing;
297                status.message = None;
298                patch_status(&restore, &ctx, status).await?;
299                Ok(Action::requeue(REQUEUE))
300            }
301            Err(error) => {
302                set_phase(
303                    &restore,
304                    &ctx,
305                    KanidmRestorePhase::Failed,
306                    Some(error.to_string()),
307                )
308                .await?;
309                Ok(Action::requeue(Duration::from_secs(300)))
310            }
311        },
312        KanidmRestorePhase::Quiescing => {
313            let target = get_target(&restore, &ctx).await?;
314            scale_all(&target, &ctx, 0).await?;
315            if target_pods_stopped(&target, &ctx).await?
316                && target_volumes_detached(&target, &ctx).await?
317            {
318                let next_phase = if requires_safety_backup(&restore) {
319                    KanidmRestorePhase::SafetyBackup
320                } else {
321                    record_break_glass(&restore, &ctx).await?;
322                    KanidmRestorePhase::PreparingSource
323                };
324                if next_phase == KanidmRestorePhase::PreparingSource {
325                    let refreshed = Api::<KanidmRestore>::namespaced(
326                        ctx.client.clone(),
327                        restore.namespace().as_deref().unwrap_or_default(),
328                    )
329                    .get(&restore.name_any())
330                    .await
331                    .map_err(|error| {
332                        Error::kube_error(
333                            "get",
334                            "KanidmRestore",
335                            restore.namespace().as_deref().unwrap_or_default(),
336                            restore.name_any(),
337                            error,
338                        )
339                    })?;
340                    set_phase(&refreshed, &ctx, next_phase, None).await?;
341                } else {
342                    set_phase(&restore, &ctx, next_phase, None).await?;
343                }
344            }
345            Ok(Action::requeue(REQUEUE))
346        }
347        KanidmRestorePhase::SafetyBackup => {
348            let target = get_target(&restore, &ctx).await?;
349            let name = safety_backup_job_name(&restore);
350            let expected_backup_id = compute_safety_backup_id(&restore)?;
351            let expected_manifest_key = compute_safety_manifest_key(&restore)?;
352            ensure_safety_backup_job(
353                &restore,
354                &target,
355                &ctx,
356                &name,
357                &expected_backup_id,
358                &expected_manifest_key,
359            )
360            .await?;
361            let mut status = restore.status.clone().unwrap_or_default();
362            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
363            match job_state(&restore, &ctx, &name).await? {
364                JobState::Complete => {
365                    let result =
366                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
367                    match result {
368                        Ok(verified) => {
369                            status.safety_backup_job_name = Some(name);
370                            status.safety_backup_ref =
371                                Some(format!("safety-{}", restore.name_any()));
372                            status.safety_backup_manifest_key = Some(verified.manifest_key);
373                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
374                            record_safety_backup_duration(&restore, &ctx).await;
375                            status.phase = KanidmRestorePhase::PreparingSource;
376                            status.message = None;
377                            patch_status(&restore, &ctx, status).await?;
378                        }
379                        Err(error) => {
380                            resume_before_mutation(&restore, &ctx).await?;
381                            set_phase(
382                                &restore,
383                                &ctx,
384                                KanidmRestorePhase::Failed,
385                                Some(format!("safety backup result verification failed: {error}")),
386                            )
387                            .await?;
388                        }
389                    }
390                }
391                JobState::Failed => {
392                    resume_before_mutation(&restore, &ctx).await?;
393                    set_phase(
394                        &restore,
395                        &ctx,
396                        KanidmRestorePhase::Failed,
397                        Some(
398                            "safety backup job failed; target restored to original state"
399                                .to_string(),
400                        ),
401                    )
402                    .await?;
403                }
404                JobState::Running => {
405                    patch_status(&restore, &ctx, status).await?;
406                }
407            }
408            Ok(Action::requeue(REQUEUE))
409        }
410        KanidmRestorePhase::PreparingSource => {
411            if is_remote_source(&restore) {
412                let target = get_target(&restore, &ctx).await?;
413                let name = source_prep_job_name(&restore);
414                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
415                match job_state(&restore, &ctx, &name).await? {
416                    JobState::Complete => {
417                        let backup_id = restore
418                            .spec
419                            .source
420                            .backup_ref
421                            .as_ref()
422                            .map(|b| b.name.clone())
423                            .unwrap_or_default();
424                        let backup = {
425                            let ns = restore.namespace().unwrap();
426                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
427                                .get(&backup_id)
428                                .await
429                                .map_err(|e| {
430                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
431                                })?
432                        };
433                        let expected_backup_id = &backup.spec.backup_id;
434                        let result =
435                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
436                                .await;
437                        match result {
438                            Ok(verified) => {
439                                if let Some(expected_sha256) = backup
440                                    .status
441                                    .as_ref()
442                                    .and_then(|s| s.payload_sha256.as_deref())
443                                {
444                                    if verified.payload_sha256 != expected_sha256 {
445                                        resume_before_mutation(&restore, &ctx).await?;
446                                        set_phase(
447                                            &restore,
448                                            &ctx,
449                                            KanidmRestorePhase::Failed,
450                                            Some(format!(
451                                                "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
452                                                verified.payload_sha256
453                                            )),
454                                        )
455                                        .await?;
456                                        return Ok(Action::requeue(REQUEUE));
457                                    }
458                                }
459                                let mut status = restore.status.clone().unwrap_or_default();
460                                status.source_prep_job_name = Some(name);
461                                status.phase = KanidmRestorePhase::RestoringPrimary;
462                                status.message = None;
463                                patch_status(&restore, &ctx, status).await?;
464                            }
465                            Err(error) => {
466                                resume_before_mutation(&restore, &ctx).await?;
467                                set_phase(
468                                    &restore,
469                                    &ctx,
470                                    KanidmRestorePhase::Failed,
471                                    Some(format!(
472                                        "source preparation result verification failed: {error}"
473                                    )),
474                                )
475                                .await?;
476                            }
477                        }
478                    }
479                    JobState::Failed => {
480                        resume_before_mutation(&restore, &ctx).await?;
481                        set_phase(
482                            &restore,
483                            &ctx,
484                            KanidmRestorePhase::Failed,
485                            Some(
486                                "source preparation job failed; target restored to original state"
487                                    .to_string(),
488                            ),
489                        )
490                        .await?;
491                    }
492                    JobState::Running => {}
493                }
494            } else {
495                let target = get_target(&restore, &ctx).await?;
496                let name = source_check_job_name(&restore);
497                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
498                match job_state(&restore, &ctx, &name).await? {
499                    JobState::Complete => {
500                        let mut status = restore.status.clone().unwrap_or_default();
501                        status.source_prep_job_name = Some(name);
502                        status.phase = KanidmRestorePhase::RestoringPrimary;
503                        status.message = None;
504                        patch_status(&restore, &ctx, status).await?;
505                    }
506                    JobState::Failed => {
507                        resume_before_mutation(&restore, &ctx).await?;
508                        set_phase(
509                            &restore,
510                            &ctx,
511                            KanidmRestorePhase::Failed,
512                            Some(
513                                "local backup file check failed; target restored to original state"
514                                    .to_string(),
515                            ),
516                        )
517                        .await?;
518                    }
519                    JobState::Running => {}
520                }
521            }
522            Ok(Action::requeue(REQUEUE))
523        }
524        KanidmRestorePhase::RestoringPrimary => {
525            let target = get_target(&restore, &ctx).await?;
526            let status = restore.status.as_ref().cloned().unwrap_or_default();
527            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
528                return Err(Error::MissingData(
529                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
530                ));
531            }
532            if !status.database_mutation_started {
533                let mut status = status;
534                status.database_mutation_started = true;
535                status.message = Some(
536                    "database mutation boundary persisted; starting primary restore".to_string(),
537                );
538                patch_status(&restore, &ctx, status).await?;
539                return Ok(Action::requeue(REQUEUE));
540            }
541            let name = restore_job_name(&restore);
542            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
543            match job_state(&restore, &ctx, &name).await? {
544                JobState::Complete => {
545                    let mut status = restore.status.clone().unwrap_or_default();
546                    status.restore_job_name = Some(name);
547                    status.phase = KanidmRestorePhase::Verifying;
548                    status.message = None;
549                    patch_status(&restore, &ctx, status).await?;
550                }
551                JobState::Failed => {
552                    let status = restore.status.clone().unwrap_or_default();
553                    let failed_status = restore_primary_failed_status(&status, &name);
554                    patch_status(&restore, &ctx, failed_status).await?;
555                }
556                JobState::Running => {}
557            }
558            Ok(Action::requeue(REQUEUE))
559        }
560        KanidmRestorePhase::Verifying => {
561            let target = get_target(&restore, &ctx).await?;
562            let name = verify_job_name(&restore);
563            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
564            match job_state(&restore, &ctx, &name).await? {
565                JobState::Complete => {
566                    let mut status = restore.status.clone().unwrap_or_default();
567                    status.verify_job_name = Some(name);
568                    status.phase = KanidmRestorePhase::RebuildingReplicas;
569                    status.message = None;
570                    patch_status(&restore, &ctx, status).await?;
571                }
572                JobState::Failed => {
573                    let status = restore.status.clone().unwrap_or_default();
574                    let failed_status = verifying_failed_status(&status, &name);
575                    patch_status(&restore, &ctx, failed_status).await?;
576                }
577                JobState::Running => {}
578            }
579            Ok(Action::requeue(REQUEUE))
580        }
581        KanidmRestorePhase::RebuildingReplicas => {
582            let target = get_target(&restore, &ctx).await?;
583            let mut status = restore.status.clone().unwrap_or_default();
584            if !status.replicas_cleared {
585                if delete_secondary_pvcs(&target, &ctx).await? {
586                    status.replicas_cleared = true;
587                    status.message = Some("secondary database state cleared".to_string());
588                    patch_status(&restore, &ctx, status).await?;
589                }
590                return Ok(Action::requeue(REQUEUE));
591            }
592
593            scale_primary(&target, &ctx, 1).await?;
594            if !primary_ready(&target, &ctx).await? {
595                return Ok(Action::requeue(REQUEUE));
596            }
597            if !status.certificates_cleared {
598                delete_replica_cert_secrets(&target, &ctx).await?;
599                delete_admin_secret(&target, &ctx).await?;
600                status.certificates_cleared = true;
601                status.message = Some(
602                    "replica certificates and admin secret cleared for regeneration".to_string(),
603                );
604                patch_status(&restore, &ctx, status).await?;
605                return Ok(Action::requeue(REQUEUE));
606            }
607            status.phase = KanidmRestorePhase::Resuming;
608            status.message = None;
609            patch_status(&restore, &ctx, status).await?;
610            Ok(Action::requeue(REQUEUE))
611        }
612        KanidmRestorePhase::Resuming => {
613            let target = get_target(&restore, &ctx).await?;
614            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
615            clear_restoring(&restore, &target, &ctx).await?;
616            Ok(Action::requeue(Duration::from_secs(3600)))
617        }
618        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
619            Ok(Action::requeue(Duration::from_secs(3600)))
620        }
621    }
622}
623
624async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
625    let target = get_target(restore, ctx).await?;
626    let actual_uid = target
627        .uid()
628        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
629    if actual_uid != restore.spec.target_ref.uid {
630        return Err(Error::MissingData(format!(
631            "target UID mismatch: expected {}, got {}",
632            restore.spec.target_ref.uid, actual_uid
633        )));
634    }
635    validate_source(restore)?;
636    if is_remote_source(restore) {
637        validate_backup_ref(restore, &target, ctx).await?;
638    }
639    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
640    {
641        return Err(Error::MissingData(format!(
642            "restoreImage must be the target's pinned Kanidm image (target image is {})",
643            target.spec.image
644        )));
645    }
646    let storage = target
647        .spec
648        .storage
649        .as_ref()
650        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
651    if storage.empty_dir.is_some()
652        || storage.ephemeral.is_some()
653        || storage.volume_claim_template.is_none()
654    {
655        return Err(Error::MissingData(
656            "restore requires PVC-backed Kanidm storage".to_string(),
657        ));
658    }
659    let primaries = target
660        .spec
661        .replica_groups
662        .iter()
663        .filter(|rg| rg.primary_node)
664        .count();
665    if primaries != 1 {
666        return Err(Error::MissingData(
667            "backup/restore requires exactly one primary replica group".to_string(),
668        ));
669    }
670    let ns = restore.namespace().unwrap();
671    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
672        .list(&ListParams::default())
673        .await
674        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
675    if restores.items.iter().any(|other| {
676        other.name_any() != restore.name_any()
677            && other.spec.target_ref.name == restore.spec.target_ref.name
678            && !matches!(
679                other.status.as_ref().map(|s| s.phase),
680                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
681            )
682    }) {
683        return Err(Error::MissingData(
684            "another active restore targets this Kanidm".to_string(),
685        ));
686    }
687    validate_safety_backup_config(restore)?;
688    Ok(target)
689}
690
691fn safe_basename(name: &str) -> bool {
692    !name.is_empty()
693        && name != "."
694        && name != ".."
695        && !name.contains('/')
696        && !name.contains('\\')
697        && !name.contains('\0')
698}
699
700fn mutable_image(image: &str) -> bool {
701    image == "kanidm/server:latest"
702        || image.ends_with(":latest")
703        || (!image.contains('@')
704            && !image
705                .rsplit('/')
706                .next()
707                .is_some_and(|part| part.contains(':')))
708}
709
710async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
711    let ns = restore
712        .namespace()
713        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
714    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
715        .get(&restore.spec.target_ref.name)
716        .await
717        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
718}
719
720async fn mark_restoring(
721    restore: &KanidmRestore,
722    target: &Kanidm,
723    ctx: &RestoreContext,
724) -> Result<()> {
725    let ns = target.namespace().unwrap();
726    let uid = restore
727        .uid()
728        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
729    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
730        .patch(
731            &target.name_any(),
732            &PatchParams::default(),
733            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
734        )
735        .await
736        .map(|_| ())
737        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
738}
739
740async fn clear_restoring(
741    restore: &KanidmRestore,
742    target: &Kanidm,
743    ctx: &RestoreContext,
744) -> Result<()> {
745    let ns = target.namespace().unwrap();
746    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
747        return Ok(());
748    }
749    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
750        .patch(
751            &target.name_any(),
752            &PatchParams::default(),
753            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
754        )
755        .await
756        .map(|_| ())
757        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
758}
759
760async fn patch_status(
761    restore: &KanidmRestore,
762    ctx: &RestoreContext,
763    mut status: KanidmRestoreStatus,
764) -> Result<()> {
765    let ns = restore.namespace().unwrap();
766    let previous_phase = restore.status.as_ref().map(|current| current.phase);
767    let next_phase = status.phase;
768    status.observed_generation = restore.metadata.generation;
769    update_restore_conditions(&mut status, restore.metadata.generation);
770
771    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
772        .patch_status(
773            &restore.name_any(),
774            &PatchParams::default(),
775            &Patch::Merge(json!({"status": &status})),
776        )
777        .await
778        .map_err(|e| {
779            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
780        })?;
781
782    if previous_phase != Some(next_phase) {
783        record_restore_transition(
784            restore,
785            ctx,
786            previous_phase,
787            next_phase,
788            status.message.as_deref(),
789        )
790        .await;
791    }
792    Ok(())
793}
794
795fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
796    let previous = status.conditions.clone();
797    let phase = status.phase;
798    let phase_reason = format!("{phase:?}");
799    let message = status
800        .message
801        .clone()
802        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
803    let terminal_success = phase == KanidmRestorePhase::Completed;
804    let terminal_failure = phase == KanidmRestorePhase::Failed;
805    let progressing = !terminal_success && !terminal_failure;
806
807    let break_glass = previous
808        .iter()
809        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
810        .cloned();
811    status.conditions = vec![
812        restore_condition(
813            &previous,
814            CONDITION_PROGRESSING,
815            if progressing {
816                CONDITION_TRUE
817            } else {
818                CONDITION_FALSE
819            },
820            &phase_reason,
821            &message,
822            generation,
823        ),
824        restore_condition(
825            &previous,
826            CONDITION_READY,
827            if terminal_success {
828                CONDITION_TRUE
829            } else {
830                CONDITION_FALSE
831            },
832            if terminal_success {
833                "RestoreCompleted"
834            } else {
835                "RestoreNotCompleted"
836            },
837            if terminal_success {
838                "Kanidm restore completed successfully."
839            } else {
840                "Kanidm restore has not completed successfully."
841            },
842            generation,
843        ),
844        restore_condition(
845            &previous,
846            CONDITION_FAILED,
847            if terminal_failure {
848                CONDITION_TRUE
849            } else {
850                CONDITION_FALSE
851            },
852            if terminal_failure {
853                "RestoreFailed"
854            } else {
855                "NoRestoreFailure"
856            },
857            if terminal_failure {
858                status
859                    .message
860                    .as_deref()
861                    .unwrap_or("Kanidm restore failed.")
862            } else {
863                "No terminal restore failure has been recorded."
864            },
865            generation,
866        ),
867    ];
868    if let Some(condition) = break_glass {
869        status.conditions.push(condition);
870    }
871}
872
873fn restore_condition(
874    previous: &[Condition],
875    condition_type: &str,
876    condition_status: &str,
877    reason: &str,
878    message: &str,
879    generation: Option<i64>,
880) -> Condition {
881    let last_transition_time = previous
882        .iter()
883        .find(|condition| {
884            condition.type_ == condition_type
885                && condition.status == condition_status
886                && condition.reason == reason
887        })
888        .map(|condition| condition.last_transition_time.clone())
889        .unwrap_or_else(|| Time(Timestamp::now()));
890    Condition {
891        type_: condition_type.to_string(),
892        status: condition_status.to_string(),
893        reason: reason.to_string(),
894        message: message.to_string(),
895        last_transition_time,
896        observed_generation: generation,
897    }
898}
899
900async fn record_restore_transition(
901    restore: &KanidmRestore,
902    ctx: &RestoreContext,
903    previous_phase: Option<KanidmRestorePhase>,
904    phase: KanidmRestorePhase,
905    message: Option<&str>,
906) {
907    if phase == KanidmRestorePhase::Validating {
908        ctx.metrics.attempts.add(1, &[]);
909    }
910
911    let result = match phase {
912        KanidmRestorePhase::Completed => Some("success"),
913        KanidmRestorePhase::Failed => Some("failure"),
914        _ => None,
915    };
916    if let Some(result) = result {
917        let attributes = [KeyValue::new("result", result)];
918        ctx.metrics.outcomes.add(1, &attributes);
919        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
920            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
921            ctx.metrics.duration_seconds.record(elapsed, &attributes);
922        }
923    }
924
925    let reason = if phase == KanidmRestorePhase::Failed {
926        "RestoreFailed"
927    } else {
928        "RestorePhaseChanged"
929    };
930    let note = message.map(str::to_string).or_else(|| {
931        Some(format!(
932            "Kanidm restore phase changed from {} to {phase:?}.",
933            previous_phase
934                .map(|previous| format!("{previous:?}"))
935                .unwrap_or_else(|| "None".to_string())
936        ))
937    });
938    if let Err(error) = ctx
939        .recorder
940        .publish(
941            &Event {
942                type_: if phase == KanidmRestorePhase::Failed {
943                    EventType::Warning
944                } else {
945                    EventType::Normal
946                },
947                reason: reason.to_string(),
948                note,
949                action: "Restore".to_string(),
950                secondary: None,
951            },
952            &restore.object_ref(&()),
953        )
954        .await
955    {
956        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
957    }
958}
959
960async fn set_phase(
961    restore: &KanidmRestore,
962    ctx: &RestoreContext,
963    phase: KanidmRestorePhase,
964    message: Option<String>,
965) -> Result<()> {
966    let mut status = restore.status.clone().unwrap_or_default();
967    status.phase = phase;
968    status.message = message;
969    patch_status(restore, ctx, status).await
970}
971
972async fn fail_after_mutation(
973    restore: &KanidmRestore,
974    ctx: &RestoreContext,
975    message: &str,
976) -> Result<()> {
977    set_phase(
978        restore,
979        ctx,
980        KanidmRestorePhase::Failed,
981        Some(message.to_string()),
982    )
983    .await
984}
985
986fn restore_primary_failed_status(
987    existing: &KanidmRestoreStatus,
988    job_name: &str,
989) -> KanidmRestoreStatus {
990    let mut status = existing.clone();
991    status.restore_job_name = Some(job_name.to_string());
992    status.phase = KanidmRestorePhase::Failed;
993    status.message = Some("database restore job failed".to_string());
994    status
995}
996
997fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
998    let mut status = existing.clone();
999    status.verify_job_name = Some(job_name.to_string());
1000    status.phase = KanidmRestorePhase::Failed;
1001    status.message = Some("database verification failed".to_string());
1002    status
1003}
1004
1005fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1006    target
1007        .spec
1008        .replica_groups
1009        .iter()
1010        .find(|rg| rg.primary_node)
1011        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1012}
1013
1014async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1015    let ns = target.namespace().unwrap();
1016    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1017    for rg in &target.spec.replica_groups {
1018        let name = target.statefulset_name(&rg.name);
1019        api.patch(
1020            &name,
1021            &PatchParams::default(),
1022            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1023        )
1024        .await
1025        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1026    }
1027    Ok(())
1028}
1029
1030async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1031    let ns = target.namespace().unwrap();
1032    let rg = primary_group(target)?;
1033    let name = target.statefulset_name(&rg.name);
1034    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1035        .patch(
1036            &name,
1037            &PatchParams::default(),
1038            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1039        )
1040        .await
1041        .map(|_| ())
1042        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1043}
1044
1045async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1046    let ns = target.namespace().unwrap();
1047    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1048    for rg in &target.spec.replica_groups {
1049        let name = target.statefulset_name(&rg.name);
1050        api.patch(
1051            &name,
1052            &PatchParams::default(),
1053            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1054        )
1055        .await
1056        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1057    }
1058    Ok(())
1059}
1060
1061async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1062    let ns = target.namespace().unwrap();
1063    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1064        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1065        .await
1066        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1067    Ok(pods.items.is_empty())
1068}
1069
1070async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1071    let ns = target.namespace().unwrap();
1072    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1073    let mut pv_names = BTreeSet::new();
1074    let primary_name = primary_pvc_name(target)?;
1075    let primary_pvc = pvc_api
1076        .get(&primary_name)
1077        .await
1078        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1079    let primary_pv = primary_pvc
1080        .spec
1081        .as_ref()
1082        .and_then(|spec| spec.volume_name.clone())
1083        .ok_or_else(|| {
1084            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1085        })?;
1086    pv_names.insert(primary_pv);
1087
1088    for rg in &target.spec.replica_groups {
1089        let sts = target.statefulset_name(&rg.name);
1090        for ordinal in 0..rg.replicas {
1091            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1092            if name == primary_name {
1093                continue;
1094            }
1095            if let Some(pvc) = pvc_api
1096                .get_opt(&name)
1097                .await
1098                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1099                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1100            {
1101                pv_names.insert(pv_name);
1102            }
1103        }
1104    }
1105
1106    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1107        .list(&ListParams::default())
1108        .await
1109        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1110    Ok(!attachments.items.iter().any(|attachment| {
1111        attachment
1112            .spec
1113            .source
1114            .persistent_volume_name
1115            .as_ref()
1116            .is_some_and(|pv_name| pv_names.contains(pv_name))
1117    }))
1118}
1119
1120async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1121    let ns = target.namespace().unwrap();
1122    let name = target.statefulset_name(&primary_group(target)?.name);
1123    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1124        .get(&name)
1125        .await
1126        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1127    Ok(sts
1128        .status
1129        .as_ref()
1130        .and_then(|s| s.ready_replicas)
1131        .unwrap_or(0)
1132        >= 1)
1133}
1134
1135fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1136    let rg = primary_group(target)?;
1137    Ok(format!(
1138        "{DATA_VOLUME}-{}-0",
1139        target.statefulset_name(&rg.name)
1140    ))
1141}
1142
1143async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1144    let ns = target.namespace().unwrap();
1145    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1146    let mut all_absent = true;
1147    for rg in &target.spec.replica_groups {
1148        let sts = target.statefulset_name(&rg.name);
1149        for ordinal in 0..rg.replicas {
1150            if rg.primary_node && ordinal == 0 {
1151                continue;
1152            }
1153            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1154            match api
1155                .get_opt(&name)
1156                .await
1157                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1158            {
1159                None => {}
1160                Some(pvc) => {
1161                    all_absent = false;
1162                    if pvc.metadata.deletion_timestamp.is_none() {
1163                        match api.delete(&name, &DeleteParams::default()).await {
1164                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1165                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1166                            Err(error) => {
1167                                return Err(Error::kube_error(
1168                                    "delete",
1169                                    "PersistentVolumeClaim",
1170                                    &ns,
1171                                    &name,
1172                                    error,
1173                                ));
1174                            }
1175                        }
1176                    }
1177                }
1178            }
1179        }
1180    }
1181    Ok(all_absent)
1182}
1183
1184async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1185    let ns = target.namespace().unwrap();
1186    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1187    let label_selector = format!(
1188        "{}={},{}={}",
1189        CLUSTER_LABEL,
1190        target.name_any(),
1191        SECRET_TYPE_LABEL,
1192        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1193    );
1194    let secrets = api
1195        .list(&ListParams::default().labels(&label_selector))
1196        .await
1197        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1198    let mut all_absent = secrets.items.is_empty();
1199    for secret in secrets.items {
1200        let name = secret.name_any();
1201        if secret.metadata.deletion_timestamp.is_none() {
1202            match api.delete(&name, &DeleteParams::default()).await {
1203                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1204                Err(kube::Error::Api(status)) if status.code == 404 => {}
1205                Err(error) => {
1206                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1207                }
1208            }
1209            all_absent = false;
1210        }
1211    }
1212    Ok(all_absent)
1213}
1214
1215async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1216    let ns = target.namespace().unwrap();
1217    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1218    let name = format!("{}-admin-passwords", target.name_any());
1219    match api
1220        .get_opt(&name)
1221        .await
1222        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1223    {
1224        None => Ok(true),
1225        Some(secret) => {
1226            if secret.metadata.deletion_timestamp.is_some() {
1227                return Ok(false);
1228            }
1229            match api.delete(&name, &DeleteParams::default()).await {
1230                Ok(_) => {
1231                    debug!(secret = %name, "deleting stale admin secret");
1232                    Ok(false)
1233                }
1234                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1235                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1236            }
1237        }
1238    }
1239}
1240
1241fn restore_job_name(restore: &KanidmRestore) -> String {
1242    format!("{}-restore", restore.name_any())
1243}
1244fn verify_job_name(restore: &KanidmRestore) -> String {
1245    format!("{}-verify", restore.name_any())
1246}
1247
1248fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1249    let origin = target
1250        .spec
1251        .origin
1252        .clone()
1253        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1254    vec![
1255        EnvVar {
1256            name: "KANIDM_DB_PATH".to_string(),
1257            value: Some("/data/kanidm.db".to_string()),
1258            ..Default::default()
1259        },
1260        EnvVar {
1261            name: "KANIDM_DOMAIN".to_string(),
1262            value: Some(target.spec.domain.clone()),
1263            ..Default::default()
1264        },
1265        EnvVar {
1266            name: "KANIDM_ORIGIN".to_string(),
1267            value: Some(origin),
1268            ..Default::default()
1269        },
1270        EnvVar {
1271            name: "KANIDM_TLS_CHAIN".to_string(),
1272            value: Some(format!("{TLS_PATH}/tls.crt")),
1273            ..Default::default()
1274        },
1275        EnvVar {
1276            name: "KANIDM_TLS_KEY".to_string(),
1277            value: Some(format!("{TLS_PATH}/tls.key")),
1278            ..Default::default()
1279        },
1280    ]
1281}
1282
1283async fn ensure_database_job(
1284    restore: &KanidmRestore,
1285    target: &Kanidm,
1286    ctx: &RestoreContext,
1287    name: &str,
1288    verify: bool,
1289) -> Result<()> {
1290    let ns = restore.namespace().unwrap();
1291    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1292    if api
1293        .get_opt(name)
1294        .await
1295        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1296        .is_some()
1297    {
1298        return Ok(());
1299    }
1300    let mut command = vec![
1301        "kanidmd".to_string(),
1302        "database".to_string(),
1303        if verify { "backup" } else { "restore" }.to_string(),
1304    ];
1305    if verify {
1306        command.push("/verify/verification.json.gz".to_string());
1307    } else {
1308        if is_remote_source(restore) {
1309            command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1310        } else {
1311            let file_name = restore
1312                .spec
1313                .source
1314                .local
1315                .as_ref()
1316                .map(|l| l.file_name.clone())
1317                .unwrap_or_default();
1318            command.push(format!("{BACKUP_PATH}/{file_name}"));
1319        }
1320    }
1321    let job = Job {
1322        metadata: ObjectMeta {
1323            name: Some(name.to_string()),
1324            namespace: Some(ns.clone()),
1325            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1326            ..Default::default()
1327        },
1328        spec: Some(JobSpec {
1329            backoff_limit: Some(0),
1330            template: PodTemplateSpec {
1331                metadata: Some(ObjectMeta::default()),
1332                spec: Some(PodSpec {
1333                    automount_service_account_token: Some(false),
1334                    restart_policy: Some("Never".to_string()),
1335                    security_context: Some(source_job_pod_security_context(target)),
1336                    containers: vec![Container {
1337                        name: if verify { "verify" } else { "restore" }.to_string(),
1338                        image: Some(restore.spec.restore_image.clone()),
1339                        command: Some(command),
1340                        env: Some(build_kanidm_db_env_vars(target)),
1341                        security_context: Some(kanidm_job_security_context()),
1342                        resources: Some(job_resource_requirements()),
1343                        volume_mounts: Some(vec![
1344                            VolumeMount {
1345                                name: DATA_VOLUME.to_string(),
1346                                mount_path: DATA_PATH.to_string(),
1347                                ..Default::default()
1348                            },
1349                            VolumeMount {
1350                                name: TLS_VOLUME.to_string(),
1351                                mount_path: TLS_PATH.to_string(),
1352                                read_only: Some(true),
1353                                ..Default::default()
1354                            },
1355                            VolumeMount {
1356                                name: "verification-output".to_string(),
1357                                mount_path: "/verify".to_string(),
1358                                ..Default::default()
1359                            },
1360                        ]),
1361                        ..Default::default()
1362                    }],
1363                    volumes: Some(vec![
1364                        Volume {
1365                            name: DATA_VOLUME.to_string(),
1366                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1367                                claim_name: primary_pvc_name(target)?,
1368                                read_only: Some(false),
1369                            }),
1370                            ..Default::default()
1371                        },
1372                        Volume {
1373                            name: TLS_VOLUME.to_string(),
1374                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1375                                secret_name: Some(target.effective_tls_secret_name()),
1376                                ..Default::default()
1377                            }),
1378                            ..Default::default()
1379                        },
1380                        Volume {
1381                            name: "verification-output".to_string(),
1382                            empty_dir: Some(Default::default()),
1383                            ..Default::default()
1384                        },
1385                    ]),
1386                    ..Default::default()
1387                }),
1388            },
1389            ..Default::default()
1390        }),
1391        ..Default::default()
1392    };
1393    api.create(&PostParams::default(), &job)
1394        .await
1395        .map(|_| ())
1396        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1397}
1398
1399enum JobState {
1400    Running,
1401    Complete,
1402    Failed,
1403}
1404
1405async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1406    let ns = restore.namespace().unwrap();
1407    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1408        .get(name)
1409        .await
1410        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1411    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1412        return Ok(JobState::Failed);
1413    }
1414    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1415        return Ok(JobState::Complete);
1416    }
1417    Ok(JobState::Running)
1418}
1419
1420fn is_remote_source(restore: &KanidmRestore) -> bool {
1421    restore.spec.source.backup_ref.is_some()
1422}
1423
1424fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1425    let safety = restore.spec.safety_backup.as_ref();
1426    let skip = safety.is_some_and(|s| s.skip);
1427    !skip
1428}
1429
1430fn validate_source(restore: &KanidmRestore) -> Result<()> {
1431    let has_local = restore.spec.source.local.is_some();
1432    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1433    if has_local && has_backup_ref {
1434        return Err(Error::MissingData(
1435            "source.local and source.backupRef are mutually exclusive".to_string(),
1436        ));
1437    }
1438    if !has_local && !has_backup_ref {
1439        return Err(Error::MissingData(
1440            "source must specify either local or backupRef".to_string(),
1441        ));
1442    }
1443    if let Some(local) = &restore.spec.source.local {
1444        if !safe_basename(&local.file_name) {
1445            return Err(Error::MissingData(
1446                "restore source fileName must be a safe basename".to_string(),
1447            ));
1448        }
1449    }
1450    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1451        if backup_ref.name.is_empty() {
1452            return Err(Error::MissingData(
1453                "source.backupRef.name must not be empty".to_string(),
1454            ));
1455        }
1456    }
1457    Ok(())
1458}
1459
1460fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1461    let safety = restore.spec.safety_backup.as_ref();
1462    let skip = safety.is_some_and(|s| s.skip);
1463    if skip {
1464        let annotations = restore.metadata.annotations.as_ref();
1465        let reason = annotations
1466            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1467            .map(|s| s.trim())
1468            .unwrap_or("");
1469        let approver = annotations
1470            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1471            .map(|s| s.trim())
1472            .unwrap_or("");
1473        if reason.is_empty() {
1474            return Err(Error::MissingData(format!(
1475                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1476            )));
1477        }
1478        if approver.is_empty() {
1479            return Err(Error::MissingData(format!(
1480                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1481            )));
1482        }
1483    }
1484    if !skip {
1485        let has_repo = safety
1486            .and_then(|s| s.repository_ref.as_ref())
1487            .is_some_and(|r| !r.name.is_empty());
1488        if !has_repo {
1489            return Err(Error::MissingData(
1490                "safety backup requires safetyBackup.repositoryRef when skip is not set"
1491                    .to_string(),
1492            ));
1493        }
1494    }
1495    Ok(())
1496}
1497
1498fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1499    target
1500        .spec
1501        .replica_groups
1502        .iter()
1503        .map(|rg| ReplicaCountEntry {
1504            group: rg.name.clone(),
1505            replicas: rg.replicas,
1506        })
1507        .collect()
1508}
1509
1510async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1511    if let Ok(target) = get_target(restore, ctx).await {
1512        let owns_maintenance =
1513            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1514        if owns_maintenance {
1515            scale_desired(&target, ctx).await?;
1516            clear_restoring(restore, &target, ctx).await?;
1517        }
1518    }
1519    Ok(())
1520}
1521
1522async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1523    ctx.metrics.break_glass_total.add(1, &[]);
1524    let reason = restore
1525        .metadata
1526        .annotations
1527        .as_ref()
1528        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1529        .cloned()
1530        .unwrap_or_default();
1531    let approver = restore
1532        .metadata
1533        .annotations
1534        .as_ref()
1535        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1536        .cloned()
1537        .unwrap_or_default();
1538    let mut status = restore.status.clone().unwrap_or_default();
1539    let break_glass_condition = restore_condition(
1540        &status.conditions,
1541        CONDITION_BREAK_GLASS,
1542        CONDITION_TRUE,
1543        "BreakGlassOverride",
1544        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1545        restore.metadata.generation,
1546    );
1547    status
1548        .conditions
1549        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1550    status.conditions.push(break_glass_condition);
1551    patch_status(restore, ctx, status).await?;
1552    if let Err(error) = ctx
1553        .recorder
1554        .publish(
1555            &Event {
1556                type_: EventType::Warning,
1557                reason: "SafetyBackupSkipped".to_string(),
1558                note: Some(format!(
1559                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1560                )),
1561                action: "Restore".to_string(),
1562                secondary: None,
1563            },
1564            &restore.object_ref(&()),
1565        )
1566        .await
1567    {
1568        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1569    }
1570    warn!(
1571        restore = %restore.name_any(),
1572        reason = %reason,
1573        approver = %approver,
1574        "break-glass safety backup override activated"
1575    );
1576    Ok(())
1577}
1578
1579async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1580    let status = restore.status.as_ref().cloned().unwrap_or_default();
1581    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1582        if let Ok(start_ts) = start.parse::<Timestamp>() {
1583            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1584            ctx.metrics
1585                .safety_backup_duration_seconds
1586                .record(elapsed, &[]);
1587        }
1588    }
1589}
1590
1591fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1592    format!("{}-safety-backup", restore.name_any())
1593}
1594
1595fn source_prep_job_name(restore: &KanidmRestore) -> String {
1596    format!("{}-source-prep", restore.name_any())
1597}
1598
1599fn source_check_job_name(restore: &KanidmRestore) -> String {
1600    format!("{}-source-check", restore.name_any())
1601}
1602
1603async fn ensure_source_check_job(
1604    restore: &KanidmRestore,
1605    target: &Kanidm,
1606    ctx: &RestoreContext,
1607    name: &str,
1608) -> Result<()> {
1609    let ns = restore.namespace().unwrap();
1610    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1611    if api
1612        .get_opt(name)
1613        .await
1614        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1615        .is_some()
1616    {
1617        return Ok(());
1618    }
1619    let file_name = restore
1620        .spec
1621        .source
1622        .local
1623        .as_ref()
1624        .map(|l| l.file_name.clone())
1625        .unwrap_or_default();
1626    let operation_doc = serde_json::json!({
1627        "apiVersion": "backup.kaniop.rs/v1alpha1",
1628        "kind": "OperationDocument",
1629        "operation": "check",
1630        "path": format!("{BACKUP_PATH}/{file_name}"),
1631        "resultPath": "/run/kaniop-result/result.json",
1632        "format": "kanidmJsonGzip",
1633    })
1634    .to_string();
1635    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1636    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1637    let data_mover_image = data_mover_image();
1638    let job = Job {
1639        metadata: ObjectMeta {
1640            name: Some(name.to_string()),
1641            namespace: Some(ns.clone()),
1642            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1643            ..Default::default()
1644        },
1645        spec: Some(JobSpec {
1646            backoff_limit: Some(0),
1647            template: PodTemplateSpec {
1648                metadata: Some(ObjectMeta::default()),
1649                spec: Some(PodSpec {
1650                    automount_service_account_token: Some(false),
1651                    restart_policy: Some("Never".to_string()),
1652                    security_context: Some(source_job_pod_security_context(target)),
1653                    containers: vec![Container {
1654                        name: "source-check".to_string(),
1655                        image: Some(data_mover_image),
1656                        command: Some(vec![
1657                            "/bin/kaniop-data-mover".to_string(),
1658                            "check".to_string(),
1659                            "--operation-doc".to_string(),
1660                            "/run/kaniop/operation.json".to_string(),
1661                        ]),
1662                        security_context: Some(hardened_security_context()),
1663                        resources: Some(job_resource_requirements()),
1664                        env: Some(vec![EnvVar {
1665                            name: "RUST_LOG".to_string(),
1666                            value: Some("info".to_string()),
1667                            ..Default::default()
1668                        }]),
1669                        volume_mounts: Some(vec![
1670                            VolumeMount {
1671                                name: DATA_VOLUME.to_string(),
1672                                mount_path: DATA_PATH.to_string(),
1673                                read_only: Some(true),
1674                                ..Default::default()
1675                            },
1676                            VolumeMount {
1677                                name: "operation".to_string(),
1678                                mount_path: "/run/kaniop".to_string(),
1679                                read_only: Some(true),
1680                                ..Default::default()
1681                            },
1682                            VolumeMount {
1683                                name: "result".to_string(),
1684                                mount_path: "/run/kaniop-result".to_string(),
1685                                ..Default::default()
1686                            },
1687                        ]),
1688                        ..Default::default()
1689                    }],
1690                    volumes: Some(vec![
1691                        Volume {
1692                            name: DATA_VOLUME.to_string(),
1693                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1694                                claim_name: primary_pvc_name(target)?,
1695                                read_only: Some(true),
1696                            }),
1697                            ..Default::default()
1698                        },
1699                        Volume {
1700                            name: "operation".to_string(),
1701                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1702                                name: operation_cm_name,
1703                                ..Default::default()
1704                            }),
1705                            ..Default::default()
1706                        },
1707                        Volume {
1708                            name: "result".to_string(),
1709                            empty_dir: Some(Default::default()),
1710                            ..Default::default()
1711                        },
1712                    ]),
1713                    ..Default::default()
1714                }),
1715            },
1716            ..Default::default()
1717        }),
1718        ..Default::default()
1719    };
1720    api.create(&PostParams::default(), &job)
1721        .await
1722        .map(|_| ())
1723        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1724}
1725
1726const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1727
1728fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1729    let ns = restore.namespace().unwrap_or_default();
1730    let uid = restore.uid().ok_or_else(|| {
1731        Error::MissingData(
1732            "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1733                .to_string(),
1734        )
1735    })?;
1736    let namespace_uuid =
1737        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1738    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1739    Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1740}
1741
1742fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1743    let ns = restore.namespace().unwrap_or_default();
1744    let kanidm_uid = &restore.spec.target_ref.uid;
1745    let backup_id = compute_safety_backup_id(restore)?;
1746    Ok(format!(
1747        "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1748    ))
1749}
1750
1751struct VerifiedSafetyBackup {
1752    manifest_key: String,
1753    payload_sha256: String,
1754}
1755
1756async fn read_safety_backup_result(
1757    restore: &KanidmRestore,
1758    ctx: &RestoreContext,
1759    job_name: &str,
1760    expected_backup_id: &str,
1761) -> Result<VerifiedSafetyBackup> {
1762    let ns = restore.namespace().unwrap();
1763    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1764    let pod_list = pods
1765        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1766        .await
1767        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1768    let pod = pod_list.items.first().ok_or_else(|| {
1769        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1770    })?;
1771    let pod_name = pod.name_any();
1772    let container_status = pod
1773        .status
1774        .as_ref()
1775        .and_then(|s| s.container_statuses.as_ref())
1776        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1777        .ok_or_else(|| {
1778            Error::MissingData(format!(
1779                "safety-upload container status not found in pod {pod_name}"
1780            ))
1781        })?;
1782    let termination_message = container_status
1783        .state
1784        .as_ref()
1785        .and_then(|state| state.terminated.as_ref())
1786        .and_then(|t| t.message.as_ref())
1787        .ok_or_else(|| {
1788            Error::MissingData(format!(
1789                "termination message not found in pod {pod_name}; result transport failed"
1790            ))
1791        })?;
1792    if termination_message.is_empty() {
1793        return Err(Error::MissingData(format!(
1794            "termination message is empty in pod {pod_name}; result document was not written"
1795        )));
1796    }
1797    let result_doc = parse_result_document(termination_message).map_err(|e| {
1798        Error::ParseError(format!(
1799            "failed to parse result document from pod {pod_name}: {e}"
1800        ))
1801    })?;
1802    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1803        return Err(Error::ParseError(format!(
1804            "unsupported result document version: {}",
1805            result_doc.api_version
1806        )));
1807    }
1808    if result_doc.kind != "ResultDocument" {
1809        return Err(Error::ParseError(format!(
1810            "invalid result document kind: {}",
1811            result_doc.kind
1812        )));
1813    }
1814    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1815        return Err(Error::ParseError(format!(
1816            "result document operation mismatch: expected '{}', got '{}'",
1817            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1818        )));
1819    }
1820    if !result_doc.success {
1821        return Err(Error::ParseError(format!(
1822            "safety backup reported failure with exit code {:?}",
1823            result_doc.exit_code
1824        )));
1825    }
1826    if result_doc.exit_code != ExitCode::Success {
1827        return Err(Error::ParseError(format!(
1828            "safety backup exit code is not Success: {:?}",
1829            result_doc.exit_code
1830        )));
1831    }
1832    let doc_backup_id = result_doc
1833        .backup_id
1834        .as_deref()
1835        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1836    if doc_backup_id != expected_backup_id {
1837        return Err(Error::ParseError(format!(
1838            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1839        )));
1840    }
1841    let manifest_key = result_doc
1842        .manifest_key
1843        .clone()
1844        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1845    if manifest_key.is_empty() {
1846        return Err(Error::ParseError(
1847            "result document manifestKey is empty".to_string(),
1848        ));
1849    }
1850    let payload_sha256 = result_doc
1851        .payload_sha256
1852        .clone()
1853        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1854    if payload_sha256.is_empty() {
1855        return Err(Error::ParseError(
1856            "result document payloadSha256 is empty".to_string(),
1857        ));
1858    }
1859    info!(
1860        restore = %restore.name_any(),
1861        backup_id = %expected_backup_id,
1862        manifest_key = %manifest_key,
1863        "safety backup result verified"
1864    );
1865    Ok(VerifiedSafetyBackup {
1866        manifest_key,
1867        payload_sha256,
1868    })
1869}
1870
1871struct VerifiedSourcePrepResult {
1872    #[allow(dead_code)]
1873    manifest_key: String,
1874    payload_sha256: String,
1875}
1876
1877async fn read_source_prep_result(
1878    restore: &KanidmRestore,
1879    ctx: &RestoreContext,
1880    job_name: &str,
1881    expected_backup_id: &str,
1882) -> Result<VerifiedSourcePrepResult> {
1883    let ns = restore.namespace().unwrap();
1884    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1885    let pod_list = pods
1886        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1887        .await
1888        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1889    let pod = pod_list.items.first().ok_or_else(|| {
1890        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1891    })?;
1892    let pod_name = pod.name_any();
1893    let container_status = pod
1894        .status
1895        .as_ref()
1896        .and_then(|s| s.container_statuses.as_ref())
1897        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1898        .ok_or_else(|| {
1899            Error::MissingData(format!(
1900                "source-prep container status not found in pod {pod_name}"
1901            ))
1902        })?;
1903    let termination_message = container_status
1904        .state
1905        .as_ref()
1906        .and_then(|state| state.terminated.as_ref())
1907        .and_then(|t| t.message.as_ref())
1908        .ok_or_else(|| {
1909            Error::MissingData(format!(
1910                "termination message not found in pod {pod_name}; result transport failed"
1911            ))
1912        })?;
1913    if termination_message.is_empty() {
1914        return Err(Error::MissingData(format!(
1915            "termination message is empty in pod {pod_name}; result document was not written"
1916        )));
1917    }
1918    let result_doc = parse_result_document(termination_message).map_err(|e| {
1919        Error::ParseError(format!(
1920            "failed to parse result document from pod {pod_name}: {e}"
1921        ))
1922    })?;
1923    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1924        return Err(Error::ParseError(format!(
1925            "unsupported result document version: {}",
1926            result_doc.api_version
1927        )));
1928    }
1929    if result_doc.kind != "ResultDocument" {
1930        return Err(Error::ParseError(format!(
1931            "invalid result document kind: {}",
1932            result_doc.kind
1933        )));
1934    }
1935    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1936        return Err(Error::ParseError(format!(
1937            "result document operation mismatch: expected '{}', got '{}'",
1938            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1939        )));
1940    }
1941    if !result_doc.success {
1942        return Err(Error::ParseError(format!(
1943            "source prep reported failure with exit code {:?}",
1944            result_doc.exit_code
1945        )));
1946    }
1947    if result_doc.exit_code != ExitCode::Success {
1948        return Err(Error::ParseError(format!(
1949            "source prep exit code is not Success: {:?}",
1950            result_doc.exit_code
1951        )));
1952    }
1953    let doc_backup_id = result_doc
1954        .backup_id
1955        .as_deref()
1956        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1957    if doc_backup_id != expected_backup_id {
1958        return Err(Error::ParseError(format!(
1959            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1960        )));
1961    }
1962    let manifest_key = result_doc
1963        .manifest_key
1964        .clone()
1965        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1966    if manifest_key.is_empty() {
1967        return Err(Error::ParseError(
1968            "result document manifestKey is empty".to_string(),
1969        ));
1970    }
1971    let payload_sha256 = result_doc
1972        .payload_sha256
1973        .clone()
1974        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1975    if payload_sha256.is_empty() {
1976        return Err(Error::ParseError(
1977            "result document payloadSha256 is empty".to_string(),
1978        ));
1979    }
1980    info!(
1981        restore = %restore.name_any(),
1982        backup_id = %expected_backup_id,
1983        manifest_key = %manifest_key,
1984        payload_sha256 = %payload_sha256,
1985        "source prep result verified"
1986    );
1987    Ok(VerifiedSourcePrepResult {
1988        manifest_key,
1989        payload_sha256,
1990    })
1991}
1992
1993fn source_job_pod_security_context(
1994    target: &Kanidm,
1995) -> k8s_openapi::api::core::v1::PodSecurityContext {
1996    let mut security_context = target.spec.security_context.clone().unwrap_or_default();
1997    security_context.seccomp_profile = Some(SeccompProfile {
1998        type_: "RuntimeDefault".to_string(),
1999        ..Default::default()
2000    });
2001    security_context
2002}
2003
2004fn hardened_security_context() -> SecurityContext {
2005    SecurityContext {
2006        allow_privilege_escalation: Some(false),
2007        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2008            drop: Some(vec!["ALL".to_string()]),
2009            ..Default::default()
2010        }),
2011        read_only_root_filesystem: Some(true),
2012        run_as_non_root: Some(true),
2013        seccomp_profile: Some(SeccompProfile {
2014            type_: "RuntimeDefault".to_string(),
2015            ..Default::default()
2016        }),
2017        ..Default::default()
2018    }
2019}
2020
2021fn kanidm_job_security_context() -> SecurityContext {
2022    SecurityContext {
2023        allow_privilege_escalation: Some(false),
2024        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2025            drop: Some(vec!["ALL".to_string()]),
2026            ..Default::default()
2027        }),
2028        seccomp_profile: Some(SeccompProfile {
2029            type_: "RuntimeDefault".to_string(),
2030            ..Default::default()
2031        }),
2032        ..Default::default()
2033    }
2034}
2035
2036fn job_resource_requirements() -> ResourceRequirements {
2037    ResourceRequirements {
2038        requests: Some(std::collections::BTreeMap::from([
2039            ("cpu".to_string(), Quantity("100m".to_string())),
2040            ("memory".to_string(), Quantity("128Mi".to_string())),
2041        ])),
2042        limits: Some(std::collections::BTreeMap::from([
2043            ("cpu".to_string(), Quantity("2".to_string())),
2044            ("memory".to_string(), Quantity("2Gi".to_string())),
2045        ])),
2046        ..Default::default()
2047    }
2048}
2049
2050async fn ensure_safety_backup_job(
2051    restore: &KanidmRestore,
2052    target: &Kanidm,
2053    ctx: &RestoreContext,
2054    name: &str,
2055    backup_id: &str,
2056    manifest_key: &str,
2057) -> Result<()> {
2058    let ns = restore.namespace().unwrap();
2059    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2060    if api
2061        .get_opt(name)
2062        .await
2063        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2064        .is_some()
2065    {
2066        return Ok(());
2067    }
2068    let safety = restore
2069        .spec
2070        .safety_backup
2071        .as_ref()
2072        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2073    let repo_ref = safety
2074        .repository_ref
2075        .as_ref()
2076        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2077    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2078        .get(&repo_ref.name)
2079        .await
2080        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2081    let _namespace_uid = &ns;
2082    let _kanidm_uid = &restore.spec.target_ref.uid;
2083    let endpoint = &repo.spec.s3.endpoint;
2084    let region = &repo.spec.s3.region;
2085    let operation_doc = build_safety_upload_operation_doc(
2086        restore,
2087        target,
2088        backup_id,
2089        manifest_key,
2090        &repo.spec.s3.bucket,
2091        &repo.spec.s3.prefix,
2092        endpoint,
2093        region,
2094        repo.spec.s3.force_path_style,
2095        repo.spec.s3.insecure,
2096        repo.spec
2097            .s3
2098            .ca_bundle_ref
2099            .as_deref()
2100            .map(|_| ca_bundle_path())
2101            .as_deref(),
2102        repo.spec.encryption.as_ref(),
2103    )?;
2104    let operation_cm_name = format!("{name}-op");
2105    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2106    let auth_method = &repo.spec.authentication.writer;
2107    let data_mover_image = data_mover_image();
2108    let init_command = vec![
2109        "kanidmd".to_string(),
2110        "database".to_string(),
2111        "backup".to_string(),
2112        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2113    ];
2114    let upload_script = format!(
2115        r#"set -eu
2116/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2117RESULT_FILE="/run/kaniop-result/result.json"
2118TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2119if [ ! -f "$RESULT_FILE" ]; then
2120  echo "ERROR: result document not found at $RESULT_FILE"
2121  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2122  exit 1
2123fi
2124if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2125  echo "ERROR: safety backup upload reported failure"
2126  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2127  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2128  exit 1
2129fi
2130dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2131echo "safety backup upload completed successfully"
2132"#
2133    );
2134    let job = Job {
2135        metadata: ObjectMeta {
2136            name: Some(name.to_string()),
2137            namespace: Some(ns.clone()),
2138            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2139            ..Default::default()
2140        },
2141        spec: Some(JobSpec {
2142            backoff_limit: Some(0),
2143            template: PodTemplateSpec {
2144                metadata: Some(ObjectMeta::default()),
2145                spec: Some(PodSpec {
2146                    automount_service_account_token: Some(false),
2147                    restart_policy: Some("Never".to_string()),
2148                    security_context: Some(source_job_pod_security_context(target)),
2149                    init_containers: Some(vec![Container {
2150                        name: "safety-backup".to_string(),
2151                        image: Some(restore.spec.restore_image.clone()),
2152                        command: Some(init_command),
2153                        security_context: Some(SecurityContext {
2154                            allow_privilege_escalation: Some(false),
2155                            capabilities: Some(Capabilities {
2156                                drop: Some(vec!["ALL".to_string()]),
2157                                ..Default::default()
2158                            }),
2159                            read_only_root_filesystem: Some(true),
2160                            seccomp_profile: Some(SeccompProfile {
2161                                type_: "RuntimeDefault".to_string(),
2162                                ..Default::default()
2163                            }),
2164                            ..Default::default()
2165                        }),
2166                        resources: Some(job_resource_requirements()),
2167                        env: Some(build_kanidm_db_env_vars(target)),
2168                        volume_mounts: Some(vec![
2169                            VolumeMount {
2170                                name: DATA_VOLUME.to_string(),
2171                                mount_path: DATA_PATH.to_string(),
2172                                read_only: Some(false),
2173                                ..Default::default()
2174                            },
2175                            VolumeMount {
2176                                name: TLS_VOLUME.to_string(),
2177                                mount_path: TLS_PATH.to_string(),
2178                                read_only: Some(true),
2179                                ..Default::default()
2180                            },
2181                            VolumeMount {
2182                                name: SHARED_VOLUME.to_string(),
2183                                mount_path: SHARED_VOL_PATH.to_string(),
2184                                ..Default::default()
2185                            },
2186                        ]),
2187                        ..Default::default()
2188                    }]),
2189                    containers: vec![Container {
2190                        name: "safety-upload".to_string(),
2191                        image: Some(data_mover_image),
2192                        command: Some(vec!["/bin/sh".to_string()]),
2193                        args: Some(vec!["-c".to_string(), upload_script]),
2194                        security_context: Some(hardened_security_context()),
2195                        resources: Some(job_resource_requirements()),
2196                        env: {
2197                            let mut env_vars =
2198                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2199                            env_vars.push(EnvVar {
2200                                name: "RUST_LOG".to_string(),
2201                                value: Some("info".to_string()),
2202                                ..Default::default()
2203                            });
2204                            if repo.spec.s3.ca_bundle_ref.is_some() {
2205                                env_vars.push(ca_bundle_env_var());
2206                            }
2207                            env_vars.extend(build_encryption_env_vars(
2208                                repo.spec
2209                                    .encryption
2210                                    .as_ref()
2211                                    .and_then(|e| e.key_ref.as_ref()),
2212                            ));
2213                            Some(env_vars)
2214                        },
2215                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2216                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2217                        volume_mounts: {
2218                            let mut mounts = vec![
2219                                VolumeMount {
2220                                    name: SHARED_VOLUME.to_string(),
2221                                    mount_path: SHARED_VOL_PATH.to_string(),
2222                                    read_only: Some(true),
2223                                    ..Default::default()
2224                                },
2225                                VolumeMount {
2226                                    name: "operation".to_string(),
2227                                    mount_path: "/run/kaniop".to_string(),
2228                                    read_only: Some(true),
2229                                    ..Default::default()
2230                                },
2231                                VolumeMount {
2232                                    name: "result".to_string(),
2233                                    mount_path: "/run/kaniop-result".to_string(),
2234                                    ..Default::default()
2235                                },
2236                            ];
2237                            mounts.extend(build_auth_volume_mounts(auth_method));
2238                            if repo.spec.s3.ca_bundle_ref.is_some() {
2239                                mounts.push(build_ca_bundle_volume_mount());
2240                            }
2241                            Some(mounts)
2242                        },
2243                        ..Default::default()
2244                    }],
2245                    volumes: {
2246                        let mut vols = vec![
2247                            Volume {
2248                                name: DATA_VOLUME.to_string(),
2249                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2250                                    claim_name: primary_pvc_name(target)?,
2251                                    read_only: Some(false),
2252                                }),
2253                                ..Default::default()
2254                            },
2255                            Volume {
2256                                name: TLS_VOLUME.to_string(),
2257                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2258                                    secret_name: Some(target.effective_tls_secret_name()),
2259                                    ..Default::default()
2260                                }),
2261                                ..Default::default()
2262                            },
2263                            Volume {
2264                                name: SHARED_VOLUME.to_string(),
2265                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2266                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2267                                    ..Default::default()
2268                                }),
2269                                ..Default::default()
2270                            },
2271                            Volume {
2272                                name: "operation".to_string(),
2273                                config_map: Some(ConfigMapVolumeSource {
2274                                    name: operation_cm_name,
2275                                    ..Default::default()
2276                                }),
2277                                ..Default::default()
2278                            },
2279                            Volume {
2280                                name: "result".to_string(),
2281                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2282                                    size_limit: Some(Quantity("16Mi".to_string())),
2283                                    ..Default::default()
2284                                }),
2285                                ..Default::default()
2286                            },
2287                        ];
2288                        vols.extend(build_auth_volumes(auth_method));
2289                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2290                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2291                        }
2292                        Some(vols)
2293                    },
2294                    ..Default::default()
2295                }),
2296            },
2297            ..Default::default()
2298        }),
2299        ..Default::default()
2300    };
2301    api.create(&PostParams::default(), &job)
2302        .await
2303        .map(|_| ())
2304        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2305}
2306
2307async fn ensure_source_prep_job(
2308    restore: &KanidmRestore,
2309    target: &Kanidm,
2310    ctx: &RestoreContext,
2311    name: &str,
2312) -> Result<()> {
2313    let ns = restore.namespace().unwrap();
2314    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2315    if api
2316        .get_opt(name)
2317        .await
2318        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2319        .is_some()
2320    {
2321        return Ok(());
2322    }
2323    let backup_name = restore
2324        .spec
2325        .source
2326        .backup_ref
2327        .as_ref()
2328        .map(|b| b.name.clone())
2329        .unwrap_or_default();
2330    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2331        .get(&backup_name)
2332        .await
2333        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2334    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2335        .get(&backup.spec.repository_ref.name)
2336        .await
2337        .map_err(|e| {
2338            Error::kube_error(
2339                "get",
2340                "KanidmBackupRepository",
2341                &ns,
2342                &backup.spec.repository_ref.name,
2343                e,
2344            )
2345        })?;
2346    let endpoint = &repo.spec.s3.endpoint;
2347    let region = &repo.spec.s3.region;
2348    let operation_doc = build_download_operation_doc(
2349        restore,
2350        target,
2351        &backup.spec.manifest_key,
2352        &backup.spec.backup_id,
2353        &repo.spec.s3.bucket,
2354        &repo.spec.s3.prefix,
2355        endpoint,
2356        region,
2357        repo.spec.s3.force_path_style,
2358        repo.spec.s3.insecure,
2359        repo.spec
2360            .s3
2361            .ca_bundle_ref
2362            .as_deref()
2363            .map(|_| ca_bundle_path())
2364            .as_deref(),
2365        repo.spec.encryption.as_ref(),
2366    );
2367    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2368    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2369    let auth_method = &repo.spec.authentication.reader;
2370    let data_mover_image = data_mover_image();
2371    let download_script = format!(
2372        r#"set -eu
2373/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2374RESULT_FILE="/run/kaniop-result/result.json"
2375TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2376if [ ! -f "$RESULT_FILE" ]; then
2377  echo "ERROR: result document not found at $RESULT_FILE"
2378  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2379  exit 1
2380fi
2381if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2382  echo "ERROR: source preparation download reported failure"
2383  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2384  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2385  exit 1
2386fi
2387dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2388echo "source preparation download completed successfully"
2389"#
2390    );
2391    let job = Job {
2392        metadata: ObjectMeta {
2393            name: Some(name.to_string()),
2394            namespace: Some(ns.clone()),
2395            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2396            ..Default::default()
2397        },
2398        spec: Some(JobSpec {
2399            backoff_limit: Some(0),
2400            template: PodTemplateSpec {
2401                metadata: Some(ObjectMeta::default()),
2402                spec: Some(PodSpec {
2403                    automount_service_account_token: Some(false),
2404                    restart_policy: Some("Never".to_string()),
2405                    security_context: Some(source_job_pod_security_context(target)),
2406                    containers: vec![Container {
2407                        name: "source-prep".to_string(),
2408                        image: Some(data_mover_image),
2409                        command: Some(vec!["/bin/sh".to_string()]),
2410                        args: Some(vec!["-c".to_string(), download_script]),
2411                        security_context: Some(hardened_security_context()),
2412                        resources: Some(job_resource_requirements()),
2413                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2414                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2415                        env: {
2416                            let mut env_vars = build_auth_env_vars(
2417                                auth_method,
2418                                &backup.spec.repository_ref.name,
2419                                AuthRole::Reader,
2420                            );
2421                            env_vars.push(EnvVar {
2422                                name: "RUST_LOG".to_string(),
2423                                value: Some("info".to_string()),
2424                                ..Default::default()
2425                            });
2426                            if repo.spec.s3.ca_bundle_ref.is_some() {
2427                                env_vars.push(ca_bundle_env_var());
2428                            }
2429                            env_vars.extend(build_encryption_env_vars(
2430                                repo.spec
2431                                    .encryption
2432                                    .as_ref()
2433                                    .and_then(|e| e.key_ref.as_ref()),
2434                            ));
2435                            Some(env_vars)
2436                        },
2437                        volume_mounts: {
2438                            let mut mounts = vec![
2439                                VolumeMount {
2440                                    name: DATA_VOLUME.to_string(),
2441                                    mount_path: DATA_PATH.to_string(),
2442                                    ..Default::default()
2443                                },
2444                                VolumeMount {
2445                                    name: "operation".to_string(),
2446                                    mount_path: "/run/kaniop".to_string(),
2447                                    read_only: Some(true),
2448                                    ..Default::default()
2449                                },
2450                                VolumeMount {
2451                                    name: "result".to_string(),
2452                                    mount_path: "/run/kaniop-result".to_string(),
2453                                    ..Default::default()
2454                                },
2455                            ];
2456                            mounts.extend(build_auth_volume_mounts(auth_method));
2457                            if repo.spec.s3.ca_bundle_ref.is_some() {
2458                                mounts.push(build_ca_bundle_volume_mount());
2459                            }
2460                            Some(mounts)
2461                        },
2462                        ..Default::default()
2463                    }],
2464                    volumes: {
2465                        let mut vols = vec![
2466                            Volume {
2467                                name: DATA_VOLUME.to_string(),
2468                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2469                                    claim_name: primary_pvc_name(target)?,
2470                                    read_only: Some(false),
2471                                }),
2472                                ..Default::default()
2473                            },
2474                            Volume {
2475                                name: "operation".to_string(),
2476                                config_map: Some(ConfigMapVolumeSource {
2477                                    name: operation_cm_name,
2478                                    ..Default::default()
2479                                }),
2480                                ..Default::default()
2481                            },
2482                            Volume {
2483                                name: "result".to_string(),
2484                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2485                                    size_limit: Some(Quantity("16Mi".to_string())),
2486                                    ..Default::default()
2487                                }),
2488                                ..Default::default()
2489                            },
2490                        ];
2491                        vols.extend(build_auth_volumes(auth_method));
2492                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2493                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2494                        }
2495                        Some(vols)
2496                    },
2497                    ..Default::default()
2498                }),
2499            },
2500            ..Default::default()
2501        }),
2502        ..Default::default()
2503    };
2504    api.create(&PostParams::default(), &job)
2505        .await
2506        .map(|_| ())
2507        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2508}
2509
2510#[allow(clippy::too_many_arguments)]
2511fn build_safety_upload_operation_doc(
2512    restore: &KanidmRestore,
2513    target: &Kanidm,
2514    backup_id: &str,
2515    manifest_key: &str,
2516    bucket: &str,
2517    prefix: &str,
2518    endpoint: &str,
2519    region: &str,
2520    force_path_style: bool,
2521    insecure: bool,
2522    ca_bundle_path: Option<&str>,
2523    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2524) -> Result<String> {
2525    let enc_mode =
2526        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2527    let enc_key_id = encryption
2528        .and_then(|e| e.key_id.as_ref())
2529        .map(|s| serde_json::Value::String(s.clone()));
2530    let op = UploadOperation {
2531        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2532        bucket: bucket.to_string(),
2533        prefix: prefix.to_string(),
2534        endpoint: endpoint.to_string(),
2535        region: region.to_string(),
2536        force_path_style,
2537        insecure,
2538        ca_bundle_path: ca_bundle_path.map(str::to_string),
2539        backup_id: backup_id.to_string(),
2540        namespace_uid: restore.namespace().unwrap_or_default(),
2541        kanidm_uid: restore.spec.target_ref.uid.clone(),
2542        kanidm_name: restore.spec.target_ref.name.clone(),
2543        domain: target.spec.domain.clone(),
2544        kanidm_version: target
2545            .status
2546            .as_ref()
2547            .and_then(|s| s.version.as_ref())
2548            .map(|v| v.image_tag.clone())
2549            .unwrap_or_default(),
2550        image_digest: None,
2551        consistency: "kanidm-offline".to_string(),
2552        reason: "restore-safety".to_string(),
2553        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2554        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2555        result_path: "/run/kaniop-result/result.json".to_string(),
2556        max_concurrent_parts: 4,
2557        max_retries: 3,
2558    };
2559    let _ = manifest_key;
2560    let doc = OperationDocument {
2561        api_version: OPERATION_DOC_VERSION.to_string(),
2562        kind: "OperationDocument".to_string(),
2563        spec: OperationSpec::Upload(op),
2564    };
2565    serde_json::to_string(&doc)
2566        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2567}
2568
2569async fn validate_backup_ref(
2570    restore: &KanidmRestore,
2571    target: &Kanidm,
2572    ctx: &RestoreContext,
2573) -> Result<()> {
2574    let backup_ref = restore
2575        .spec
2576        .source
2577        .backup_ref
2578        .as_ref()
2579        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2580    let ns = restore.namespace().unwrap();
2581    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2582        .get(&backup_ref.name)
2583        .await
2584        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2585    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2586        return Err(Error::MissingData(format!(
2587            "KanidmBackup '{}' is not Ready (phase: {:?})",
2588            backup_ref.name,
2589            backup.status.as_ref().map(|s| &s.phase)
2590        )));
2591    }
2592    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2593        return Err(Error::MissingData(format!(
2594            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2595            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2596        )));
2597    }
2598    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2599        return Err(Error::MissingData(format!(
2600            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2601            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2602        )));
2603    }
2604    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2605        .get(&backup.spec.repository_ref.name)
2606        .await
2607        .map_err(|e| {
2608            Error::kube_error(
2609                "get",
2610                "KanidmBackupRepository",
2611                &ns,
2612                &backup.spec.repository_ref.name,
2613                e,
2614            )
2615        })?;
2616    if !has_accepted_condition(
2617        &repo
2618            .status
2619            .as_ref()
2620            .map(|s| &s.conditions)
2621            .cloned()
2622            .unwrap_or_default(),
2623    ) {
2624        return Err(Error::MissingData(format!(
2625            "KanidmBackupRepository '{}' configuration has not been accepted",
2626            backup.spec.repository_ref.name
2627        )));
2628    }
2629    validate_backup_compatibility(restore, target, &backup)?;
2630    Ok(())
2631}
2632
2633fn has_accepted_condition(conditions: &[Condition]) -> bool {
2634    conditions
2635        .iter()
2636        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2637}
2638
2639fn validate_backup_compatibility(
2640    restore: &KanidmRestore,
2641    target: &Kanidm,
2642    backup: &KanidmBackup,
2643) -> Result<()> {
2644    let status = backup.status.as_ref();
2645    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2646        let target_version = target
2647            .status
2648            .as_ref()
2649            .and_then(|s| s.version.as_ref())
2650            .map(|v| v.image_tag.as_str());
2651        if let Some(tv) = target_version {
2652            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2653                return Err(Error::MissingData(format!(
2654                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2655                )));
2656            }
2657        }
2658    }
2659    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2660        if !backup_digest.is_empty()
2661            && restore.spec.restore_image.contains('@')
2662            && !restore.spec.restore_image.contains(backup_digest)
2663        {
2664            return Err(Error::MissingData(format!(
2665                "restore image digest does not match backup image digest '{backup_digest}'"
2666            )));
2667        }
2668    }
2669    Ok(())
2670}
2671
2672#[allow(clippy::too_many_arguments)]
2673fn build_download_operation_doc(
2674    restore: &KanidmRestore,
2675    target: &Kanidm,
2676    manifest_key: &str,
2677    expected_backup_id: &str,
2678    bucket: &str,
2679    prefix: &str,
2680    endpoint: &str,
2681    region: &str,
2682    force_path_style: bool,
2683    insecure: bool,
2684    ca_bundle_path: Option<&str>,
2685    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2686) -> String {
2687    let enc_mode =
2688        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2689    let enc_key_id = encryption
2690        .and_then(|e| e.key_id.as_ref())
2691        .map(|s| serde_json::Value::String(s.clone()));
2692    serde_json::json!({
2693        "apiVersion": "backup.kaniop.rs/v1alpha1",
2694        "kind": "OperationDocument",
2695        "operation": "download",
2696        "manifestKey": manifest_key,
2697        "bucket": bucket,
2698        "prefix": prefix,
2699        "endpoint": endpoint,
2700        "region": region,
2701        "forcePathStyle": force_path_style,
2702        "insecure": insecure,
2703        "caBundlePath": ca_bundle_path,
2704        "expectedBackupId": expected_backup_id,
2705        "expectedKanidmUid": restore.spec.target_ref.uid,
2706        "expectedDomain": target.spec.domain,
2707        "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2708        "resultPath": "/run/kaniop-result/result.json",
2709        "maxRetries": 3,
2710        "encryptionMode": enc_mode,
2711        "encryptionKeyId": enc_key_id,
2712    })
2713    .to_string()
2714}
2715
2716async fn ensure_operation_configmap(
2717    restore: &KanidmRestore,
2718    name: &str,
2719    data: &str,
2720    ns: &str,
2721    ctx: &RestoreContext,
2722) -> Result<()> {
2723    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2724    if api
2725        .get_opt(name)
2726        .await
2727        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2728        .is_some()
2729    {
2730        return Ok(());
2731    }
2732    let cm = ConfigMap {
2733        metadata: ObjectMeta {
2734            name: Some(name.to_string()),
2735            namespace: Some(ns.to_string()),
2736            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2737            ..Default::default()
2738        },
2739        data: Some(std::collections::BTreeMap::from([(
2740            "operation.json".to_string(),
2741            data.to_string(),
2742        )])),
2743        ..Default::default()
2744    };
2745    api.create(&PostParams::default(), &cm)
2746        .await
2747        .map(|_| ())
2748        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2749}
2750
2751#[cfg(test)]
2752mod tests {
2753    use super::{
2754        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2755        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2756        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2757        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2758        hardened_security_context, has_accepted_condition, is_remote_source,
2759        kanidm_job_security_context, mutable_image, requires_safety_backup, restore_job_name,
2760        safe_basename, validate_safety_backup_config, validate_source, verify_job_name,
2761    };
2762    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2763    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2764    use kube::api::ObjectMeta as ApiObjectMeta;
2765    use std::collections::BTreeMap;
2766
2767    fn make_restore(
2768        source: KanidmRestoreSource,
2769        safety: Option<SafetyBackupConfig>,
2770    ) -> KanidmRestore {
2771        KanidmRestore {
2772            metadata: ApiObjectMeta {
2773                name: Some("test-restore".to_string()),
2774                namespace: Some("default".to_string()),
2775                uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2776                ..Default::default()
2777            },
2778            spec: KanidmRestoreSpec {
2779                target_ref: KanidmRestoreTargetRef {
2780                    name: "corp-idm".to_string(),
2781                    uid: "test-uid-123".to_string(),
2782                },
2783                source,
2784                restore_image: "kanidm/server@sha256:abc".to_string(),
2785                safety_backup: safety,
2786            },
2787            status: None,
2788        }
2789    }
2790
2791    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2792        super::super::crd::Kanidm {
2793            status: Some(super::super::crd::KanidmStatus {
2794                available_replicas: 1,
2795                replicas: 1,
2796                unavailable_replicas: 0,
2797                updated_replicas: 1,
2798                replica_statuses: vec![],
2799                replica_column: "1/1".to_string(),
2800                secret_name: None,
2801                version: Some(super::super::crd::KanidmVersionStatus {
2802                    image_tag: image_tag.to_string(),
2803                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2804                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2805                }),
2806                domain_appearance_image: None,
2807                mail_sender: None,
2808                conditions: None,
2809            }),
2810            ..Default::default()
2811        }
2812    }
2813
2814    #[test]
2815    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2816        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2817    }
2818
2819    #[test]
2820    fn completed_restore_sets_ready_condition() {
2821        let mut status = KanidmRestoreStatus {
2822            phase: KanidmRestorePhase::Completed,
2823            ..Default::default()
2824        };
2825        super::update_restore_conditions(&mut status, Some(7));
2826        assert!(status.conditions.iter().any(|condition| {
2827            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2828        }));
2829        assert!(!status.conditions.iter().any(|condition| {
2830            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2831        }));
2832    }
2833
2834    #[test]
2835    fn failed_restore_sets_failed_condition() {
2836        let mut status = KanidmRestoreStatus {
2837            phase: KanidmRestorePhase::Failed,
2838            message: Some("verification failed".to_string()),
2839            ..Default::default()
2840        };
2841        super::update_restore_conditions(&mut status, Some(8));
2842        assert!(status.conditions.iter().any(|condition| {
2843            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2844        }));
2845    }
2846
2847    #[test]
2848    fn restore_filename_is_confined_to_backup_directory() {
2849        assert!(safe_basename("backup.json.gz"));
2850        assert!(!safe_basename("../backup.json.gz"));
2851        assert!(!safe_basename("a/b"));
2852        assert!(!safe_basename(""));
2853    }
2854
2855    #[test]
2856    fn restore_image_rejects_latest_and_unpinned_names() {
2857        assert!(mutable_image("kanidm/server:latest"));
2858        assert!(mutable_image("kanidm/server"));
2859        assert!(!mutable_image("kanidm/server:1.10.0"));
2860        assert!(!mutable_image("kanidm/server@sha256:abc"));
2861    }
2862
2863    #[test]
2864    fn source_validation_rejects_both_local_and_backup_ref() {
2865        let source = KanidmRestoreSource {
2866            local: Some(super::KanidmRestoreLocalSource {
2867                file_name: "backup.json".to_string(),
2868            }),
2869            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2870                name: "some-backup".to_string(),
2871            }),
2872        };
2873        let restore = make_restore(source, None);
2874        assert!(validate_source(&restore).is_err());
2875    }
2876
2877    #[test]
2878    fn source_validation_rejects_empty_source() {
2879        let source = KanidmRestoreSource {
2880            local: None,
2881            backup_ref: None,
2882        };
2883        let restore = make_restore(source, None);
2884        assert!(validate_source(&restore).is_err());
2885    }
2886
2887    #[test]
2888    fn source_validation_accepts_local_source() {
2889        let source = KanidmRestoreSource {
2890            local: Some(super::KanidmRestoreLocalSource {
2891                file_name: "backup.json".to_string(),
2892            }),
2893            backup_ref: None,
2894        };
2895        let restore = make_restore(source, None);
2896        assert!(validate_source(&restore).is_ok());
2897    }
2898
2899    #[test]
2900    fn source_validation_accepts_backup_ref_source() {
2901        let source = KanidmRestoreSource {
2902            local: None,
2903            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2904                name: "corp-idm-019c7c76".to_string(),
2905            }),
2906        };
2907        let restore = make_restore(source, None);
2908        assert!(validate_source(&restore).is_ok());
2909    }
2910
2911    #[test]
2912    fn source_validation_rejects_empty_backup_ref_name() {
2913        let source = KanidmRestoreSource {
2914            local: None,
2915            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2916                name: "".to_string(),
2917            }),
2918        };
2919        let restore = make_restore(source, None);
2920        assert!(validate_source(&restore).is_err());
2921    }
2922
2923    #[test]
2924    fn is_remote_source_true_for_backup_ref() {
2925        let source = KanidmRestoreSource {
2926            local: None,
2927            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2928                name: "backup-1".to_string(),
2929            }),
2930        };
2931        let restore = make_restore(source, None);
2932        assert!(is_remote_source(&restore));
2933    }
2934
2935    #[test]
2936    fn is_remote_source_false_for_local() {
2937        let source = KanidmRestoreSource {
2938            local: Some(super::KanidmRestoreLocalSource {
2939                file_name: "backup.json".to_string(),
2940            }),
2941            backup_ref: None,
2942        };
2943        let restore = make_restore(source, None);
2944        assert!(!is_remote_source(&restore));
2945    }
2946
2947    #[test]
2948    fn requires_safety_backup_default_true() {
2949        let source = KanidmRestoreSource {
2950            local: None,
2951            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2952                name: "backup-1".to_string(),
2953            }),
2954        };
2955        let restore = make_restore(source, None);
2956        assert!(requires_safety_backup(&restore));
2957    }
2958
2959    #[test]
2960    fn requires_safety_backup_false_when_skip() {
2961        let source = KanidmRestoreSource {
2962            local: None,
2963            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2964                name: "backup-1".to_string(),
2965            }),
2966        };
2967        let safety = SafetyBackupConfig {
2968            repository_ref: Some(SafetyBackupRepositoryRef {
2969                name: "offsite".to_string(),
2970            }),
2971            skip: true,
2972        };
2973        let restore = make_restore(source, Some(safety));
2974        assert!(!requires_safety_backup(&restore));
2975    }
2976
2977    #[test]
2978    fn break_glass_validation_rejects_skip_without_reason() {
2979        let source = KanidmRestoreSource {
2980            local: None,
2981            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2982                name: "backup-1".to_string(),
2983            }),
2984        };
2985        let safety = SafetyBackupConfig {
2986            repository_ref: Some(SafetyBackupRepositoryRef {
2987                name: "offsite".to_string(),
2988            }),
2989            skip: true,
2990        };
2991        let mut restore = make_restore(source, Some(safety));
2992        restore.metadata.annotations = Some(BTreeMap::from([(
2993            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
2994            "commander@example.com".to_string(),
2995        )]));
2996        assert!(validate_safety_backup_config(&restore).is_err());
2997    }
2998
2999    #[test]
3000    fn break_glass_validation_rejects_skip_without_approver() {
3001        let source = KanidmRestoreSource {
3002            local: None,
3003            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3004                name: "backup-1".to_string(),
3005            }),
3006        };
3007        let safety = SafetyBackupConfig {
3008            repository_ref: Some(SafetyBackupRepositoryRef {
3009                name: "offsite".to_string(),
3010            }),
3011            skip: true,
3012        };
3013        let mut restore = make_restore(source, Some(safety));
3014        restore.metadata.annotations = Some(BTreeMap::from([(
3015            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3016            "PVC is unreadable".to_string(),
3017        )]));
3018        assert!(validate_safety_backup_config(&restore).is_err());
3019    }
3020
3021    #[test]
3022    fn break_glass_validation_accepts_valid_annotations() {
3023        let source = KanidmRestoreSource {
3024            local: None,
3025            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3026                name: "backup-1".to_string(),
3027            }),
3028        };
3029        let safety = SafetyBackupConfig {
3030            repository_ref: Some(SafetyBackupRepositoryRef {
3031                name: "offsite".to_string(),
3032            }),
3033            skip: true,
3034        };
3035        let mut restore = make_restore(source, Some(safety));
3036        restore.metadata.annotations = Some(BTreeMap::from([
3037            (
3038                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3039                "PVC is unreadable".to_string(),
3040            ),
3041            (
3042                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3043                "commander@example.com".to_string(),
3044            ),
3045        ]));
3046        assert!(validate_safety_backup_config(&restore).is_ok());
3047    }
3048
3049    #[test]
3050    fn remote_source_requires_repository_ref_when_not_skipping() {
3051        let source = KanidmRestoreSource {
3052            local: None,
3053            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3054                name: "backup-1".to_string(),
3055            }),
3056        };
3057        let safety = SafetyBackupConfig {
3058            repository_ref: None,
3059            skip: false,
3060        };
3061        let restore = make_restore(source, Some(safety));
3062        assert!(validate_safety_backup_config(&restore).is_err());
3063    }
3064
3065    #[test]
3066    fn remote_source_accepts_repository_ref() {
3067        let source = KanidmRestoreSource {
3068            local: None,
3069            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3070                name: "backup-1".to_string(),
3071            }),
3072        };
3073        let safety = SafetyBackupConfig {
3074            repository_ref: Some(SafetyBackupRepositoryRef {
3075                name: "offsite".to_string(),
3076            }),
3077            skip: false,
3078        };
3079        let restore = make_restore(source, Some(safety));
3080        assert!(validate_safety_backup_config(&restore).is_ok());
3081    }
3082
3083    #[test]
3084    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3085        let phases = vec![
3086            KanidmRestorePhase::Pending,
3087            KanidmRestorePhase::Validating,
3088            KanidmRestorePhase::Quiescing,
3089            KanidmRestorePhase::SafetyBackup,
3090            KanidmRestorePhase::PreparingSource,
3091            KanidmRestorePhase::RestoringPrimary,
3092            KanidmRestorePhase::Verifying,
3093            KanidmRestorePhase::RebuildingReplicas,
3094            KanidmRestorePhase::Resuming,
3095            KanidmRestorePhase::Completed,
3096        ];
3097        assert_eq!(phases.len(), 10);
3098        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3099        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3100    }
3101
3102    #[test]
3103    fn status_default_has_empty_phase_timestamps() {
3104        let status = KanidmRestoreStatus::default();
3105        assert!(status.phase_timestamps.is_empty());
3106    }
3107
3108    #[test]
3109    fn status_default_has_empty_original_replicas() {
3110        let status = KanidmRestoreStatus::default();
3111        assert!(status.original_replicas.is_empty());
3112    }
3113
3114    #[test]
3115    fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3116        let mut target = super::super::crd::Kanidm::default();
3117        target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3118            run_as_non_root: Some(false),
3119            run_as_user: Some(389),
3120            run_as_group: Some(389),
3121            fs_group: Some(389),
3122            fs_group_change_policy: Some("OnRootMismatch".to_string()),
3123            supplemental_groups: Some(vec![390, 391]),
3124            ..Default::default()
3125        });
3126
3127        let ctx = super::source_job_pod_security_context(&target);
3128
3129        assert_eq!(ctx.run_as_non_root, Some(false));
3130        assert_eq!(ctx.run_as_user, Some(389));
3131        assert_eq!(ctx.run_as_group, Some(389));
3132        assert_eq!(ctx.fs_group, Some(389));
3133        assert_eq!(
3134            ctx.fs_group_change_policy.as_deref(),
3135            Some("OnRootMismatch")
3136        );
3137        assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3138        assert_eq!(
3139            ctx.seccomp_profile
3140                .as_ref()
3141                .map(|profile| profile.type_.as_str()),
3142            Some("RuntimeDefault")
3143        );
3144    }
3145
3146    #[test]
3147    fn hardened_security_context_drops_all_capabilities() {
3148        let ctx = hardened_security_context();
3149        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3150        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3151        assert_eq!(ctx.run_as_non_root, Some(true));
3152        let caps = ctx.capabilities.unwrap();
3153        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3154        assert_eq!(
3155            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3156            Some("RuntimeDefault")
3157        );
3158    }
3159
3160    #[test]
3161    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3162        let ctx = kanidm_job_security_context();
3163        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3164        assert!(ctx.run_as_non_root.is_none());
3165        assert!(ctx.read_only_root_filesystem.is_none());
3166        let caps = ctx.capabilities.unwrap();
3167        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3168        assert_eq!(
3169            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3170            Some("RuntimeDefault")
3171        );
3172    }
3173
3174    #[test]
3175    fn safety_backup_job_name_is_deterministic() {
3176        let source = KanidmRestoreSource {
3177            local: Some(super::KanidmRestoreLocalSource {
3178                file_name: "backup.json".to_string(),
3179            }),
3180            backup_ref: None,
3181        };
3182        let restore = make_restore(source, None);
3183        let name1 = super::safety_backup_job_name(&restore);
3184        let name2 = super::safety_backup_job_name(&restore);
3185        assert_eq!(name1, name2);
3186        assert_eq!(name1, "test-restore-safety-backup");
3187    }
3188
3189    #[test]
3190    fn source_prep_job_name_is_deterministic() {
3191        let source = KanidmRestoreSource {
3192            local: None,
3193            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3194                name: "backup-1".to_string(),
3195            }),
3196        };
3197        let restore = make_restore(source, None);
3198        let name = super::source_prep_job_name(&restore);
3199        assert_eq!(name, "test-restore-source-prep");
3200    }
3201
3202    #[test]
3203    fn break_glass_condition_is_recorded_in_status() {
3204        let mut status = KanidmRestoreStatus::default();
3205        let condition = super::restore_condition(
3206            &status.conditions,
3207            CONDITION_BREAK_GLASS,
3208            CONDITION_TRUE,
3209            "BreakGlassOverride",
3210            "Safety backup skipped",
3211            Some(1),
3212        );
3213        status.conditions.push(condition);
3214        assert!(status.conditions.iter().any(|c| {
3215            c.type_ == CONDITION_BREAK_GLASS
3216                && c.status == CONDITION_TRUE
3217                && c.reason == "BreakGlassOverride"
3218        }));
3219    }
3220
3221    #[test]
3222    fn replica_count_entry_serialization() {
3223        let entry = ReplicaCountEntry {
3224            group: "primary".to_string(),
3225            replicas: 3,
3226        };
3227        let json = serde_json::to_string(&entry).unwrap();
3228        assert!(json.contains("\"group\":\"primary\""));
3229        assert!(json.contains("\"replicas\":3"));
3230    }
3231
3232    #[test]
3233    fn local_source_requires_repository_ref_when_not_skipping() {
3234        let source = KanidmRestoreSource {
3235            local: Some(super::KanidmRestoreLocalSource {
3236                file_name: "backup.json".to_string(),
3237            }),
3238            backup_ref: None,
3239        };
3240        let safety = SafetyBackupConfig {
3241            repository_ref: None,
3242            skip: false,
3243        };
3244        let restore = make_restore(source, Some(safety));
3245        assert!(validate_safety_backup_config(&restore).is_err());
3246    }
3247
3248    #[test]
3249    fn local_source_accepts_repository_ref_when_not_skipping() {
3250        let source = KanidmRestoreSource {
3251            local: Some(super::KanidmRestoreLocalSource {
3252                file_name: "backup.json".to_string(),
3253            }),
3254            backup_ref: None,
3255        };
3256        let safety = SafetyBackupConfig {
3257            repository_ref: Some(SafetyBackupRepositoryRef {
3258                name: "offsite".to_string(),
3259            }),
3260            skip: false,
3261        };
3262        let restore = make_restore(source, Some(safety));
3263        assert!(validate_safety_backup_config(&restore).is_ok());
3264    }
3265
3266    #[test]
3267    fn skip_without_break_glass_annotations_is_rejected() {
3268        let source = KanidmRestoreSource {
3269            local: Some(super::KanidmRestoreLocalSource {
3270                file_name: "backup.json".to_string(),
3271            }),
3272            backup_ref: None,
3273        };
3274        let safety = SafetyBackupConfig {
3275            repository_ref: None,
3276            skip: true,
3277        };
3278        let restore = make_restore(source, Some(safety));
3279        assert!(validate_safety_backup_config(&restore).is_err());
3280    }
3281
3282    #[test]
3283    fn has_accepted_condition_true() {
3284        let conditions = vec![Condition {
3285            type_: "Ready".to_string(),
3286            status: "True".to_string(),
3287            reason: "Accepted".to_string(),
3288            message: String::new(),
3289            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3290                k8s_openapi::jiff::Timestamp::now(),
3291            ),
3292            observed_generation: None,
3293        }];
3294        assert!(has_accepted_condition(&conditions));
3295    }
3296
3297    #[test]
3298    fn has_accepted_condition_false_when_not_accepted_reason() {
3299        let conditions = vec![Condition {
3300            type_: "Ready".to_string(),
3301            status: "True".to_string(),
3302            reason: "SomeOtherReason".to_string(),
3303            message: String::new(),
3304            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3305                k8s_openapi::jiff::Timestamp::now(),
3306            ),
3307            observed_generation: None,
3308        }];
3309        assert!(!has_accepted_condition(&conditions));
3310    }
3311
3312    #[test]
3313    fn has_accepted_condition_false_when_status_false() {
3314        let conditions = vec![Condition {
3315            type_: "Ready".to_string(),
3316            status: "False".to_string(),
3317            reason: "Accepted".to_string(),
3318            message: String::new(),
3319            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3320                k8s_openapi::jiff::Timestamp::now(),
3321            ),
3322            observed_generation: None,
3323        }];
3324        assert!(!has_accepted_condition(&conditions));
3325    }
3326
3327    #[test]
3328    fn has_accepted_condition_empty() {
3329        assert!(!has_accepted_condition(&[]));
3330    }
3331
3332    #[test]
3333    fn safety_backup_upload_operation_doc_is_valid() {
3334        let source = KanidmRestoreSource {
3335            local: None,
3336            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3337                name: "backup-1".to_string(),
3338            }),
3339        };
3340        let mut restore = make_restore(source, None);
3341        restore.metadata.namespace = Some("test-ns".to_string());
3342        let target = super::super::crd::Kanidm::default();
3343        let doc_str = super::build_safety_upload_operation_doc(
3344            &restore,
3345            &target,
3346            "test-backup-id",
3347            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3348            "my-bucket",
3349            "prod",
3350            "https://s3.example.com",
3351            "us-east-1",
3352            false,
3353            false,
3354            None,
3355            None,
3356        )
3357        .unwrap();
3358        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3359        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3360        assert_eq!(parsed["kind"], "OperationDocument");
3361        assert_eq!(parsed["operation"], "upload");
3362        assert_eq!(parsed["bucket"], "my-bucket");
3363        assert_eq!(parsed["prefix"], "prod");
3364        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3365        assert_eq!(parsed["region"], "us-east-1");
3366        assert_eq!(parsed["backupId"], "test-backup-id");
3367        assert_eq!(parsed["consistency"], "kanidm-offline");
3368        assert_eq!(parsed["reason"], "restore-safety");
3369        assert_eq!(
3370            parsed["payloadPath"],
3371            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3372        );
3373        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3374    }
3375
3376    #[test]
3377    fn download_operation_doc_uses_real_repository_data() {
3378        let source = KanidmRestoreSource {
3379            local: None,
3380            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3381                name: "backup-1".to_string(),
3382            }),
3383        };
3384        let restore = make_restore(source, None);
3385        let target = super::super::crd::Kanidm::default();
3386        let doc_str = super::build_download_operation_doc(
3387            &restore,
3388            &target,
3389            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3390            "019c7c76-f423-7a12-8f41-2bea7588a303",
3391            "real-bucket",
3392            "real-prefix",
3393            "https://real-endpoint.com",
3394            "eu-west-1",
3395            true,
3396            false,
3397            None,
3398            None,
3399        );
3400        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3401        assert_eq!(parsed["bucket"], "real-bucket");
3402        assert_eq!(parsed["prefix"], "real-prefix");
3403        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3404        assert_eq!(parsed["region"], "eu-west-1");
3405        assert_eq!(parsed["forcePathStyle"], true);
3406        assert_eq!(
3407            parsed["manifestKey"],
3408            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3409        );
3410        assert_eq!(
3411            parsed["expectedBackupId"],
3412            "019c7c76-f423-7a12-8f41-2bea7588a303"
3413        );
3414    }
3415
3416    #[test]
3417    fn safety_backup_ref_is_set_after_verification() {
3418        let status = KanidmRestoreStatus {
3419            phase: KanidmRestorePhase::SafetyBackup,
3420            safety_backup_job_name: Some("test-safety-backup".to_string()),
3421            safety_backup_ref: Some("safety-test-restore".to_string()),
3422            safety_backup_expected_backup_id: Some(
3423                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3424            ),
3425            safety_backup_manifest_key: Some(
3426                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3427            ),
3428            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3429            ..Default::default()
3430        };
3431        assert!(status.safety_backup_ref.is_some());
3432        assert!(
3433            status
3434                .safety_backup_ref
3435                .as_ref()
3436                .unwrap()
3437                .starts_with("safety-")
3438        );
3439        assert!(status.safety_backup_expected_backup_id.is_some());
3440        assert!(status.safety_backup_manifest_key.is_some());
3441        assert!(status.safety_backup_payload_sha256.is_some());
3442    }
3443
3444    #[test]
3445    fn database_mutation_not_started_without_safety_backup_ref() {
3446        let status = KanidmRestoreStatus {
3447            phase: KanidmRestorePhase::PreparingSource,
3448            safety_backup_ref: None,
3449            database_mutation_started: false,
3450            ..Default::default()
3451        };
3452        assert!(!status.database_mutation_started);
3453        assert!(status.safety_backup_ref.is_none());
3454    }
3455
3456    #[test]
3457    fn pre_boundary_failure_does_not_set_mutation_started() {
3458        let status = KanidmRestoreStatus {
3459            phase: KanidmRestorePhase::Failed,
3460            message: Some(
3461                "safety backup job failed; target restored to original state".to_string(),
3462            ),
3463            database_mutation_started: false,
3464            ..Default::default()
3465        };
3466        assert!(!status.database_mutation_started);
3467    }
3468
3469    #[test]
3470    fn shared_volume_constants_are_distinct() {
3471        assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3472        assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3473    }
3474
3475    #[test]
3476    fn validate_backup_compatibility_matching_versions_passes() {
3477        let source = KanidmRestoreSource {
3478            local: None,
3479            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3480                name: "backup-1".to_string(),
3481            }),
3482        };
3483        let restore = make_restore(source, None);
3484        let target = kanidm_with_version("1.10.4");
3485        let backup = kaniop_backup_core::crd::KanidmBackup {
3486            metadata: kube::api::ObjectMeta::default(),
3487            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3488                backup_id: "id".to_string(),
3489                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3490                    name: "k".to_string(),
3491                    uid: "uid".to_string(),
3492                },
3493                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3494                    name: "repo".to_string(),
3495                },
3496                manifest_key: "key".to_string(),
3497            },
3498            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3499                kanidm_version: Some("1.10.4".to_string()),
3500                ..Default::default()
3501            }),
3502        };
3503        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3504    }
3505
3506    #[test]
3507    fn validate_backup_compatibility_version_mismatch_fails() {
3508        let source = KanidmRestoreSource {
3509            local: None,
3510            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3511                name: "backup-1".to_string(),
3512            }),
3513        };
3514        let restore = make_restore(source, None);
3515        let target = kanidm_with_version("1.11.0");
3516        let backup = kaniop_backup_core::crd::KanidmBackup {
3517            metadata: kube::api::ObjectMeta::default(),
3518            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3519                backup_id: "id".to_string(),
3520                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3521                    name: "k".to_string(),
3522                    uid: "uid".to_string(),
3523                },
3524                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3525                    name: "repo".to_string(),
3526                },
3527                manifest_key: "key".to_string(),
3528            },
3529            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3530                kanidm_version: Some("1.10.4".to_string()),
3531                ..Default::default()
3532            }),
3533        };
3534        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3535    }
3536
3537    #[test]
3538    fn validate_backup_compatibility_no_backup_version_passes() {
3539        let source = KanidmRestoreSource {
3540            local: None,
3541            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3542                name: "backup-1".to_string(),
3543            }),
3544        };
3545        let restore = make_restore(source, None);
3546        let target = super::super::crd::Kanidm::default();
3547        let backup = kaniop_backup_core::crd::KanidmBackup {
3548            metadata: kube::api::ObjectMeta::default(),
3549            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3550                backup_id: "id".to_string(),
3551                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3552                    name: "k".to_string(),
3553                    uid: "uid".to_string(),
3554                },
3555                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3556                    name: "repo".to_string(),
3557                },
3558                manifest_key: "key".to_string(),
3559            },
3560            status: None,
3561        };
3562        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3563    }
3564
3565    #[test]
3566    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3567        let source = KanidmRestoreSource {
3568            local: None,
3569            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3570                name: "backup-1".to_string(),
3571            }),
3572        };
3573        let mut restore = make_restore(source, None);
3574        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3575        let target = super::super::crd::Kanidm::default();
3576        let backup = kaniop_backup_core::crd::KanidmBackup {
3577            metadata: kube::api::ObjectMeta::default(),
3578            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3579                backup_id: "id".to_string(),
3580                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3581                    name: "k".to_string(),
3582                    uid: "uid".to_string(),
3583                },
3584                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3585                    name: "repo".to_string(),
3586                },
3587                manifest_key: "key".to_string(),
3588            },
3589            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3590                image_digest: Some("sha256:originaldigest".to_string()),
3591                ..Default::default()
3592            }),
3593        };
3594        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3595    }
3596
3597    #[test]
3598    fn validate_backup_compatibility_image_digest_match_passes() {
3599        let source = KanidmRestoreSource {
3600            local: None,
3601            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3602                name: "backup-1".to_string(),
3603            }),
3604        };
3605        let mut restore = make_restore(source, None);
3606        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3607        let target = super::super::crd::Kanidm::default();
3608        let backup = kaniop_backup_core::crd::KanidmBackup {
3609            metadata: kube::api::ObjectMeta::default(),
3610            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3611                backup_id: "id".to_string(),
3612                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3613                    name: "k".to_string(),
3614                    uid: "uid".to_string(),
3615                },
3616                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3617                    name: "repo".to_string(),
3618                },
3619                manifest_key: "key".to_string(),
3620            },
3621            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3622                image_digest: Some("sha256:abc123".to_string()),
3623                ..Default::default()
3624            }),
3625        };
3626        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3627    }
3628
3629    #[test]
3630    fn safety_backup_id_is_deterministic() {
3631        let source = KanidmRestoreSource {
3632            local: None,
3633            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3634                name: "backup-1".to_string(),
3635            }),
3636        };
3637        let restore = make_restore(source, None);
3638        let id1 = super::compute_safety_backup_id(&restore).unwrap();
3639        let id2 = super::compute_safety_backup_id(&restore).unwrap();
3640        assert_eq!(id1, id2);
3641        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3642    }
3643
3644    #[test]
3645    fn safety_manifest_key_is_deterministic() {
3646        let source = KanidmRestoreSource {
3647            local: None,
3648            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3649                name: "backup-1".to_string(),
3650            }),
3651        };
3652        let restore = make_restore(source, None);
3653        let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3654        let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3655        assert_eq!(key1, key2);
3656        assert!(key1.contains("manifest.json"));
3657        assert!(key1.starts_with("v1/tenants/"));
3658    }
3659
3660    #[test]
3661    fn safety_backup_id_survives_restart() {
3662        let source = KanidmRestoreSource {
3663            local: None,
3664            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3665                name: "backup-1".to_string(),
3666            }),
3667        };
3668        let mut restore = make_restore(source, None);
3669        let original_id = super::compute_safety_backup_id(&restore).unwrap();
3670        restore.status = Some(KanidmRestoreStatus {
3671            phase: KanidmRestorePhase::SafetyBackup,
3672            safety_backup_expected_backup_id: Some(original_id.clone()),
3673            ..Default::default()
3674        });
3675        let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3676        assert_eq!(original_id, recomputed_id);
3677    }
3678
3679    #[test]
3680    fn safety_backup_id_derived_from_uid_not_name() {
3681        let source = KanidmRestoreSource {
3682            local: None,
3683            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3684                name: "backup-1".to_string(),
3685            }),
3686        };
3687        let restore1 = make_restore(source.clone(), None);
3688        let mut restore2 = make_restore(source, None);
3689        restore2.metadata.name = Some("different-name-same-uid".to_string());
3690        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3691        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3692        assert_eq!(id1, id2);
3693    }
3694
3695    #[test]
3696    fn safety_backup_id_differs_for_different_uids() {
3697        let source = KanidmRestoreSource {
3698            local: None,
3699            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3700                name: "backup-1".to_string(),
3701            }),
3702        };
3703        let restore1 = make_restore(source.clone(), None);
3704        let mut restore2 = make_restore(source, None);
3705        restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3706        let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3707        let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3708        assert_ne!(id1, id2);
3709    }
3710
3711    #[test]
3712    fn safety_backup_id_errors_without_uid() {
3713        let source = KanidmRestoreSource {
3714            local: None,
3715            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3716                name: "backup-1".to_string(),
3717            }),
3718        };
3719        let mut restore = make_restore(source, None);
3720        restore.metadata.uid = None;
3721        assert!(super::compute_safety_backup_id(&restore).is_err());
3722    }
3723
3724    #[test]
3725    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3726        let source = KanidmRestoreSource {
3727            local: None,
3728            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3729                name: "backup-1".to_string(),
3730            }),
3731        };
3732        let restore = make_restore(source, None);
3733        assert!(super::requires_safety_backup(&restore));
3734        let status_without_ref = KanidmRestoreStatus {
3735            phase: KanidmRestorePhase::RestoringPrimary,
3736            safety_backup_ref: None,
3737            database_mutation_started: false,
3738            ..Default::default()
3739        };
3740        assert!(status_without_ref.safety_backup_ref.is_none());
3741        assert!(!status_without_ref.database_mutation_started);
3742    }
3743
3744    #[test]
3745    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3746        let status_with_ref = KanidmRestoreStatus {
3747            phase: KanidmRestorePhase::RestoringPrimary,
3748            safety_backup_ref: Some("safety-test-restore".to_string()),
3749            safety_backup_manifest_key: Some(
3750                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3751            ),
3752            safety_backup_payload_sha256: Some("abc123".to_string()),
3753            database_mutation_started: false,
3754            ..Default::default()
3755        };
3756        assert!(status_with_ref.safety_backup_ref.is_some());
3757        assert!(!status_with_ref.database_mutation_started);
3758    }
3759
3760    #[test]
3761    fn result_document_verification_rejects_missing_backup_id() {
3762        let result_json = r#"{
3763            "apiVersion": "backup.kaniop.rs/v1alpha1",
3764            "kind": "ResultDocument",
3765            "operation": "upload",
3766            "success": true,
3767            "exitCode": "success",
3768            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3769            "payloadSha256": "abc123"
3770        }"#;
3771        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3772        assert!(result.backup_id.is_none());
3773    }
3774
3775    #[test]
3776    fn result_document_verification_rejects_missing_manifest_key() {
3777        let result_json = r#"{
3778            "apiVersion": "backup.kaniop.rs/v1alpha1",
3779            "kind": "ResultDocument",
3780            "operation": "upload",
3781            "success": true,
3782            "exitCode": "success",
3783            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3784            "payloadSha256": "abc123"
3785        }"#;
3786        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3787        assert!(result.manifest_key.is_none());
3788    }
3789
3790    #[test]
3791    fn result_document_verification_rejects_missing_payload_sha256() {
3792        let result_json = r#"{
3793            "apiVersion": "backup.kaniop.rs/v1alpha1",
3794            "kind": "ResultDocument",
3795            "operation": "upload",
3796            "success": true,
3797            "exitCode": "success",
3798            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3799            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3800        }"#;
3801        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3802        assert!(result.payload_sha256.is_none());
3803    }
3804
3805    #[test]
3806    fn result_document_verification_rejects_failure_exit_code() {
3807        let result_json = r#"{
3808            "apiVersion": "backup.kaniop.rs/v1alpha1",
3809            "kind": "ResultDocument",
3810            "operation": "upload",
3811            "success": false,
3812            "exitCode": "integrity",
3813            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3814            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3815            "payloadSha256": "abc123"
3816        }"#;
3817        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3818        assert!(!result.success);
3819        assert_eq!(
3820            result.exit_code,
3821            kaniop_backup_core::result::ExitCode::Integrity
3822        );
3823    }
3824
3825    #[test]
3826    fn result_document_verification_rejects_wrong_operation() {
3827        let result_json = r#"{
3828            "apiVersion": "backup.kaniop.rs/v1alpha1",
3829            "kind": "ResultDocument",
3830            "operation": "download",
3831            "success": true,
3832            "exitCode": "success",
3833            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3834            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3835            "payloadSha256": "abc123"
3836        }"#;
3837        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3838        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3839    }
3840
3841    #[test]
3842    fn result_document_verification_accepts_valid_document() {
3843        let result_json = r#"{
3844            "apiVersion": "backup.kaniop.rs/v1alpha1",
3845            "kind": "ResultDocument",
3846            "operation": "upload",
3847            "success": true,
3848            "exitCode": "success",
3849            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3850            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3851            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3852        }"#;
3853        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3854        assert!(result.success);
3855        assert_eq!(
3856            result.exit_code,
3857            kaniop_backup_core::result::ExitCode::Success
3858        );
3859        assert_eq!(result.operation, "upload");
3860        assert!(result.backup_id.is_some());
3861        assert!(result.manifest_key.is_some());
3862        assert!(result.payload_sha256.is_some());
3863    }
3864
3865    #[test]
3866    fn termination_message_path_constant_is_valid() {
3867        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3868        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3869    }
3870
3871    #[test]
3872    fn status_tracks_safety_backup_evidence() {
3873        let status = KanidmRestoreStatus {
3874            phase: KanidmRestorePhase::PreparingSource,
3875            safety_backup_ref: Some("safety-test-restore".to_string()),
3876            safety_backup_expected_backup_id: Some(
3877                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3878            ),
3879            safety_backup_manifest_key: Some(
3880                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3881            ),
3882            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3883            ..Default::default()
3884        };
3885        assert!(status.safety_backup_ref.is_some());
3886        assert!(status.safety_backup_expected_backup_id.is_some());
3887        assert!(status.safety_backup_manifest_key.is_some());
3888        assert!(status.safety_backup_payload_sha256.is_some());
3889    }
3890
3891    #[test]
3892    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3893        let invalid_json = "not valid json";
3894        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3895        assert!(result.is_err());
3896    }
3897
3898    #[test]
3899    fn fail_closed_semantics_preserved_on_oversized_result() {
3900        let oversized_json = format!(
3901            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3902            kaniop_backup_core::result::RESULT_DOC_VERSION,
3903            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3904        );
3905        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3906        assert!(result.is_err());
3907    }
3908
3909    #[test]
3910    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3911        let source = KanidmRestoreSource {
3912            local: None,
3913            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3914                name: "backup-1".to_string(),
3915            }),
3916        };
3917        let mut restore = make_restore(source, None);
3918        restore.metadata.namespace = Some("test-ns".to_string());
3919        let target = super::super::crd::Kanidm::default();
3920        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3921        let doc_str = super::build_safety_upload_operation_doc(
3922            &restore,
3923            &target,
3924            "test-backup-id",
3925            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3926            "my-bucket",
3927            "prod",
3928            "https://s3.example.com",
3929            "us-east-1",
3930            false,
3931            false,
3932            Some(&ca_path),
3933            None,
3934        )
3935        .unwrap();
3936        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3937        assert_eq!(parsed["caBundlePath"], ca_path);
3938    }
3939
3940    #[test]
3941    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3942        let source = KanidmRestoreSource {
3943            local: None,
3944            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3945                name: "backup-1".to_string(),
3946            }),
3947        };
3948        let mut restore = make_restore(source, None);
3949        restore.metadata.namespace = Some("test-ns".to_string());
3950        let target = super::super::crd::Kanidm::default();
3951        let doc_str = super::build_safety_upload_operation_doc(
3952            &restore,
3953            &target,
3954            "test-backup-id",
3955            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3956            "my-bucket",
3957            "prod",
3958            "https://s3.example.com",
3959            "us-east-1",
3960            false,
3961            false,
3962            None,
3963            None,
3964        )
3965        .unwrap();
3966        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3967        assert!(parsed["caBundlePath"].is_null());
3968    }
3969
3970    #[test]
3971    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3972        let source = KanidmRestoreSource {
3973            local: None,
3974            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3975                name: "backup-1".to_string(),
3976            }),
3977        };
3978        let restore = make_restore(source, None);
3979        let target = super::super::crd::Kanidm::default();
3980        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3981        let doc_str = super::build_download_operation_doc(
3982            &restore,
3983            &target,
3984            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3985            "019c7c76-f423-7a12-8f41-2bea7588a303",
3986            "real-bucket",
3987            "real-prefix",
3988            "https://real-endpoint.com",
3989            "eu-west-1",
3990            true,
3991            false,
3992            Some(&ca_path),
3993            None,
3994        );
3995        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3996        assert_eq!(parsed["caBundlePath"], ca_path);
3997    }
3998
3999    #[test]
4000    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4001        let source = KanidmRestoreSource {
4002            local: None,
4003            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4004                name: "backup-1".to_string(),
4005            }),
4006        };
4007        let restore = make_restore(source, None);
4008        let target = super::super::crd::Kanidm::default();
4009        let doc_str = super::build_download_operation_doc(
4010            &restore,
4011            &target,
4012            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4013            "019c7c76-f423-7a12-8f41-2bea7588a303",
4014            "real-bucket",
4015            "real-prefix",
4016            "https://real-endpoint.com",
4017            "eu-west-1",
4018            true,
4019            false,
4020            None,
4021            None,
4022        );
4023        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4024        assert!(parsed["caBundlePath"].is_null());
4025    }
4026
4027    #[test]
4028    fn source_prep_result_operation_is_download() {
4029        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4030        assert_ne!(
4031            super::SOURCE_PREP_RESULT_OPERATION,
4032            super::SAFETY_BACKUP_RESULT_OPERATION
4033        );
4034    }
4035
4036    #[test]
4037    fn download_result_document_verification_accepts_valid_document() {
4038        let result_json = r#"{
4039            "apiVersion": "backup.kaniop.rs/v1alpha1",
4040            "kind": "ResultDocument",
4041            "operation": "download",
4042            "success": true,
4043            "exitCode": "success",
4044            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4045            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4046            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4047        }"#;
4048        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4049        assert!(result.success);
4050        assert_eq!(
4051            result.exit_code,
4052            kaniop_backup_core::result::ExitCode::Success
4053        );
4054        assert_eq!(result.operation, "download");
4055        assert_eq!(
4056            result.backup_id.as_deref(),
4057            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4058        );
4059        assert!(result.manifest_key.is_some());
4060        assert!(result.payload_sha256.is_some());
4061    }
4062
4063    #[test]
4064    fn download_result_document_rejects_wrong_operation() {
4065        let result_json = r#"{
4066            "apiVersion": "backup.kaniop.rs/v1alpha1",
4067            "kind": "ResultDocument",
4068            "operation": "upload",
4069            "success": true,
4070            "exitCode": "success",
4071            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4072            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4073            "payloadSha256": "abc123"
4074        }"#;
4075        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4076        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4077    }
4078
4079    #[test]
4080    fn download_result_document_rejects_failure_success_field() {
4081        let result_json = r#"{
4082            "apiVersion": "backup.kaniop.rs/v1alpha1",
4083            "kind": "ResultDocument",
4084            "operation": "download",
4085            "success": false,
4086            "exitCode": "retryable",
4087            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4088            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4089            "payloadSha256": "abc123"
4090        }"#;
4091        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4092        assert!(!result.success);
4093        assert_ne!(
4094            result.exit_code,
4095            kaniop_backup_core::result::ExitCode::Success
4096        );
4097    }
4098
4099    #[test]
4100    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4101        let status = KanidmRestoreStatus {
4102            phase: KanidmRestorePhase::PreparingSource,
4103            database_mutation_started: false,
4104            ..Default::default()
4105        };
4106        assert!(!status.database_mutation_started);
4107        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4108    }
4109
4110    #[test]
4111    fn backup_job_volume_size_default_is_10gi() {
4112        let qty = crate::controller::backup_job_volume_size();
4113        assert_eq!(qty, Quantity("10Gi".to_string()));
4114    }
4115
4116    #[test]
4117    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4118        use kaniop_backup_core::auth::build_encryption_env_vars;
4119        use kaniop_backup_core::crd::SecretRef;
4120
4121        let key_ref = SecretRef {
4122            name: "kek-secret".to_string(),
4123        };
4124        let env_vars = build_encryption_env_vars(Some(&key_ref));
4125        assert_eq!(env_vars.len(), 1);
4126        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4127        let secret_ref = env_vars[0]
4128            .value_from
4129            .as_ref()
4130            .unwrap()
4131            .secret_key_ref
4132            .as_ref()
4133            .unwrap();
4134        assert_eq!(secret_ref.name, "kek-secret");
4135        assert_eq!(secret_ref.key, "encryption-key");
4136    }
4137
4138    #[test]
4139    fn encryption_env_vars_absent_when_no_key_ref() {
4140        use kaniop_backup_core::auth::build_encryption_env_vars;
4141
4142        let env_vars = build_encryption_env_vars(None);
4143        assert!(
4144            env_vars.is_empty(),
4145            "encryption env vars must be empty when key_ref is None"
4146        );
4147    }
4148
4149    #[test]
4150    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4151        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4152
4153        let source = KanidmRestoreSource {
4154            local: None,
4155            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4156                name: "backup-1".to_string(),
4157            }),
4158        };
4159        let mut restore = make_restore(source, None);
4160        restore.metadata.namespace = Some("test-ns".to_string());
4161        let target = super::super::crd::Kanidm::default();
4162        let encryption = RepositoryEncryption {
4163            mode: EncryptionMode::ClientSide,
4164            key_id: None,
4165            key_ref: Some(SecretRef {
4166                name: "kek-secret".to_string(),
4167            }),
4168        };
4169        let doc_str = super::build_safety_upload_operation_doc(
4170            &restore,
4171            &target,
4172            "test-backup-id",
4173            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4174            "my-bucket",
4175            "prod",
4176            "https://s3.example.com",
4177            "us-east-1",
4178            false,
4179            false,
4180            None,
4181            Some(&encryption),
4182        )
4183        .unwrap();
4184        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4185        assert_eq!(parsed["encryptionMode"], "clientSide");
4186        assert!(parsed["encryptionKeyId"].is_null());
4187    }
4188
4189    #[test]
4190    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4191        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4192
4193        let source = KanidmRestoreSource {
4194            local: None,
4195            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4196                name: "backup-1".to_string(),
4197            }),
4198        };
4199        let restore = make_restore(source, None);
4200        let target = super::super::crd::Kanidm::default();
4201        let encryption = RepositoryEncryption {
4202            mode: EncryptionMode::ProviderKms,
4203            key_id: Some("alias/kaniop-backups".to_string()),
4204            key_ref: None,
4205        };
4206        let doc_str = super::build_download_operation_doc(
4207            &restore,
4208            &target,
4209            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4210            "019c7c76-f423-7a12-8f41-2bea7588a303",
4211            "real-bucket",
4212            "real-prefix",
4213            "https://real-endpoint.com",
4214            "eu-west-1",
4215            true,
4216            false,
4217            None,
4218            Some(&encryption),
4219        );
4220        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4221        assert_eq!(parsed["encryptionMode"], "providerKms");
4222        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4223    }
4224
4225    #[test]
4226    fn post_mutation_failure_records_job_names() {
4227        let existing = KanidmRestoreStatus {
4228            phase: KanidmRestorePhase::RestoringPrimary,
4229            database_mutation_started: true,
4230            ..Default::default()
4231        };
4232
4233        let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4234        assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4235        assert_eq!(
4236            restore_failed.restore_job_name.as_deref(),
4237            Some("my-restore-job")
4238        );
4239        assert!(restore_failed.verify_job_name.is_none());
4240        assert!(restore_failed.database_mutation_started);
4241        assert_eq!(
4242            restore_failed.message.as_deref(),
4243            Some("database restore job failed")
4244        );
4245
4246        let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4247        assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4248        assert_eq!(
4249            verify_failed.verify_job_name.as_deref(),
4250            Some("my-verify-job")
4251        );
4252        assert!(verify_failed.restore_job_name.is_none());
4253        assert!(verify_failed.database_mutation_started);
4254        assert_eq!(
4255            verify_failed.message.as_deref(),
4256            Some("database verification failed")
4257        );
4258    }
4259
4260    #[test]
4261    fn restore_and_verify_job_names_are_deterministic() {
4262        let source = KanidmRestoreSource {
4263            local: Some(super::KanidmRestoreLocalSource {
4264                file_name: "backup.json".to_string(),
4265            }),
4266            backup_ref: None,
4267        };
4268        let restore = make_restore(source, None);
4269        assert_eq!(restore_job_name(&restore), "test-restore-restore");
4270        assert_eq!(verify_job_name(&restore), "test-restore-verify");
4271    }
4272}