Skip to main content

kaniop_operator/kanidm/
restore.rs

1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7    AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8    build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9    ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14    OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27    Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28    PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29    SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const STAGING_VOLUME: &str = "restore-staging";
54const SHARED_VOLUME: &str = "safety-backup-shared";
55const DATA_PATH: &str = "/data";
56const TLS_VOLUME: &str = "kanidm-certs";
57const TLS_PATH: &str = "/etc/kanidm/tls";
58const BACKUP_PATH: &str = "/data";
59const STAGING_PATH: &str = "/staging";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78    feature = "schemars",
79    schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82    category = "kaniop",
83    group = "kaniop.rs",
84    version = "v1beta1",
85    kind = "KanidmRestore",
86    plural = "kanidmrestores",
87    singular = "kanidmrestore",
88    shortname = "idmrestore",
89    namespaced,
90    status = "KanidmRestoreStatus",
91    printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92    printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93    printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97    pub target_ref: KanidmRestoreTargetRef,
98    pub source: KanidmRestoreSource,
99    pub restore_image: String,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108    pub name: String,
109    pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115    feature = "schemars",
116    schemars(extend("x-kubernetes-validations" = [
117        {
118            "message": "exactly one of local or backupRef must be set",
119            "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120        }
121    ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125    /// Local file source. Mutually exclusive with `backupRef`.
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub local: Option<KanidmRestoreLocalSource>,
128    /// Remote cataloged backup reference. Mutually exclusive with `local`.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137    /// Basename of a backup below /data/backups. Paths and traversal are rejected.
138    pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145    /// Name of a KanidmBackup resource in the same namespace representing a committed remote backup.
146    pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153    /// Reference to the KanidmBackupRepository used for the pre-restore safety backup.
154    #[serde(skip_serializing_if = "Option::is_none")]
155    pub repository_ref: Option<SafetyBackupRepositoryRef>,
156    /// Skip the mandatory safety backup. Requires break-glass annotations.
157    #[serde(default)]
158    pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165    pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171    #[default]
172    Pending,
173    Validating,
174    Quiescing,
175    SafetyBackup,
176    PreparingSource,
177    RestoringPrimary,
178    Verifying,
179    RebuildingReplicas,
180    Resuming,
181    Completed,
182    Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189    #[serde(skip_serializing_if = "Option::is_none")]
190    pub observed_generation: Option<i64>,
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub observed_target_uid: Option<String>,
193    #[serde(default)]
194    pub phase: KanidmRestorePhase,
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub restore_job_name: Option<String>,
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub verify_job_name: Option<String>,
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub safety_backup_job_name: Option<String>,
201    #[serde(skip_serializing_if = "Option::is_none")]
202    pub source_prep_job_name: Option<String>,
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub safety_backup_ref: Option<String>,
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub safety_backup_expected_backup_id: Option<String>,
207    #[serde(skip_serializing_if = "Option::is_none")]
208    pub safety_backup_manifest_key: Option<String>,
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub safety_backup_payload_sha256: Option<String>,
211    #[serde(default)]
212    pub replicas_cleared: bool,
213    #[serde(default)]
214    pub certificates_cleared: bool,
215    #[serde(default)]
216    pub database_mutation_started: bool,
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub original_replicas: Vec<ReplicaCountEntry>,
219    #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220    pub phase_timestamps: std::collections::BTreeMap<String, String>,
221    #[serde(default, skip_serializing_if = "Vec::is_empty")]
222    #[cfg_attr(
223        feature = "schemars",
224        schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225    )]
226    pub conditions: Vec<Condition>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235    pub group: String,
236    pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241    attempts: Counter<u64>,
242    outcomes: Counter<u64>,
243    duration_seconds: Histogram<f64>,
244    break_glass_total: Counter<u64>,
245    safety_backup_duration_seconds: Histogram<f64>,
246}
247
248impl RestoreMetrics {
249    fn new() -> Self {
250        let meter = global::meter("kaniop");
251        Self {
252            attempts: meter
253                .u64_counter("kaniop_restore_attempts_total")
254                .with_description("Number of Kanidm restore attempts started")
255                .build(),
256            outcomes: meter
257                .u64_counter("kaniop_restore_outcomes_total")
258                .with_description("Number of terminal Kanidm restore outcomes")
259                .build(),
260            duration_seconds: meter
261                .f64_histogram("kaniop_restore_duration_seconds")
262                .with_description("Kanidm restore duration from object creation to terminal phase")
263                .with_unit("s")
264                .build(),
265            break_glass_total: meter
266                .u64_counter("kaniop_restore_break_glass_total")
267                .with_description("Number of break-glass safety backup overrides")
268                .build(),
269            safety_backup_duration_seconds: meter
270                .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
271                .with_description("Duration of pre-restore safety backup creation")
272                .with_unit("s")
273                .build(),
274        }
275    }
276}
277
278#[derive(Clone)]
279struct RestoreContext {
280    client: Client,
281    recorder: Recorder,
282    metrics: RestoreMetrics,
283}
284
285pub async fn run(client: Client) {
286    let api = Api::<KanidmRestore>::all(client.clone());
287    let recorder = Recorder::new(
288        client.clone(),
289        Reporter {
290            controller: CONTROLLER_ID.into(),
291            instance: None,
292        },
293    );
294    let ctx = Arc::new(RestoreContext {
295        client,
296        recorder,
297        metrics: RestoreMetrics::new(),
298    });
299    info!("starting {CONTROLLER_ID} controller");
300    Controller::new(api, watcher::Config::default().any_semantic())
301        .shutdown_on_signal()
302        .run(reconcile_restore, error_policy, ctx)
303        .for_each(|result| async move {
304            if let Err(error) = result {
305                error!(%error, "KanidmRestore reconciliation failed");
306            }
307        })
308        .await;
309}
310
311fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
312    warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
313    Action::requeue(Duration::from_secs(5))
314}
315
316async fn reconcile_restore(
317    restore: Arc<KanidmRestore>,
318    ctx: Arc<RestoreContext>,
319) -> Result<Action> {
320    let namespace = restore
321        .namespace()
322        .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
323    let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
324    finalizer(&api, RESTORE_FINALIZER, restore, |event| {
325        let ctx = ctx.clone();
326        async move {
327            match event {
328                Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
329                Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
330            }
331        }
332    })
333    .await
334    .map_err(|error| {
335        Error::FinalizerError(
336            "failed on KanidmRestore finalizer".to_string(),
337            Box::new(error),
338        )
339    })
340}
341
342async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
343    let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
344    match phase {
345        KanidmRestorePhase::Pending => {
346            set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
347            Ok(Action::requeue(REQUEUE))
348        }
349        KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
350            Ok(target) => {
351                mark_restoring(&restore, &target, &ctx).await?;
352                let mut status = restore.status.clone().unwrap_or_default();
353                status.observed_target_uid = target.uid();
354                status.original_replicas = capture_original_replicas(&target);
355                status.phase = KanidmRestorePhase::Quiescing;
356                status.message = None;
357                patch_status(&restore, &ctx, status).await?;
358                Ok(Action::requeue(REQUEUE))
359            }
360            Err(error) => {
361                set_phase(
362                    &restore,
363                    &ctx,
364                    KanidmRestorePhase::Failed,
365                    Some(error.to_string()),
366                )
367                .await?;
368                Ok(Action::requeue(Duration::from_secs(300)))
369            }
370        },
371        KanidmRestorePhase::Quiescing => {
372            let target = get_target(&restore, &ctx).await?;
373            scale_all(&target, &ctx, 0).await?;
374            if target_pods_stopped(&target, &ctx).await?
375                && target_volumes_detached(&target, &ctx).await?
376            {
377                let next_phase = if requires_safety_backup(&restore) {
378                    KanidmRestorePhase::SafetyBackup
379                } else {
380                    record_break_glass(&restore, &ctx).await?;
381                    KanidmRestorePhase::PreparingSource
382                };
383                if next_phase == KanidmRestorePhase::PreparingSource {
384                    let refreshed = Api::<KanidmRestore>::namespaced(
385                        ctx.client.clone(),
386                        restore.namespace().as_deref().unwrap_or_default(),
387                    )
388                    .get(&restore.name_any())
389                    .await
390                    .map_err(|error| {
391                        Error::kube_error(
392                            "get",
393                            "KanidmRestore",
394                            restore.namespace().as_deref().unwrap_or_default(),
395                            restore.name_any(),
396                            error,
397                        )
398                    })?;
399                    set_phase(&refreshed, &ctx, next_phase, None).await?;
400                } else {
401                    set_phase(&restore, &ctx, next_phase, None).await?;
402                }
403            }
404            Ok(Action::requeue(REQUEUE))
405        }
406        KanidmRestorePhase::SafetyBackup => {
407            let target = get_target(&restore, &ctx).await?;
408            let name = safety_backup_job_name(&restore);
409            let expected_backup_id = compute_safety_backup_id(&restore);
410            let expected_manifest_key = compute_safety_manifest_key(&restore);
411            ensure_safety_backup_job(
412                &restore,
413                &target,
414                &ctx,
415                &name,
416                &expected_backup_id,
417                &expected_manifest_key,
418            )
419            .await?;
420            let mut status = restore.status.clone().unwrap_or_default();
421            status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
422            match job_state(&restore, &ctx, &name).await? {
423                JobState::Complete => {
424                    let result =
425                        read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
426                    match result {
427                        Ok(verified) => {
428                            status.safety_backup_job_name = Some(name);
429                            status.safety_backup_ref =
430                                Some(format!("safety-{}", restore.name_any()));
431                            status.safety_backup_manifest_key = Some(verified.manifest_key);
432                            status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
433                            record_safety_backup_duration(&restore, &ctx).await;
434                            status.phase = KanidmRestorePhase::PreparingSource;
435                            status.message = None;
436                            patch_status(&restore, &ctx, status).await?;
437                        }
438                        Err(error) => {
439                            resume_before_mutation(&restore, &ctx).await?;
440                            set_phase(
441                                &restore,
442                                &ctx,
443                                KanidmRestorePhase::Failed,
444                                Some(format!("safety backup result verification failed: {error}")),
445                            )
446                            .await?;
447                        }
448                    }
449                }
450                JobState::Failed => {
451                    resume_before_mutation(&restore, &ctx).await?;
452                    set_phase(
453                        &restore,
454                        &ctx,
455                        KanidmRestorePhase::Failed,
456                        Some(
457                            "safety backup job failed; target restored to original state"
458                                .to_string(),
459                        ),
460                    )
461                    .await?;
462                }
463                JobState::Running => {
464                    patch_status(&restore, &ctx, status).await?;
465                }
466            }
467            Ok(Action::requeue(REQUEUE))
468        }
469        KanidmRestorePhase::PreparingSource => {
470            if is_remote_source(&restore) {
471                let target = get_target(&restore, &ctx).await?;
472                let name = source_prep_job_name(&restore);
473                ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
474                match job_state(&restore, &ctx, &name).await? {
475                    JobState::Complete => {
476                        let backup_id = restore
477                            .spec
478                            .source
479                            .backup_ref
480                            .as_ref()
481                            .map(|b| b.name.clone())
482                            .unwrap_or_default();
483                        let backup = {
484                            let ns = restore.namespace().unwrap();
485                            Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
486                                .get(&backup_id)
487                                .await
488                                .map_err(|e| {
489                                    Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
490                                })?
491                        };
492                        let expected_backup_id = &backup.spec.backup_id;
493                        let result =
494                            read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
495                                .await;
496                        match result {
497                            Ok(_verified) => {
498                                let mut status = restore.status.clone().unwrap_or_default();
499                                status.source_prep_job_name = Some(name);
500                                status.phase = KanidmRestorePhase::RestoringPrimary;
501                                status.message = None;
502                                patch_status(&restore, &ctx, status).await?;
503                            }
504                            Err(error) => {
505                                resume_before_mutation(&restore, &ctx).await?;
506                                set_phase(
507                                    &restore,
508                                    &ctx,
509                                    KanidmRestorePhase::Failed,
510                                    Some(format!(
511                                        "source preparation result verification failed: {error}"
512                                    )),
513                                )
514                                .await?;
515                            }
516                        }
517                    }
518                    JobState::Failed => {
519                        resume_before_mutation(&restore, &ctx).await?;
520                        set_phase(
521                            &restore,
522                            &ctx,
523                            KanidmRestorePhase::Failed,
524                            Some(
525                                "source preparation job failed; target restored to original state"
526                                    .to_string(),
527                            ),
528                        )
529                        .await?;
530                    }
531                    JobState::Running => {}
532                }
533            } else {
534                let target = get_target(&restore, &ctx).await?;
535                let name = source_check_job_name(&restore);
536                ensure_source_check_job(&restore, &target, &ctx, &name).await?;
537                match job_state(&restore, &ctx, &name).await? {
538                    JobState::Complete => {
539                        let mut status = restore.status.clone().unwrap_or_default();
540                        status.source_prep_job_name = Some(name);
541                        status.phase = KanidmRestorePhase::RestoringPrimary;
542                        status.message = None;
543                        patch_status(&restore, &ctx, status).await?;
544                    }
545                    JobState::Failed => {
546                        resume_before_mutation(&restore, &ctx).await?;
547                        set_phase(
548                            &restore,
549                            &ctx,
550                            KanidmRestorePhase::Failed,
551                            Some(
552                                "local backup file check failed; target restored to original state"
553                                    .to_string(),
554                            ),
555                        )
556                        .await?;
557                    }
558                    JobState::Running => {}
559                }
560            }
561            Ok(Action::requeue(REQUEUE))
562        }
563        KanidmRestorePhase::RestoringPrimary => {
564            let target = get_target(&restore, &ctx).await?;
565            let status = restore.status.as_ref().cloned().unwrap_or_default();
566            if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
567                return Err(Error::MissingData(
568                    "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
569                ));
570            }
571            if !status.database_mutation_started {
572                let mut status = status;
573                status.database_mutation_started = true;
574                status.message = Some(
575                    "database mutation boundary persisted; starting primary restore".to_string(),
576                );
577                patch_status(&restore, &ctx, status).await?;
578                return Ok(Action::requeue(REQUEUE));
579            }
580            let name = restore_job_name(&restore);
581            ensure_database_job(&restore, &target, &ctx, &name, false).await?;
582            match job_state(&restore, &ctx, &name).await? {
583                JobState::Complete => {
584                    let mut status = restore.status.clone().unwrap_or_default();
585                    status.restore_job_name = Some(name);
586                    status.phase = KanidmRestorePhase::Verifying;
587                    status.message = None;
588                    patch_status(&restore, &ctx, status).await?;
589                }
590                JobState::Failed => {
591                    fail_after_mutation(&restore, &ctx, "database restore job failed").await?
592                }
593                JobState::Running => {}
594            }
595            Ok(Action::requeue(REQUEUE))
596        }
597        KanidmRestorePhase::Verifying => {
598            let target = get_target(&restore, &ctx).await?;
599            let name = verify_job_name(&restore);
600            ensure_database_job(&restore, &target, &ctx, &name, true).await?;
601            match job_state(&restore, &ctx, &name).await? {
602                JobState::Complete => {
603                    let mut status = restore.status.clone().unwrap_or_default();
604                    status.verify_job_name = Some(name);
605                    status.phase = KanidmRestorePhase::RebuildingReplicas;
606                    status.message = None;
607                    patch_status(&restore, &ctx, status).await?;
608                }
609                JobState::Failed => {
610                    fail_after_mutation(&restore, &ctx, "database verification failed").await?
611                }
612                JobState::Running => {}
613            }
614            Ok(Action::requeue(REQUEUE))
615        }
616        KanidmRestorePhase::RebuildingReplicas => {
617            let target = get_target(&restore, &ctx).await?;
618            let mut status = restore.status.clone().unwrap_or_default();
619            if !status.replicas_cleared {
620                if delete_secondary_pvcs(&target, &ctx).await? {
621                    status.replicas_cleared = true;
622                    status.message = Some("secondary database state cleared".to_string());
623                    patch_status(&restore, &ctx, status).await?;
624                }
625                return Ok(Action::requeue(REQUEUE));
626            }
627
628            scale_primary(&target, &ctx, 1).await?;
629            if !primary_ready(&target, &ctx).await? {
630                return Ok(Action::requeue(REQUEUE));
631            }
632            if !status.certificates_cleared {
633                delete_replica_cert_secrets(&target, &ctx).await?;
634                delete_admin_secret(&target, &ctx).await?;
635                status.certificates_cleared = true;
636                status.message = Some(
637                    "replica certificates and admin secret cleared for regeneration".to_string(),
638                );
639                patch_status(&restore, &ctx, status).await?;
640                return Ok(Action::requeue(REQUEUE));
641            }
642            status.phase = KanidmRestorePhase::Resuming;
643            status.message = None;
644            patch_status(&restore, &ctx, status).await?;
645            Ok(Action::requeue(REQUEUE))
646        }
647        KanidmRestorePhase::Resuming => {
648            let target = get_target(&restore, &ctx).await?;
649            set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
650            clear_restoring(&restore, &target, &ctx).await?;
651            Ok(Action::requeue(Duration::from_secs(3600)))
652        }
653        KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
654            Ok(Action::requeue(Duration::from_secs(3600)))
655        }
656    }
657}
658
659async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
660    let status = restore.status.clone().unwrap_or_default();
661    if status.database_mutation_started
662        && status.phase != KanidmRestorePhase::Completed
663        && status.phase != KanidmRestorePhase::Failed
664    {
665        reconcile_apply(restore.clone(), ctx.clone()).await?;
666        return Ok(Action::requeue(REQUEUE));
667    }
668    if let Ok(target) = get_target(&restore, &ctx).await {
669        let owns_maintenance =
670            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
671        if owns_maintenance && !status.database_mutation_started {
672            scale_desired(&target, &ctx).await?;
673        }
674        clear_restoring(&restore, &target, &ctx).await?;
675    }
676    Ok(Action::await_change())
677}
678
679async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
680    let target = get_target(restore, ctx).await?;
681    let actual_uid = target
682        .uid()
683        .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
684    if actual_uid != restore.spec.target_ref.uid {
685        return Err(Error::MissingData(format!(
686            "target UID mismatch: expected {}, got {}",
687            restore.spec.target_ref.uid, actual_uid
688        )));
689    }
690    validate_source(restore)?;
691    if is_remote_source(restore) {
692        validate_backup_ref(restore, &target, ctx).await?;
693    }
694    if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
695    {
696        return Err(Error::MissingData(format!(
697            "restoreImage must be the target's pinned Kanidm image (target image is {})",
698            target.spec.image
699        )));
700    }
701    let storage = target
702        .spec
703        .storage
704        .as_ref()
705        .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
706    if storage.empty_dir.is_some()
707        || storage.ephemeral.is_some()
708        || storage.volume_claim_template.is_none()
709    {
710        return Err(Error::MissingData(
711            "restore requires PVC-backed Kanidm storage".to_string(),
712        ));
713    }
714    let primaries = target
715        .spec
716        .replica_groups
717        .iter()
718        .filter(|rg| rg.primary_node)
719        .count();
720    if primaries != 1 {
721        return Err(Error::MissingData(
722            "backup/restore requires exactly one primary replica group".to_string(),
723        ));
724    }
725    let ns = restore.namespace().unwrap();
726    let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
727        .list(&ListParams::default())
728        .await
729        .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
730    if restores.items.iter().any(|other| {
731        other.name_any() != restore.name_any()
732            && other.spec.target_ref.name == restore.spec.target_ref.name
733            && !matches!(
734                other.status.as_ref().map(|s| s.phase),
735                Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
736            )
737    }) {
738        return Err(Error::MissingData(
739            "another active restore targets this Kanidm".to_string(),
740        ));
741    }
742    validate_safety_backup_config(restore)?;
743    Ok(target)
744}
745
746fn safe_basename(name: &str) -> bool {
747    !name.is_empty()
748        && name != "."
749        && name != ".."
750        && !name.contains('/')
751        && !name.contains('\\')
752        && !name.contains('\0')
753}
754
755fn mutable_image(image: &str) -> bool {
756    image == "kanidm/server:latest"
757        || image.ends_with(":latest")
758        || (!image.contains('@')
759            && !image
760                .rsplit('/')
761                .next()
762                .is_some_and(|part| part.contains(':')))
763}
764
765async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
766    let ns = restore
767        .namespace()
768        .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
769    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
770        .get(&restore.spec.target_ref.name)
771        .await
772        .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
773}
774
775async fn mark_restoring(
776    restore: &KanidmRestore,
777    target: &Kanidm,
778    ctx: &RestoreContext,
779) -> Result<()> {
780    let ns = target.namespace().unwrap();
781    let uid = restore
782        .uid()
783        .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
784    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
785        .patch(
786            &target.name_any(),
787            &PatchParams::default(),
788            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
789        )
790        .await
791        .map(|_| ())
792        .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
793}
794
795async fn clear_restoring(
796    restore: &KanidmRestore,
797    target: &Kanidm,
798    ctx: &RestoreContext,
799) -> Result<()> {
800    let ns = target.namespace().unwrap();
801    if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
802        return Ok(());
803    }
804    Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
805        .patch(
806            &target.name_any(),
807            &PatchParams::default(),
808            &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
809        )
810        .await
811        .map(|_| ())
812        .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
813}
814
815async fn patch_status(
816    restore: &KanidmRestore,
817    ctx: &RestoreContext,
818    mut status: KanidmRestoreStatus,
819) -> Result<()> {
820    let ns = restore.namespace().unwrap();
821    let previous_phase = restore.status.as_ref().map(|current| current.phase);
822    let next_phase = status.phase;
823    status.observed_generation = restore.metadata.generation;
824    update_restore_conditions(&mut status, restore.metadata.generation);
825
826    Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
827        .patch_status(
828            &restore.name_any(),
829            &PatchParams::default(),
830            &Patch::Merge(json!({"status": &status})),
831        )
832        .await
833        .map_err(|e| {
834            Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
835        })?;
836
837    if previous_phase != Some(next_phase) {
838        record_restore_transition(
839            restore,
840            ctx,
841            previous_phase,
842            next_phase,
843            status.message.as_deref(),
844        )
845        .await;
846    }
847    Ok(())
848}
849
850fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
851    let previous = status.conditions.clone();
852    let phase = status.phase;
853    let phase_reason = format!("{phase:?}");
854    let message = status
855        .message
856        .clone()
857        .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
858    let terminal_success = phase == KanidmRestorePhase::Completed;
859    let terminal_failure = phase == KanidmRestorePhase::Failed;
860    let progressing = !terminal_success && !terminal_failure;
861
862    let break_glass = previous
863        .iter()
864        .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
865        .cloned();
866    status.conditions = vec![
867        restore_condition(
868            &previous,
869            CONDITION_PROGRESSING,
870            if progressing {
871                CONDITION_TRUE
872            } else {
873                CONDITION_FALSE
874            },
875            &phase_reason,
876            &message,
877            generation,
878        ),
879        restore_condition(
880            &previous,
881            CONDITION_READY,
882            if terminal_success {
883                CONDITION_TRUE
884            } else {
885                CONDITION_FALSE
886            },
887            if terminal_success {
888                "RestoreCompleted"
889            } else {
890                "RestoreNotCompleted"
891            },
892            if terminal_success {
893                "Kanidm restore completed successfully."
894            } else {
895                "Kanidm restore has not completed successfully."
896            },
897            generation,
898        ),
899        restore_condition(
900            &previous,
901            CONDITION_FAILED,
902            if terminal_failure {
903                CONDITION_TRUE
904            } else {
905                CONDITION_FALSE
906            },
907            if terminal_failure {
908                "RestoreFailed"
909            } else {
910                "NoRestoreFailure"
911            },
912            if terminal_failure {
913                status
914                    .message
915                    .as_deref()
916                    .unwrap_or("Kanidm restore failed.")
917            } else {
918                "No terminal restore failure has been recorded."
919            },
920            generation,
921        ),
922    ];
923    if let Some(condition) = break_glass {
924        status.conditions.push(condition);
925    }
926}
927
928fn restore_condition(
929    previous: &[Condition],
930    condition_type: &str,
931    condition_status: &str,
932    reason: &str,
933    message: &str,
934    generation: Option<i64>,
935) -> Condition {
936    let last_transition_time = previous
937        .iter()
938        .find(|condition| {
939            condition.type_ == condition_type
940                && condition.status == condition_status
941                && condition.reason == reason
942        })
943        .map(|condition| condition.last_transition_time.clone())
944        .unwrap_or_else(|| Time(Timestamp::now()));
945    Condition {
946        type_: condition_type.to_string(),
947        status: condition_status.to_string(),
948        reason: reason.to_string(),
949        message: message.to_string(),
950        last_transition_time,
951        observed_generation: generation,
952    }
953}
954
955async fn record_restore_transition(
956    restore: &KanidmRestore,
957    ctx: &RestoreContext,
958    previous_phase: Option<KanidmRestorePhase>,
959    phase: KanidmRestorePhase,
960    message: Option<&str>,
961) {
962    if phase == KanidmRestorePhase::Validating {
963        ctx.metrics.attempts.add(1, &[]);
964    }
965
966    let result = match phase {
967        KanidmRestorePhase::Completed => Some("success"),
968        KanidmRestorePhase::Failed => Some("failure"),
969        _ => None,
970    };
971    if let Some(result) = result {
972        let attributes = [KeyValue::new("result", result)];
973        ctx.metrics.outcomes.add(1, &attributes);
974        if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
975            let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
976            ctx.metrics.duration_seconds.record(elapsed, &attributes);
977        }
978    }
979
980    let reason = if phase == KanidmRestorePhase::Failed {
981        "RestoreFailed"
982    } else {
983        "RestorePhaseChanged"
984    };
985    let note = message.map(str::to_string).or_else(|| {
986        Some(format!(
987            "Kanidm restore phase changed from {} to {phase:?}.",
988            previous_phase
989                .map(|previous| format!("{previous:?}"))
990                .unwrap_or_else(|| "None".to_string())
991        ))
992    });
993    if let Err(error) = ctx
994        .recorder
995        .publish(
996            &Event {
997                type_: if phase == KanidmRestorePhase::Failed {
998                    EventType::Warning
999                } else {
1000                    EventType::Normal
1001                },
1002                reason: reason.to_string(),
1003                note,
1004                action: "Restore".to_string(),
1005                secondary: None,
1006            },
1007            &restore.object_ref(&()),
1008        )
1009        .await
1010    {
1011        warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1012    }
1013}
1014
1015async fn set_phase(
1016    restore: &KanidmRestore,
1017    ctx: &RestoreContext,
1018    phase: KanidmRestorePhase,
1019    message: Option<String>,
1020) -> Result<()> {
1021    let mut status = restore.status.clone().unwrap_or_default();
1022    status.phase = phase;
1023    status.message = message;
1024    patch_status(restore, ctx, status).await
1025}
1026
1027async fn fail_after_mutation(
1028    restore: &KanidmRestore,
1029    ctx: &RestoreContext,
1030    message: &str,
1031) -> Result<()> {
1032    set_phase(
1033        restore,
1034        ctx,
1035        KanidmRestorePhase::Failed,
1036        Some(message.to_string()),
1037    )
1038    .await
1039}
1040
1041fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1042    target
1043        .spec
1044        .replica_groups
1045        .iter()
1046        .find(|rg| rg.primary_node)
1047        .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1048}
1049
1050async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1051    let ns = target.namespace().unwrap();
1052    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1053    for rg in &target.spec.replica_groups {
1054        let name = target.statefulset_name(&rg.name);
1055        api.patch(
1056            &name,
1057            &PatchParams::default(),
1058            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1059        )
1060        .await
1061        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1062    }
1063    Ok(())
1064}
1065
1066async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1067    let ns = target.namespace().unwrap();
1068    let rg = primary_group(target)?;
1069    let name = target.statefulset_name(&rg.name);
1070    Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1071        .patch(
1072            &name,
1073            &PatchParams::default(),
1074            &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1075        )
1076        .await
1077        .map(|_| ())
1078        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1079}
1080
1081async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1082    let ns = target.namespace().unwrap();
1083    let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1084    for rg in &target.spec.replica_groups {
1085        let name = target.statefulset_name(&rg.name);
1086        api.patch(
1087            &name,
1088            &PatchParams::default(),
1089            &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1090        )
1091        .await
1092        .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1093    }
1094    Ok(())
1095}
1096
1097async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098    let ns = target.namespace().unwrap();
1099    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1100        .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1101        .await
1102        .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1103    Ok(pods.items.is_empty())
1104}
1105
1106async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1107    let ns = target.namespace().unwrap();
1108    let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1109    let mut pv_names = BTreeSet::new();
1110    let primary_name = primary_pvc_name(target)?;
1111    let primary_pvc = pvc_api
1112        .get(&primary_name)
1113        .await
1114        .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1115    let primary_pv = primary_pvc
1116        .spec
1117        .as_ref()
1118        .and_then(|spec| spec.volume_name.clone())
1119        .ok_or_else(|| {
1120            Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1121        })?;
1122    pv_names.insert(primary_pv);
1123
1124    for rg in &target.spec.replica_groups {
1125        let sts = target.statefulset_name(&rg.name);
1126        for ordinal in 0..rg.replicas {
1127            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1128            if name == primary_name {
1129                continue;
1130            }
1131            if let Some(pvc) = pvc_api
1132                .get_opt(&name)
1133                .await
1134                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135                && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1136            {
1137                pv_names.insert(pv_name);
1138            }
1139        }
1140    }
1141
1142    let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1143        .list(&ListParams::default())
1144        .await
1145        .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1146    Ok(!attachments.items.iter().any(|attachment| {
1147        attachment
1148            .spec
1149            .source
1150            .persistent_volume_name
1151            .as_ref()
1152            .is_some_and(|pv_name| pv_names.contains(pv_name))
1153    }))
1154}
1155
1156async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1157    let ns = target.namespace().unwrap();
1158    let name = target.statefulset_name(&primary_group(target)?.name);
1159    let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1160        .get(&name)
1161        .await
1162        .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1163    Ok(sts
1164        .status
1165        .as_ref()
1166        .and_then(|s| s.ready_replicas)
1167        .unwrap_or(0)
1168        >= 1)
1169}
1170
1171fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1172    let rg = primary_group(target)?;
1173    Ok(format!(
1174        "{DATA_VOLUME}-{}-0",
1175        target.statefulset_name(&rg.name)
1176    ))
1177}
1178
1179async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1180    let ns = target.namespace().unwrap();
1181    let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1182    let mut all_absent = true;
1183    for rg in &target.spec.replica_groups {
1184        let sts = target.statefulset_name(&rg.name);
1185        for ordinal in 0..rg.replicas {
1186            if rg.primary_node && ordinal == 0 {
1187                continue;
1188            }
1189            let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1190            match api
1191                .get_opt(&name)
1192                .await
1193                .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1194            {
1195                None => {}
1196                Some(pvc) => {
1197                    all_absent = false;
1198                    if pvc.metadata.deletion_timestamp.is_none() {
1199                        match api.delete(&name, &DeleteParams::default()).await {
1200                            Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1201                            Err(kube::Error::Api(status)) if status.code == 404 => {}
1202                            Err(error) => {
1203                                return Err(Error::kube_error(
1204                                    "delete",
1205                                    "PersistentVolumeClaim",
1206                                    &ns,
1207                                    &name,
1208                                    error,
1209                                ));
1210                            }
1211                        }
1212                    }
1213                }
1214            }
1215        }
1216    }
1217    Ok(all_absent)
1218}
1219
1220async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1221    let ns = target.namespace().unwrap();
1222    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1223    let label_selector = format!(
1224        "{}={},{}={}",
1225        CLUSTER_LABEL,
1226        target.name_any(),
1227        SECRET_TYPE_LABEL,
1228        serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1229    );
1230    let secrets = api
1231        .list(&ListParams::default().labels(&label_selector))
1232        .await
1233        .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1234    let mut all_absent = secrets.items.is_empty();
1235    for secret in secrets.items {
1236        let name = secret.name_any();
1237        if secret.metadata.deletion_timestamp.is_none() {
1238            match api.delete(&name, &DeleteParams::default()).await {
1239                Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1240                Err(kube::Error::Api(status)) if status.code == 404 => {}
1241                Err(error) => {
1242                    return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1243                }
1244            }
1245            all_absent = false;
1246        }
1247    }
1248    Ok(all_absent)
1249}
1250
1251async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1252    let ns = target.namespace().unwrap();
1253    let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1254    let name = format!("{}-admin-passwords", target.name_any());
1255    match api
1256        .get_opt(&name)
1257        .await
1258        .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1259    {
1260        None => Ok(true),
1261        Some(secret) => {
1262            if secret.metadata.deletion_timestamp.is_some() {
1263                return Ok(false);
1264            }
1265            match api.delete(&name, &DeleteParams::default()).await {
1266                Ok(_) => {
1267                    debug!(secret = %name, "deleting stale admin secret");
1268                    Ok(false)
1269                }
1270                Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1271                Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1272            }
1273        }
1274    }
1275}
1276
1277fn restore_job_name(restore: &KanidmRestore) -> String {
1278    format!("{}-restore", restore.name_any())
1279}
1280fn verify_job_name(restore: &KanidmRestore) -> String {
1281    format!("{}-verify", restore.name_any())
1282}
1283
1284fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1285    let origin = target
1286        .spec
1287        .origin
1288        .clone()
1289        .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1290    vec![
1291        EnvVar {
1292            name: "KANIDM_DB_PATH".to_string(),
1293            value: Some("/data/kanidm.db".to_string()),
1294            ..Default::default()
1295        },
1296        EnvVar {
1297            name: "KANIDM_DOMAIN".to_string(),
1298            value: Some(target.spec.domain.clone()),
1299            ..Default::default()
1300        },
1301        EnvVar {
1302            name: "KANIDM_ORIGIN".to_string(),
1303            value: Some(origin),
1304            ..Default::default()
1305        },
1306        EnvVar {
1307            name: "KANIDM_TLS_CHAIN".to_string(),
1308            value: Some(format!("{TLS_PATH}/tls.crt")),
1309            ..Default::default()
1310        },
1311        EnvVar {
1312            name: "KANIDM_TLS_KEY".to_string(),
1313            value: Some(format!("{TLS_PATH}/tls.key")),
1314            ..Default::default()
1315        },
1316    ]
1317}
1318
1319async fn ensure_database_job(
1320    restore: &KanidmRestore,
1321    target: &Kanidm,
1322    ctx: &RestoreContext,
1323    name: &str,
1324    verify: bool,
1325) -> Result<()> {
1326    let ns = restore.namespace().unwrap();
1327    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1328    if api
1329        .get_opt(name)
1330        .await
1331        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1332        .is_some()
1333    {
1334        return Ok(());
1335    }
1336    let mut command = vec![
1337        "kanidmd".to_string(),
1338        "database".to_string(),
1339        if verify { "backup" } else { "restore" }.to_string(),
1340    ];
1341    if verify {
1342        command.push("/verify/verification.json.gz".to_string());
1343    } else {
1344        if is_remote_source(restore) {
1345            command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1346        } else {
1347            let file_name = restore
1348                .spec
1349                .source
1350                .local
1351                .as_ref()
1352                .map(|l| l.file_name.clone())
1353                .unwrap_or_default();
1354            command.push(format!("{BACKUP_PATH}/{file_name}"));
1355        }
1356    }
1357    let job = Job {
1358        metadata: ObjectMeta {
1359            name: Some(name.to_string()),
1360            namespace: Some(ns.clone()),
1361            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1362            ..Default::default()
1363        },
1364        spec: Some(JobSpec {
1365            backoff_limit: Some(0),
1366            template: PodTemplateSpec {
1367                metadata: Some(ObjectMeta::default()),
1368                spec: Some(PodSpec {
1369                    automount_service_account_token: Some(false),
1370                    restart_policy: Some("Never".to_string()),
1371                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1372                        seccomp_profile: Some(SeccompProfile {
1373                            type_: "RuntimeDefault".to_string(),
1374                            ..Default::default()
1375                        }),
1376                        ..Default::default()
1377                    }),
1378                    containers: vec![Container {
1379                        name: if verify { "verify" } else { "restore" }.to_string(),
1380                        image: Some(restore.spec.restore_image.clone()),
1381                        command: Some(command),
1382                        env: Some(build_kanidm_db_env_vars(target)),
1383                        security_context: Some(kanidm_job_security_context()),
1384                        resources: Some(job_resource_requirements()),
1385                        volume_mounts: Some(vec![
1386                            VolumeMount {
1387                                name: DATA_VOLUME.to_string(),
1388                                mount_path: DATA_PATH.to_string(),
1389                                ..Default::default()
1390                            },
1391                            VolumeMount {
1392                                name: TLS_VOLUME.to_string(),
1393                                mount_path: TLS_PATH.to_string(),
1394                                read_only: Some(true),
1395                                ..Default::default()
1396                            },
1397                            VolumeMount {
1398                                name: "verification-output".to_string(),
1399                                mount_path: "/verify".to_string(),
1400                                ..Default::default()
1401                            },
1402                        ]),
1403                        ..Default::default()
1404                    }],
1405                    volumes: Some(vec![
1406                        Volume {
1407                            name: DATA_VOLUME.to_string(),
1408                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1409                                claim_name: primary_pvc_name(target)?,
1410                                read_only: Some(false),
1411                            }),
1412                            ..Default::default()
1413                        },
1414                        Volume {
1415                            name: TLS_VOLUME.to_string(),
1416                            secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1417                                secret_name: Some(target.effective_tls_secret_name()),
1418                                ..Default::default()
1419                            }),
1420                            ..Default::default()
1421                        },
1422                        Volume {
1423                            name: "verification-output".to_string(),
1424                            empty_dir: Some(Default::default()),
1425                            ..Default::default()
1426                        },
1427                    ]),
1428                    ..Default::default()
1429                }),
1430            },
1431            ..Default::default()
1432        }),
1433        ..Default::default()
1434    };
1435    api.create(&PostParams::default(), &job)
1436        .await
1437        .map(|_| ())
1438        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1439}
1440
1441enum JobState {
1442    Running,
1443    Complete,
1444    Failed,
1445}
1446
1447async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1448    let ns = restore.namespace().unwrap();
1449    let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1450        .get(name)
1451        .await
1452        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1453    if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1454        return Ok(JobState::Failed);
1455    }
1456    if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1457        return Ok(JobState::Complete);
1458    }
1459    Ok(JobState::Running)
1460}
1461
1462fn is_remote_source(restore: &KanidmRestore) -> bool {
1463    restore.spec.source.backup_ref.is_some()
1464}
1465
1466fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1467    let safety = restore.spec.safety_backup.as_ref();
1468    let skip = safety.is_some_and(|s| s.skip);
1469    !skip
1470}
1471
1472fn validate_source(restore: &KanidmRestore) -> Result<()> {
1473    let has_local = restore.spec.source.local.is_some();
1474    let has_backup_ref = restore.spec.source.backup_ref.is_some();
1475    if has_local && has_backup_ref {
1476        return Err(Error::MissingData(
1477            "source.local and source.backupRef are mutually exclusive".to_string(),
1478        ));
1479    }
1480    if !has_local && !has_backup_ref {
1481        return Err(Error::MissingData(
1482            "source must specify either local or backupRef".to_string(),
1483        ));
1484    }
1485    if let Some(local) = &restore.spec.source.local {
1486        if !safe_basename(&local.file_name) {
1487            return Err(Error::MissingData(
1488                "restore source fileName must be a safe basename".to_string(),
1489            ));
1490        }
1491    }
1492    if let Some(backup_ref) = &restore.spec.source.backup_ref {
1493        if backup_ref.name.is_empty() {
1494            return Err(Error::MissingData(
1495                "source.backupRef.name must not be empty".to_string(),
1496            ));
1497        }
1498    }
1499    Ok(())
1500}
1501
1502fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1503    let safety = restore.spec.safety_backup.as_ref();
1504    let skip = safety.is_some_and(|s| s.skip);
1505    if skip {
1506        let annotations = restore.metadata.annotations.as_ref();
1507        let reason = annotations
1508            .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1509            .map(|s| s.trim())
1510            .unwrap_or("");
1511        let approver = annotations
1512            .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1513            .map(|s| s.trim())
1514            .unwrap_or("");
1515        if reason.is_empty() {
1516            return Err(Error::MissingData(format!(
1517                "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1518            )));
1519        }
1520        if approver.is_empty() {
1521            return Err(Error::MissingData(format!(
1522                "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1523            )));
1524        }
1525    }
1526    if is_remote_source(restore) && !skip {
1527        let has_repo = safety
1528            .and_then(|s| s.repository_ref.as_ref())
1529            .is_some_and(|r| !r.name.is_empty());
1530        if !has_repo {
1531            return Err(Error::MissingData(
1532                "remote restore requires safetyBackup.repositoryRef".to_string(),
1533            ));
1534        }
1535    }
1536    Ok(())
1537}
1538
1539fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1540    target
1541        .spec
1542        .replica_groups
1543        .iter()
1544        .map(|rg| ReplicaCountEntry {
1545            group: rg.name.clone(),
1546            replicas: rg.replicas,
1547        })
1548        .collect()
1549}
1550
1551async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1552    if let Ok(target) = get_target(restore, ctx).await {
1553        let owns_maintenance =
1554            target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1555        if owns_maintenance {
1556            scale_desired(&target, ctx).await?;
1557            clear_restoring(restore, &target, ctx).await?;
1558        }
1559    }
1560    Ok(())
1561}
1562
1563async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564    ctx.metrics.break_glass_total.add(1, &[]);
1565    let reason = restore
1566        .metadata
1567        .annotations
1568        .as_ref()
1569        .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1570        .cloned()
1571        .unwrap_or_default();
1572    let approver = restore
1573        .metadata
1574        .annotations
1575        .as_ref()
1576        .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1577        .cloned()
1578        .unwrap_or_default();
1579    let mut status = restore.status.clone().unwrap_or_default();
1580    let break_glass_condition = restore_condition(
1581        &status.conditions,
1582        CONDITION_BREAK_GLASS,
1583        CONDITION_TRUE,
1584        "BreakGlassOverride",
1585        &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1586        restore.metadata.generation,
1587    );
1588    status
1589        .conditions
1590        .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1591    status.conditions.push(break_glass_condition);
1592    patch_status(restore, ctx, status).await?;
1593    if let Err(error) = ctx
1594        .recorder
1595        .publish(
1596            &Event {
1597                type_: EventType::Warning,
1598                reason: "SafetyBackupSkipped".to_string(),
1599                note: Some(format!(
1600                    "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1601                )),
1602                action: "Restore".to_string(),
1603                secondary: None,
1604            },
1605            &restore.object_ref(&()),
1606        )
1607        .await
1608    {
1609        warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1610    }
1611    warn!(
1612        restore = %restore.name_any(),
1613        reason = %reason,
1614        approver = %approver,
1615        "break-glass safety backup override activated"
1616    );
1617    Ok(())
1618}
1619
1620async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1621    let status = restore.status.as_ref().cloned().unwrap_or_default();
1622    if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1623        if let Ok(start_ts) = start.parse::<Timestamp>() {
1624            let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1625            ctx.metrics
1626                .safety_backup_duration_seconds
1627                .record(elapsed, &[]);
1628        }
1629    }
1630}
1631
1632fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1633    format!("{}-safety-backup", restore.name_any())
1634}
1635
1636fn source_prep_job_name(restore: &KanidmRestore) -> String {
1637    format!("{}-source-prep", restore.name_any())
1638}
1639
1640fn source_check_job_name(restore: &KanidmRestore) -> String {
1641    format!("{}-source-check", restore.name_any())
1642}
1643
1644async fn ensure_source_check_job(
1645    restore: &KanidmRestore,
1646    target: &Kanidm,
1647    ctx: &RestoreContext,
1648    name: &str,
1649) -> Result<()> {
1650    let ns = restore.namespace().unwrap();
1651    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1652    if api
1653        .get_opt(name)
1654        .await
1655        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1656        .is_some()
1657    {
1658        return Ok(());
1659    }
1660    let file_name = restore
1661        .spec
1662        .source
1663        .local
1664        .as_ref()
1665        .map(|l| l.file_name.clone())
1666        .unwrap_or_default();
1667    let operation_doc = serde_json::json!({
1668        "apiVersion": "backup.kaniop.rs/v1alpha1",
1669        "kind": "OperationDocument",
1670        "operation": "check",
1671        "path": format!("{BACKUP_PATH}/{file_name}"),
1672        "resultPath": "/run/kaniop-result/result.json",
1673        "format": "kanidmJsonGzip",
1674    })
1675    .to_string();
1676    let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1677    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1678    let data_mover_image = data_mover_image();
1679    let job = Job {
1680        metadata: ObjectMeta {
1681            name: Some(name.to_string()),
1682            namespace: Some(ns.clone()),
1683            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1684            ..Default::default()
1685        },
1686        spec: Some(JobSpec {
1687            backoff_limit: Some(0),
1688            template: PodTemplateSpec {
1689                metadata: Some(ObjectMeta::default()),
1690                spec: Some(PodSpec {
1691                    automount_service_account_token: Some(false),
1692                    restart_policy: Some("Never".to_string()),
1693                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1694                        run_as_non_root: Some(true),
1695                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1696                            type_: "RuntimeDefault".to_string(),
1697                            ..Default::default()
1698                        }),
1699                        ..Default::default()
1700                    }),
1701                    containers: vec![Container {
1702                        name: "source-check".to_string(),
1703                        image: Some(data_mover_image),
1704                        command: Some(vec![
1705                            "/bin/kaniop-data-mover".to_string(),
1706                            "check".to_string(),
1707                            "--operation-doc".to_string(),
1708                            "/run/kaniop/operation.json".to_string(),
1709                        ]),
1710                        security_context: Some(hardened_security_context()),
1711                        resources: Some(job_resource_requirements()),
1712                        env: Some(vec![EnvVar {
1713                            name: "RUST_LOG".to_string(),
1714                            value: Some("info".to_string()),
1715                            ..Default::default()
1716                        }]),
1717                        volume_mounts: Some(vec![
1718                            VolumeMount {
1719                                name: DATA_VOLUME.to_string(),
1720                                mount_path: DATA_PATH.to_string(),
1721                                read_only: Some(true),
1722                                ..Default::default()
1723                            },
1724                            VolumeMount {
1725                                name: "operation".to_string(),
1726                                mount_path: "/run/kaniop".to_string(),
1727                                read_only: Some(true),
1728                                ..Default::default()
1729                            },
1730                            VolumeMount {
1731                                name: "result".to_string(),
1732                                mount_path: "/run/kaniop-result".to_string(),
1733                                ..Default::default()
1734                            },
1735                        ]),
1736                        ..Default::default()
1737                    }],
1738                    volumes: Some(vec![
1739                        Volume {
1740                            name: DATA_VOLUME.to_string(),
1741                            persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1742                                claim_name: primary_pvc_name(target)?,
1743                                read_only: Some(true),
1744                            }),
1745                            ..Default::default()
1746                        },
1747                        Volume {
1748                            name: "operation".to_string(),
1749                            config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1750                                name: operation_cm_name,
1751                                ..Default::default()
1752                            }),
1753                            ..Default::default()
1754                        },
1755                        Volume {
1756                            name: "result".to_string(),
1757                            empty_dir: Some(Default::default()),
1758                            ..Default::default()
1759                        },
1760                    ]),
1761                    ..Default::default()
1762                }),
1763            },
1764            ..Default::default()
1765        }),
1766        ..Default::default()
1767    };
1768    api.create(&PostParams::default(), &job)
1769        .await
1770        .map(|_| ())
1771        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1772}
1773
1774const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1775
1776fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1777    let ns = restore.namespace().unwrap_or_default();
1778    let name = restore.name_any();
1779    let namespace_uuid =
1780        uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1781    let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1782    uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1783}
1784
1785fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1786    let ns = restore.namespace().unwrap_or_default();
1787    let kanidm_uid = &restore.spec.target_ref.uid;
1788    let backup_id = compute_safety_backup_id(restore);
1789    format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1790}
1791
1792struct VerifiedSafetyBackup {
1793    manifest_key: String,
1794    payload_sha256: String,
1795}
1796
1797async fn read_safety_backup_result(
1798    restore: &KanidmRestore,
1799    ctx: &RestoreContext,
1800    job_name: &str,
1801    expected_backup_id: &str,
1802) -> Result<VerifiedSafetyBackup> {
1803    let ns = restore.namespace().unwrap();
1804    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1805    let pod_list = pods
1806        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1807        .await
1808        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1809    let pod = pod_list.items.first().ok_or_else(|| {
1810        Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1811    })?;
1812    let pod_name = pod.name_any();
1813    let container_status = pod
1814        .status
1815        .as_ref()
1816        .and_then(|s| s.container_statuses.as_ref())
1817        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1818        .ok_or_else(|| {
1819            Error::MissingData(format!(
1820                "safety-upload container status not found in pod {pod_name}"
1821            ))
1822        })?;
1823    let termination_message = container_status
1824        .state
1825        .as_ref()
1826        .and_then(|state| state.terminated.as_ref())
1827        .and_then(|t| t.message.as_ref())
1828        .ok_or_else(|| {
1829            Error::MissingData(format!(
1830                "termination message not found in pod {pod_name}; result transport failed"
1831            ))
1832        })?;
1833    if termination_message.is_empty() {
1834        return Err(Error::MissingData(format!(
1835            "termination message is empty in pod {pod_name}; result document was not written"
1836        )));
1837    }
1838    let result_doc = parse_result_document(termination_message).map_err(|e| {
1839        Error::ParseError(format!(
1840            "failed to parse result document from pod {pod_name}: {e}"
1841        ))
1842    })?;
1843    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1844        return Err(Error::ParseError(format!(
1845            "unsupported result document version: {}",
1846            result_doc.api_version
1847        )));
1848    }
1849    if result_doc.kind != "ResultDocument" {
1850        return Err(Error::ParseError(format!(
1851            "invalid result document kind: {}",
1852            result_doc.kind
1853        )));
1854    }
1855    if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1856        return Err(Error::ParseError(format!(
1857            "result document operation mismatch: expected '{}', got '{}'",
1858            SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1859        )));
1860    }
1861    if !result_doc.success {
1862        return Err(Error::ParseError(format!(
1863            "safety backup reported failure with exit code {:?}",
1864            result_doc.exit_code
1865        )));
1866    }
1867    if result_doc.exit_code != ExitCode::Success {
1868        return Err(Error::ParseError(format!(
1869            "safety backup exit code is not Success: {:?}",
1870            result_doc.exit_code
1871        )));
1872    }
1873    let doc_backup_id = result_doc
1874        .backup_id
1875        .as_deref()
1876        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1877    if doc_backup_id != expected_backup_id {
1878        return Err(Error::ParseError(format!(
1879            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1880        )));
1881    }
1882    let manifest_key = result_doc
1883        .manifest_key
1884        .clone()
1885        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1886    if manifest_key.is_empty() {
1887        return Err(Error::ParseError(
1888            "result document manifestKey is empty".to_string(),
1889        ));
1890    }
1891    let payload_sha256 = result_doc
1892        .payload_sha256
1893        .clone()
1894        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1895    if payload_sha256.is_empty() {
1896        return Err(Error::ParseError(
1897            "result document payloadSha256 is empty".to_string(),
1898        ));
1899    }
1900    info!(
1901        restore = %restore.name_any(),
1902        backup_id = %expected_backup_id,
1903        manifest_key = %manifest_key,
1904        "safety backup result verified"
1905    );
1906    Ok(VerifiedSafetyBackup {
1907        manifest_key,
1908        payload_sha256,
1909    })
1910}
1911
1912struct VerifiedSourcePrepResult {
1913    #[allow(dead_code)]
1914    manifest_key: String,
1915    #[allow(dead_code)]
1916    payload_sha256: String,
1917}
1918
1919async fn read_source_prep_result(
1920    restore: &KanidmRestore,
1921    ctx: &RestoreContext,
1922    job_name: &str,
1923    expected_backup_id: &str,
1924) -> Result<VerifiedSourcePrepResult> {
1925    let ns = restore.namespace().unwrap();
1926    let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1927    let pod_list = pods
1928        .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1929        .await
1930        .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1931    let pod = pod_list.items.first().ok_or_else(|| {
1932        Error::MissingData(format!("no pod found for source prep job {job_name}"))
1933    })?;
1934    let pod_name = pod.name_any();
1935    let container_status = pod
1936        .status
1937        .as_ref()
1938        .and_then(|s| s.container_statuses.as_ref())
1939        .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1940        .ok_or_else(|| {
1941            Error::MissingData(format!(
1942                "source-prep container status not found in pod {pod_name}"
1943            ))
1944        })?;
1945    let termination_message = container_status
1946        .state
1947        .as_ref()
1948        .and_then(|state| state.terminated.as_ref())
1949        .and_then(|t| t.message.as_ref())
1950        .ok_or_else(|| {
1951            Error::MissingData(format!(
1952                "termination message not found in pod {pod_name}; result transport failed"
1953            ))
1954        })?;
1955    if termination_message.is_empty() {
1956        return Err(Error::MissingData(format!(
1957            "termination message is empty in pod {pod_name}; result document was not written"
1958        )));
1959    }
1960    let result_doc = parse_result_document(termination_message).map_err(|e| {
1961        Error::ParseError(format!(
1962            "failed to parse result document from pod {pod_name}: {e}"
1963        ))
1964    })?;
1965    if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1966        return Err(Error::ParseError(format!(
1967            "unsupported result document version: {}",
1968            result_doc.api_version
1969        )));
1970    }
1971    if result_doc.kind != "ResultDocument" {
1972        return Err(Error::ParseError(format!(
1973            "invalid result document kind: {}",
1974            result_doc.kind
1975        )));
1976    }
1977    if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1978        return Err(Error::ParseError(format!(
1979            "result document operation mismatch: expected '{}', got '{}'",
1980            SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1981        )));
1982    }
1983    if !result_doc.success {
1984        return Err(Error::ParseError(format!(
1985            "source prep reported failure with exit code {:?}",
1986            result_doc.exit_code
1987        )));
1988    }
1989    if result_doc.exit_code != ExitCode::Success {
1990        return Err(Error::ParseError(format!(
1991            "source prep exit code is not Success: {:?}",
1992            result_doc.exit_code
1993        )));
1994    }
1995    let doc_backup_id = result_doc
1996        .backup_id
1997        .as_deref()
1998        .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1999    if doc_backup_id != expected_backup_id {
2000        return Err(Error::ParseError(format!(
2001            "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2002        )));
2003    }
2004    let manifest_key = result_doc
2005        .manifest_key
2006        .clone()
2007        .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2008    if manifest_key.is_empty() {
2009        return Err(Error::ParseError(
2010            "result document manifestKey is empty".to_string(),
2011        ));
2012    }
2013    let payload_sha256 = result_doc
2014        .payload_sha256
2015        .clone()
2016        .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2017    if payload_sha256.is_empty() {
2018        return Err(Error::ParseError(
2019            "result document payloadSha256 is empty".to_string(),
2020        ));
2021    }
2022    info!(
2023        restore = %restore.name_any(),
2024        backup_id = %expected_backup_id,
2025        manifest_key = %manifest_key,
2026        payload_sha256 = %payload_sha256,
2027        "source prep result verified"
2028    );
2029    Ok(VerifiedSourcePrepResult {
2030        manifest_key,
2031        payload_sha256,
2032    })
2033}
2034
2035fn hardened_security_context() -> SecurityContext {
2036    SecurityContext {
2037        allow_privilege_escalation: Some(false),
2038        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2039            drop: Some(vec!["ALL".to_string()]),
2040            ..Default::default()
2041        }),
2042        read_only_root_filesystem: Some(true),
2043        run_as_non_root: Some(true),
2044        seccomp_profile: Some(SeccompProfile {
2045            type_: "RuntimeDefault".to_string(),
2046            ..Default::default()
2047        }),
2048        ..Default::default()
2049    }
2050}
2051
2052fn kanidm_job_security_context() -> SecurityContext {
2053    SecurityContext {
2054        allow_privilege_escalation: Some(false),
2055        capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2056            drop: Some(vec!["ALL".to_string()]),
2057            ..Default::default()
2058        }),
2059        seccomp_profile: Some(SeccompProfile {
2060            type_: "RuntimeDefault".to_string(),
2061            ..Default::default()
2062        }),
2063        ..Default::default()
2064    }
2065}
2066
2067fn job_resource_requirements() -> ResourceRequirements {
2068    ResourceRequirements {
2069        requests: Some(std::collections::BTreeMap::from([
2070            ("cpu".to_string(), Quantity("100m".to_string())),
2071            ("memory".to_string(), Quantity("128Mi".to_string())),
2072        ])),
2073        limits: Some(std::collections::BTreeMap::from([
2074            ("cpu".to_string(), Quantity("2".to_string())),
2075            ("memory".to_string(), Quantity("2Gi".to_string())),
2076        ])),
2077        ..Default::default()
2078    }
2079}
2080
2081async fn ensure_safety_backup_job(
2082    restore: &KanidmRestore,
2083    target: &Kanidm,
2084    ctx: &RestoreContext,
2085    name: &str,
2086    backup_id: &str,
2087    manifest_key: &str,
2088) -> Result<()> {
2089    let ns = restore.namespace().unwrap();
2090    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2091    if api
2092        .get_opt(name)
2093        .await
2094        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2095        .is_some()
2096    {
2097        return Ok(());
2098    }
2099    let safety = restore
2100        .spec
2101        .safety_backup
2102        .as_ref()
2103        .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2104    let repo_ref = safety
2105        .repository_ref
2106        .as_ref()
2107        .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2108    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2109        .get(&repo_ref.name)
2110        .await
2111        .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2112    let _namespace_uid = &ns;
2113    let _kanidm_uid = &restore.spec.target_ref.uid;
2114    let endpoint = repo
2115        .spec
2116        .s3
2117        .endpoint
2118        .as_deref()
2119        .unwrap_or("https://s3.amazonaws.com");
2120    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2121    let operation_doc = build_safety_upload_operation_doc(
2122        restore,
2123        target,
2124        backup_id,
2125        manifest_key,
2126        &repo.spec.s3.bucket,
2127        &repo.spec.s3.prefix,
2128        endpoint,
2129        region,
2130        repo.spec.s3.force_path_style,
2131        repo.spec.s3.insecure,
2132        repo.spec
2133            .s3
2134            .ca_bundle_ref
2135            .as_deref()
2136            .map(|_| ca_bundle_path())
2137            .as_deref(),
2138        repo.spec.encryption.as_ref(),
2139    )?;
2140    let operation_cm_name = format!("{name}-op");
2141    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2142    let auth_method = &repo.spec.authentication.writer;
2143    let data_mover_image = data_mover_image();
2144    let init_command = vec![
2145        "kanidmd".to_string(),
2146        "database".to_string(),
2147        "backup".to_string(),
2148        format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2149    ];
2150    let upload_script = format!(
2151        r#"set -eu
2152/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2153RESULT_FILE="/run/kaniop-result/result.json"
2154TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2155if [ ! -f "$RESULT_FILE" ]; then
2156  echo "ERROR: result document not found at $RESULT_FILE"
2157  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2158  exit 1
2159fi
2160if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2161  echo "ERROR: safety backup upload reported failure"
2162  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2163  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2164  exit 1
2165fi
2166dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2167echo "safety backup upload completed successfully"
2168"#
2169    );
2170    let job = Job {
2171        metadata: ObjectMeta {
2172            name: Some(name.to_string()),
2173            namespace: Some(ns.clone()),
2174            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2175            ..Default::default()
2176        },
2177        spec: Some(JobSpec {
2178            backoff_limit: Some(0),
2179            template: PodTemplateSpec {
2180                metadata: Some(ObjectMeta::default()),
2181                spec: Some(PodSpec {
2182                    automount_service_account_token: Some(false),
2183                    restart_policy: Some("Never".to_string()),
2184                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2185                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2186                            type_: "RuntimeDefault".to_string(),
2187                            ..Default::default()
2188                        }),
2189                        ..Default::default()
2190                    }),
2191                    init_containers: Some(vec![Container {
2192                        name: "safety-backup".to_string(),
2193                        image: Some(restore.spec.restore_image.clone()),
2194                        command: Some(init_command),
2195                        security_context: Some(SecurityContext {
2196                            allow_privilege_escalation: Some(false),
2197                            capabilities: Some(Capabilities {
2198                                drop: Some(vec!["ALL".to_string()]),
2199                                ..Default::default()
2200                            }),
2201                            read_only_root_filesystem: Some(true),
2202                            seccomp_profile: Some(SeccompProfile {
2203                                type_: "RuntimeDefault".to_string(),
2204                                ..Default::default()
2205                            }),
2206                            ..Default::default()
2207                        }),
2208                        resources: Some(job_resource_requirements()),
2209                        env: Some(build_kanidm_db_env_vars(target)),
2210                        volume_mounts: Some(vec![
2211                            VolumeMount {
2212                                name: DATA_VOLUME.to_string(),
2213                                mount_path: DATA_PATH.to_string(),
2214                                read_only: Some(false),
2215                                ..Default::default()
2216                            },
2217                            VolumeMount {
2218                                name: TLS_VOLUME.to_string(),
2219                                mount_path: TLS_PATH.to_string(),
2220                                read_only: Some(true),
2221                                ..Default::default()
2222                            },
2223                            VolumeMount {
2224                                name: SHARED_VOLUME.to_string(),
2225                                mount_path: SHARED_VOL_PATH.to_string(),
2226                                ..Default::default()
2227                            },
2228                        ]),
2229                        ..Default::default()
2230                    }]),
2231                    containers: vec![Container {
2232                        name: "safety-upload".to_string(),
2233                        image: Some(data_mover_image),
2234                        command: Some(vec!["/bin/sh".to_string()]),
2235                        args: Some(vec!["-c".to_string(), upload_script]),
2236                        security_context: Some(hardened_security_context()),
2237                        resources: Some(job_resource_requirements()),
2238                        env: {
2239                            let mut env_vars =
2240                                build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2241                            env_vars.push(EnvVar {
2242                                name: "RUST_LOG".to_string(),
2243                                value: Some("info".to_string()),
2244                                ..Default::default()
2245                            });
2246                            if repo.spec.s3.ca_bundle_ref.is_some() {
2247                                env_vars.push(ca_bundle_env_var());
2248                            }
2249                            env_vars.extend(build_encryption_env_vars(
2250                                repo.spec
2251                                    .encryption
2252                                    .as_ref()
2253                                    .and_then(|e| e.key_ref.as_ref()),
2254                            ));
2255                            Some(env_vars)
2256                        },
2257                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2258                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2259                        volume_mounts: {
2260                            let mut mounts = vec![
2261                                VolumeMount {
2262                                    name: SHARED_VOLUME.to_string(),
2263                                    mount_path: SHARED_VOL_PATH.to_string(),
2264                                    read_only: Some(true),
2265                                    ..Default::default()
2266                                },
2267                                VolumeMount {
2268                                    name: "operation".to_string(),
2269                                    mount_path: "/run/kaniop".to_string(),
2270                                    read_only: Some(true),
2271                                    ..Default::default()
2272                                },
2273                                VolumeMount {
2274                                    name: "result".to_string(),
2275                                    mount_path: "/run/kaniop-result".to_string(),
2276                                    ..Default::default()
2277                                },
2278                            ];
2279                            mounts.extend(build_auth_volume_mounts(auth_method));
2280                            if repo.spec.s3.ca_bundle_ref.is_some() {
2281                                mounts.push(build_ca_bundle_volume_mount());
2282                            }
2283                            Some(mounts)
2284                        },
2285                        ..Default::default()
2286                    }],
2287                    volumes: {
2288                        let mut vols = vec![
2289                            Volume {
2290                                name: DATA_VOLUME.to_string(),
2291                                persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2292                                    claim_name: primary_pvc_name(target)?,
2293                                    read_only: Some(false),
2294                                }),
2295                                ..Default::default()
2296                            },
2297                            Volume {
2298                                name: TLS_VOLUME.to_string(),
2299                                secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2300                                    secret_name: Some(target.effective_tls_secret_name()),
2301                                    ..Default::default()
2302                                }),
2303                                ..Default::default()
2304                            },
2305                            Volume {
2306                                name: SHARED_VOLUME.to_string(),
2307                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2308                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2309                                    ..Default::default()
2310                                }),
2311                                ..Default::default()
2312                            },
2313                            Volume {
2314                                name: "operation".to_string(),
2315                                config_map: Some(ConfigMapVolumeSource {
2316                                    name: operation_cm_name,
2317                                    ..Default::default()
2318                                }),
2319                                ..Default::default()
2320                            },
2321                            Volume {
2322                                name: "result".to_string(),
2323                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2324                                    size_limit: Some(Quantity("16Mi".to_string())),
2325                                    ..Default::default()
2326                                }),
2327                                ..Default::default()
2328                            },
2329                        ];
2330                        vols.extend(build_auth_volumes(auth_method));
2331                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2332                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2333                        }
2334                        Some(vols)
2335                    },
2336                    ..Default::default()
2337                }),
2338            },
2339            ..Default::default()
2340        }),
2341        ..Default::default()
2342    };
2343    api.create(&PostParams::default(), &job)
2344        .await
2345        .map(|_| ())
2346        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2347}
2348
2349async fn ensure_source_prep_job(
2350    restore: &KanidmRestore,
2351    target: &Kanidm,
2352    ctx: &RestoreContext,
2353    name: &str,
2354) -> Result<()> {
2355    let ns = restore.namespace().unwrap();
2356    let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2357    if api
2358        .get_opt(name)
2359        .await
2360        .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2361        .is_some()
2362    {
2363        return Ok(());
2364    }
2365    let backup_name = restore
2366        .spec
2367        .source
2368        .backup_ref
2369        .as_ref()
2370        .map(|b| b.name.clone())
2371        .unwrap_or_default();
2372    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2373        .get(&backup_name)
2374        .await
2375        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2376    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2377        .get(&backup.spec.repository_ref.name)
2378        .await
2379        .map_err(|e| {
2380            Error::kube_error(
2381                "get",
2382                "KanidmBackupRepository",
2383                &ns,
2384                &backup.spec.repository_ref.name,
2385                e,
2386            )
2387        })?;
2388    let endpoint = repo
2389        .spec
2390        .s3
2391        .endpoint
2392        .as_deref()
2393        .unwrap_or("https://s3.amazonaws.com");
2394    let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2395    let operation_doc = build_download_operation_doc(
2396        restore,
2397        target,
2398        &backup.spec.manifest_key,
2399        &backup.spec.backup_id,
2400        &repo.spec.s3.bucket,
2401        &repo.spec.s3.prefix,
2402        endpoint,
2403        region,
2404        repo.spec.s3.force_path_style,
2405        repo.spec.s3.insecure,
2406        repo.spec
2407            .s3
2408            .ca_bundle_ref
2409            .as_deref()
2410            .map(|_| ca_bundle_path())
2411            .as_deref(),
2412        repo.spec.encryption.as_ref(),
2413    );
2414    let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2415    ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2416    let auth_method = &repo.spec.authentication.reader;
2417    let data_mover_image = data_mover_image();
2418    let download_script = format!(
2419        r#"set -eu
2420/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2421RESULT_FILE="/run/kaniop-result/result.json"
2422TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2423if [ ! -f "$RESULT_FILE" ]; then
2424  echo "ERROR: result document not found at $RESULT_FILE"
2425  printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2426  exit 1
2427fi
2428if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2429  echo "ERROR: source preparation download reported failure"
2430  dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2431  dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2432  exit 1
2433fi
2434dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2435echo "source preparation download completed successfully"
2436"#
2437    );
2438    let job = Job {
2439        metadata: ObjectMeta {
2440            name: Some(name.to_string()),
2441            namespace: Some(ns.clone()),
2442            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2443            ..Default::default()
2444        },
2445        spec: Some(JobSpec {
2446            backoff_limit: Some(0),
2447            template: PodTemplateSpec {
2448                metadata: Some(ObjectMeta::default()),
2449                spec: Some(PodSpec {
2450                    automount_service_account_token: Some(false),
2451                    restart_policy: Some("Never".to_string()),
2452                    security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2453                        run_as_non_root: Some(true),
2454                        seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2455                            type_: "RuntimeDefault".to_string(),
2456                            ..Default::default()
2457                        }),
2458                        ..Default::default()
2459                    }),
2460                    containers: vec![Container {
2461                        name: "source-prep".to_string(),
2462                        image: Some(data_mover_image),
2463                        command: Some(vec!["/bin/sh".to_string()]),
2464                        args: Some(vec!["-c".to_string(), download_script]),
2465                        security_context: Some(hardened_security_context()),
2466                        resources: Some(job_resource_requirements()),
2467                        termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2468                        termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2469                        env: {
2470                            let mut env_vars = build_auth_env_vars(
2471                                auth_method,
2472                                &backup.spec.repository_ref.name,
2473                                AuthRole::Reader,
2474                            );
2475                            env_vars.push(EnvVar {
2476                                name: "RUST_LOG".to_string(),
2477                                value: Some("info".to_string()),
2478                                ..Default::default()
2479                            });
2480                            if repo.spec.s3.ca_bundle_ref.is_some() {
2481                                env_vars.push(ca_bundle_env_var());
2482                            }
2483                            env_vars.extend(build_encryption_env_vars(
2484                                repo.spec
2485                                    .encryption
2486                                    .as_ref()
2487                                    .and_then(|e| e.key_ref.as_ref()),
2488                            ));
2489                            Some(env_vars)
2490                        },
2491                        volume_mounts: {
2492                            let mut mounts = vec![
2493                                VolumeMount {
2494                                    name: STAGING_VOLUME.to_string(),
2495                                    mount_path: STAGING_PATH.to_string(),
2496                                    ..Default::default()
2497                                },
2498                                VolumeMount {
2499                                    name: "operation".to_string(),
2500                                    mount_path: "/run/kaniop".to_string(),
2501                                    read_only: Some(true),
2502                                    ..Default::default()
2503                                },
2504                                VolumeMount {
2505                                    name: "result".to_string(),
2506                                    mount_path: "/run/kaniop-result".to_string(),
2507                                    ..Default::default()
2508                                },
2509                            ];
2510                            mounts.extend(build_auth_volume_mounts(auth_method));
2511                            if repo.spec.s3.ca_bundle_ref.is_some() {
2512                                mounts.push(build_ca_bundle_volume_mount());
2513                            }
2514                            Some(mounts)
2515                        },
2516                        ..Default::default()
2517                    }],
2518                    volumes: {
2519                        let mut vols = vec![
2520                            Volume {
2521                                name: STAGING_VOLUME.to_string(),
2522                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2523                                    size_limit: Some(crate::controller::backup_job_volume_size()),
2524                                    ..Default::default()
2525                                }),
2526                                ..Default::default()
2527                            },
2528                            Volume {
2529                                name: "operation".to_string(),
2530                                config_map: Some(ConfigMapVolumeSource {
2531                                    name: operation_cm_name,
2532                                    ..Default::default()
2533                                }),
2534                                ..Default::default()
2535                            },
2536                            Volume {
2537                                name: "result".to_string(),
2538                                empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2539                                    size_limit: Some(Quantity("16Mi".to_string())),
2540                                    ..Default::default()
2541                                }),
2542                                ..Default::default()
2543                            },
2544                        ];
2545                        vols.extend(build_auth_volumes(auth_method));
2546                        if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2547                            vols.push(build_ca_bundle_volume(ca_bundle_ref));
2548                        }
2549                        Some(vols)
2550                    },
2551                    ..Default::default()
2552                }),
2553            },
2554            ..Default::default()
2555        }),
2556        ..Default::default()
2557    };
2558    api.create(&PostParams::default(), &job)
2559        .await
2560        .map(|_| ())
2561        .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2562}
2563
2564#[allow(clippy::too_many_arguments)]
2565fn build_safety_upload_operation_doc(
2566    restore: &KanidmRestore,
2567    target: &Kanidm,
2568    backup_id: &str,
2569    manifest_key: &str,
2570    bucket: &str,
2571    prefix: &str,
2572    endpoint: &str,
2573    region: &str,
2574    force_path_style: bool,
2575    insecure: bool,
2576    ca_bundle_path: Option<&str>,
2577    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2578) -> Result<String> {
2579    let enc_mode =
2580        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2581    let enc_key_id = encryption
2582        .and_then(|e| e.key_id.as_ref())
2583        .map(|s| serde_json::Value::String(s.clone()));
2584    let op = UploadOperation {
2585        payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2586        bucket: bucket.to_string(),
2587        prefix: prefix.to_string(),
2588        endpoint: endpoint.to_string(),
2589        region: region.to_string(),
2590        force_path_style,
2591        insecure,
2592        ca_bundle_path: ca_bundle_path.map(str::to_string),
2593        backup_id: backup_id.to_string(),
2594        namespace_uid: restore.namespace().unwrap_or_default(),
2595        kanidm_uid: restore.spec.target_ref.uid.clone(),
2596        kanidm_name: restore.spec.target_ref.name.clone(),
2597        domain: target.spec.domain.clone(),
2598        kanidm_version: target
2599            .status
2600            .as_ref()
2601            .and_then(|s| s.version.as_ref())
2602            .map(|v| v.image_tag.clone())
2603            .unwrap_or_default(),
2604        image_digest: None,
2605        consistency: "kanidm-offline".to_string(),
2606        reason: "restore-safety".to_string(),
2607        encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2608        encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2609        result_path: "/run/kaniop-result/result.json".to_string(),
2610        max_concurrent_parts: 4,
2611        max_retries: 3,
2612    };
2613    let _ = manifest_key;
2614    let doc = OperationDocument {
2615        api_version: OPERATION_DOC_VERSION.to_string(),
2616        kind: "OperationDocument".to_string(),
2617        spec: OperationSpec::Upload(op),
2618    };
2619    serde_json::to_string(&doc)
2620        .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2621}
2622
2623async fn validate_backup_ref(
2624    restore: &KanidmRestore,
2625    target: &Kanidm,
2626    ctx: &RestoreContext,
2627) -> Result<()> {
2628    let backup_ref = restore
2629        .spec
2630        .source
2631        .backup_ref
2632        .as_ref()
2633        .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2634    let ns = restore.namespace().unwrap();
2635    let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2636        .get(&backup_ref.name)
2637        .await
2638        .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2639    if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2640        return Err(Error::MissingData(format!(
2641            "KanidmBackup '{}' is not Ready (phase: {:?})",
2642            backup_ref.name,
2643            backup.status.as_ref().map(|s| &s.phase)
2644        )));
2645    }
2646    if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2647        return Err(Error::MissingData(format!(
2648            "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2649            backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2650        )));
2651    }
2652    if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2653        return Err(Error::MissingData(format!(
2654            "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2655            backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2656        )));
2657    }
2658    let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2659        .get(&backup.spec.repository_ref.name)
2660        .await
2661        .map_err(|e| {
2662            Error::kube_error(
2663                "get",
2664                "KanidmBackupRepository",
2665                &ns,
2666                &backup.spec.repository_ref.name,
2667                e,
2668            )
2669        })?;
2670    if !has_accepted_condition(
2671        &repo
2672            .status
2673            .as_ref()
2674            .map(|s| &s.conditions)
2675            .cloned()
2676            .unwrap_or_default(),
2677    ) {
2678        return Err(Error::MissingData(format!(
2679            "KanidmBackupRepository '{}' configuration has not been accepted",
2680            backup.spec.repository_ref.name
2681        )));
2682    }
2683    validate_backup_compatibility(restore, target, &backup)?;
2684    Ok(())
2685}
2686
2687fn has_accepted_condition(conditions: &[Condition]) -> bool {
2688    conditions
2689        .iter()
2690        .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2691}
2692
2693fn validate_backup_compatibility(
2694    restore: &KanidmRestore,
2695    target: &Kanidm,
2696    backup: &KanidmBackup,
2697) -> Result<()> {
2698    let status = backup.status.as_ref();
2699    if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2700        let target_version = target
2701            .status
2702            .as_ref()
2703            .and_then(|s| s.version.as_ref())
2704            .map(|v| v.image_tag.as_str());
2705        if let Some(tv) = target_version {
2706            if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2707                return Err(Error::MissingData(format!(
2708                    "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2709                )));
2710            }
2711        }
2712    }
2713    if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2714        if !backup_digest.is_empty()
2715            && restore.spec.restore_image.contains('@')
2716            && !restore.spec.restore_image.contains(backup_digest)
2717        {
2718            return Err(Error::MissingData(format!(
2719                "restore image digest does not match backup image digest '{backup_digest}'"
2720            )));
2721        }
2722    }
2723    Ok(())
2724}
2725
2726#[allow(clippy::too_many_arguments)]
2727fn build_download_operation_doc(
2728    restore: &KanidmRestore,
2729    target: &Kanidm,
2730    manifest_key: &str,
2731    expected_backup_id: &str,
2732    bucket: &str,
2733    prefix: &str,
2734    endpoint: &str,
2735    region: &str,
2736    force_path_style: bool,
2737    insecure: bool,
2738    ca_bundle_path: Option<&str>,
2739    encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2740) -> String {
2741    let enc_mode =
2742        encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2743    let enc_key_id = encryption
2744        .and_then(|e| e.key_id.as_ref())
2745        .map(|s| serde_json::Value::String(s.clone()));
2746    serde_json::json!({
2747        "apiVersion": "backup.kaniop.rs/v1alpha1",
2748        "kind": "OperationDocument",
2749        "operation": "download",
2750        "manifestKey": manifest_key,
2751        "bucket": bucket,
2752        "prefix": prefix,
2753        "endpoint": endpoint,
2754        "region": region,
2755        "forcePathStyle": force_path_style,
2756        "insecure": insecure,
2757        "caBundlePath": ca_bundle_path,
2758        "expectedBackupId": expected_backup_id,
2759        "expectedKanidmUid": restore.spec.target_ref.uid,
2760        "expectedDomain": target.spec.domain,
2761        "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2762        "resultPath": "/run/kaniop-result/result.json",
2763        "maxRetries": 3,
2764        "encryptionMode": enc_mode,
2765        "encryptionKeyId": enc_key_id,
2766    })
2767    .to_string()
2768}
2769
2770async fn ensure_operation_configmap(
2771    restore: &KanidmRestore,
2772    name: &str,
2773    data: &str,
2774    ns: &str,
2775    ctx: &RestoreContext,
2776) -> Result<()> {
2777    let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2778    if api
2779        .get_opt(name)
2780        .await
2781        .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2782        .is_some()
2783    {
2784        return Ok(());
2785    }
2786    let cm = ConfigMap {
2787        metadata: ObjectMeta {
2788            name: Some(name.to_string()),
2789            namespace: Some(ns.to_string()),
2790            owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2791            ..Default::default()
2792        },
2793        data: Some(std::collections::BTreeMap::from([(
2794            "operation.json".to_string(),
2795            data.to_string(),
2796        )])),
2797        ..Default::default()
2798    };
2799    api.create(&PostParams::default(), &cm)
2800        .await
2801        .map(|_| ())
2802        .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2803}
2804
2805#[cfg(test)]
2806mod tests {
2807    use super::{
2808        BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2809        CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2810        KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2811        ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2812        hardened_security_context, has_accepted_condition, is_remote_source,
2813        kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2814        validate_safety_backup_config, validate_source,
2815    };
2816    use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2817    use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2818    use kube::api::ObjectMeta as ApiObjectMeta;
2819    use std::collections::BTreeMap;
2820
2821    fn make_restore(
2822        source: KanidmRestoreSource,
2823        safety: Option<SafetyBackupConfig>,
2824    ) -> KanidmRestore {
2825        KanidmRestore {
2826            metadata: ApiObjectMeta {
2827                name: Some("test-restore".to_string()),
2828                namespace: Some("default".to_string()),
2829                ..Default::default()
2830            },
2831            spec: KanidmRestoreSpec {
2832                target_ref: KanidmRestoreTargetRef {
2833                    name: "corp-idm".to_string(),
2834                    uid: "test-uid-123".to_string(),
2835                },
2836                source,
2837                restore_image: "kanidm/server@sha256:abc".to_string(),
2838                safety_backup: safety,
2839            },
2840            status: None,
2841        }
2842    }
2843
2844    fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2845        super::super::crd::Kanidm {
2846            status: Some(super::super::crd::KanidmStatus {
2847                available_replicas: 1,
2848                replicas: 1,
2849                unavailable_replicas: 0,
2850                updated_replicas: 1,
2851                replica_statuses: vec![],
2852                replica_column: "1/1".to_string(),
2853                secret_name: None,
2854                version: Some(super::super::crd::KanidmVersionStatus {
2855                    image_tag: image_tag.to_string(),
2856                    upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2857                    compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2858                }),
2859                domain_appearance_image: None,
2860                mail_sender: None,
2861                conditions: None,
2862            }),
2863            ..Default::default()
2864        }
2865    }
2866
2867    #[test]
2868    fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2869        assert!(!KanidmRestoreStatus::default().database_mutation_started);
2870    }
2871
2872    #[test]
2873    fn completed_restore_sets_ready_condition() {
2874        let mut status = KanidmRestoreStatus {
2875            phase: KanidmRestorePhase::Completed,
2876            ..Default::default()
2877        };
2878        super::update_restore_conditions(&mut status, Some(7));
2879        assert!(status.conditions.iter().any(|condition| {
2880            condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2881        }));
2882        assert!(!status.conditions.iter().any(|condition| {
2883            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2884        }));
2885    }
2886
2887    #[test]
2888    fn failed_restore_sets_failed_condition() {
2889        let mut status = KanidmRestoreStatus {
2890            phase: KanidmRestorePhase::Failed,
2891            message: Some("verification failed".to_string()),
2892            ..Default::default()
2893        };
2894        super::update_restore_conditions(&mut status, Some(8));
2895        assert!(status.conditions.iter().any(|condition| {
2896            condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2897        }));
2898    }
2899
2900    #[test]
2901    fn restore_filename_is_confined_to_backup_directory() {
2902        assert!(safe_basename("backup.json.gz"));
2903        assert!(!safe_basename("../backup.json.gz"));
2904        assert!(!safe_basename("a/b"));
2905        assert!(!safe_basename(""));
2906    }
2907
2908    #[test]
2909    fn restore_image_rejects_latest_and_unpinned_names() {
2910        assert!(mutable_image("kanidm/server:latest"));
2911        assert!(mutable_image("kanidm/server"));
2912        assert!(!mutable_image("kanidm/server:1.10.0"));
2913        assert!(!mutable_image("kanidm/server@sha256:abc"));
2914    }
2915
2916    #[test]
2917    fn source_validation_rejects_both_local_and_backup_ref() {
2918        let source = KanidmRestoreSource {
2919            local: Some(super::KanidmRestoreLocalSource {
2920                file_name: "backup.json".to_string(),
2921            }),
2922            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2923                name: "some-backup".to_string(),
2924            }),
2925        };
2926        let restore = make_restore(source, None);
2927        assert!(validate_source(&restore).is_err());
2928    }
2929
2930    #[test]
2931    fn source_validation_rejects_empty_source() {
2932        let source = KanidmRestoreSource {
2933            local: None,
2934            backup_ref: None,
2935        };
2936        let restore = make_restore(source, None);
2937        assert!(validate_source(&restore).is_err());
2938    }
2939
2940    #[test]
2941    fn source_validation_accepts_local_source() {
2942        let source = KanidmRestoreSource {
2943            local: Some(super::KanidmRestoreLocalSource {
2944                file_name: "backup.json".to_string(),
2945            }),
2946            backup_ref: None,
2947        };
2948        let restore = make_restore(source, None);
2949        assert!(validate_source(&restore).is_ok());
2950    }
2951
2952    #[test]
2953    fn source_validation_accepts_backup_ref_source() {
2954        let source = KanidmRestoreSource {
2955            local: None,
2956            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2957                name: "corp-idm-019c7c76".to_string(),
2958            }),
2959        };
2960        let restore = make_restore(source, None);
2961        assert!(validate_source(&restore).is_ok());
2962    }
2963
2964    #[test]
2965    fn source_validation_rejects_empty_backup_ref_name() {
2966        let source = KanidmRestoreSource {
2967            local: None,
2968            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2969                name: "".to_string(),
2970            }),
2971        };
2972        let restore = make_restore(source, None);
2973        assert!(validate_source(&restore).is_err());
2974    }
2975
2976    #[test]
2977    fn is_remote_source_true_for_backup_ref() {
2978        let source = KanidmRestoreSource {
2979            local: None,
2980            backup_ref: Some(super::KanidmRestoreBackupRefSource {
2981                name: "backup-1".to_string(),
2982            }),
2983        };
2984        let restore = make_restore(source, None);
2985        assert!(is_remote_source(&restore));
2986    }
2987
2988    #[test]
2989    fn is_remote_source_false_for_local() {
2990        let source = KanidmRestoreSource {
2991            local: Some(super::KanidmRestoreLocalSource {
2992                file_name: "backup.json".to_string(),
2993            }),
2994            backup_ref: None,
2995        };
2996        let restore = make_restore(source, None);
2997        assert!(!is_remote_source(&restore));
2998    }
2999
3000    #[test]
3001    fn requires_safety_backup_default_true() {
3002        let source = KanidmRestoreSource {
3003            local: None,
3004            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3005                name: "backup-1".to_string(),
3006            }),
3007        };
3008        let restore = make_restore(source, None);
3009        assert!(requires_safety_backup(&restore));
3010    }
3011
3012    #[test]
3013    fn requires_safety_backup_false_when_skip() {
3014        let source = KanidmRestoreSource {
3015            local: None,
3016            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3017                name: "backup-1".to_string(),
3018            }),
3019        };
3020        let safety = SafetyBackupConfig {
3021            repository_ref: Some(SafetyBackupRepositoryRef {
3022                name: "offsite".to_string(),
3023            }),
3024            skip: true,
3025        };
3026        let restore = make_restore(source, Some(safety));
3027        assert!(!requires_safety_backup(&restore));
3028    }
3029
3030    #[test]
3031    fn break_glass_validation_rejects_skip_without_reason() {
3032        let source = KanidmRestoreSource {
3033            local: None,
3034            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3035                name: "backup-1".to_string(),
3036            }),
3037        };
3038        let safety = SafetyBackupConfig {
3039            repository_ref: Some(SafetyBackupRepositoryRef {
3040                name: "offsite".to_string(),
3041            }),
3042            skip: true,
3043        };
3044        let mut restore = make_restore(source, Some(safety));
3045        restore.metadata.annotations = Some(BTreeMap::from([(
3046            BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3047            "commander@example.com".to_string(),
3048        )]));
3049        assert!(validate_safety_backup_config(&restore).is_err());
3050    }
3051
3052    #[test]
3053    fn break_glass_validation_rejects_skip_without_approver() {
3054        let source = KanidmRestoreSource {
3055            local: None,
3056            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3057                name: "backup-1".to_string(),
3058            }),
3059        };
3060        let safety = SafetyBackupConfig {
3061            repository_ref: Some(SafetyBackupRepositoryRef {
3062                name: "offsite".to_string(),
3063            }),
3064            skip: true,
3065        };
3066        let mut restore = make_restore(source, Some(safety));
3067        restore.metadata.annotations = Some(BTreeMap::from([(
3068            BREAK_GLASS_REASON_ANNOTATION.to_string(),
3069            "PVC is unreadable".to_string(),
3070        )]));
3071        assert!(validate_safety_backup_config(&restore).is_err());
3072    }
3073
3074    #[test]
3075    fn break_glass_validation_accepts_valid_annotations() {
3076        let source = KanidmRestoreSource {
3077            local: None,
3078            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3079                name: "backup-1".to_string(),
3080            }),
3081        };
3082        let safety = SafetyBackupConfig {
3083            repository_ref: Some(SafetyBackupRepositoryRef {
3084                name: "offsite".to_string(),
3085            }),
3086            skip: true,
3087        };
3088        let mut restore = make_restore(source, Some(safety));
3089        restore.metadata.annotations = Some(BTreeMap::from([
3090            (
3091                BREAK_GLASS_REASON_ANNOTATION.to_string(),
3092                "PVC is unreadable".to_string(),
3093            ),
3094            (
3095                BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3096                "commander@example.com".to_string(),
3097            ),
3098        ]));
3099        assert!(validate_safety_backup_config(&restore).is_ok());
3100    }
3101
3102    #[test]
3103    fn remote_source_requires_repository_ref_when_not_skipping() {
3104        let source = KanidmRestoreSource {
3105            local: None,
3106            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3107                name: "backup-1".to_string(),
3108            }),
3109        };
3110        let safety = SafetyBackupConfig {
3111            repository_ref: None,
3112            skip: false,
3113        };
3114        let restore = make_restore(source, Some(safety));
3115        assert!(validate_safety_backup_config(&restore).is_err());
3116    }
3117
3118    #[test]
3119    fn remote_source_accepts_repository_ref() {
3120        let source = KanidmRestoreSource {
3121            local: None,
3122            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3123                name: "backup-1".to_string(),
3124            }),
3125        };
3126        let safety = SafetyBackupConfig {
3127            repository_ref: Some(SafetyBackupRepositoryRef {
3128                name: "offsite".to_string(),
3129            }),
3130            skip: false,
3131        };
3132        let restore = make_restore(source, Some(safety));
3133        assert!(validate_safety_backup_config(&restore).is_ok());
3134    }
3135
3136    #[test]
3137    fn phase_ordering_includes_safety_backup_and_preparing_source() {
3138        let phases = vec![
3139            KanidmRestorePhase::Pending,
3140            KanidmRestorePhase::Validating,
3141            KanidmRestorePhase::Quiescing,
3142            KanidmRestorePhase::SafetyBackup,
3143            KanidmRestorePhase::PreparingSource,
3144            KanidmRestorePhase::RestoringPrimary,
3145            KanidmRestorePhase::Verifying,
3146            KanidmRestorePhase::RebuildingReplicas,
3147            KanidmRestorePhase::Resuming,
3148            KanidmRestorePhase::Completed,
3149        ];
3150        assert_eq!(phases.len(), 10);
3151        assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3152        assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3153    }
3154
3155    #[test]
3156    fn status_default_has_empty_phase_timestamps() {
3157        let status = KanidmRestoreStatus::default();
3158        assert!(status.phase_timestamps.is_empty());
3159    }
3160
3161    #[test]
3162    fn status_default_has_empty_original_replicas() {
3163        let status = KanidmRestoreStatus::default();
3164        assert!(status.original_replicas.is_empty());
3165    }
3166
3167    #[test]
3168    fn hardened_security_context_drops_all_capabilities() {
3169        let ctx = hardened_security_context();
3170        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3171        assert_eq!(ctx.read_only_root_filesystem, Some(true));
3172        assert_eq!(ctx.run_as_non_root, Some(true));
3173        let caps = ctx.capabilities.unwrap();
3174        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3175        assert_eq!(
3176            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3177            Some("RuntimeDefault")
3178        );
3179    }
3180
3181    #[test]
3182    fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3183        let ctx = kanidm_job_security_context();
3184        assert_eq!(ctx.allow_privilege_escalation, Some(false));
3185        assert!(ctx.run_as_non_root.is_none());
3186        assert!(ctx.read_only_root_filesystem.is_none());
3187        let caps = ctx.capabilities.unwrap();
3188        assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3189        assert_eq!(
3190            ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3191            Some("RuntimeDefault")
3192        );
3193    }
3194
3195    #[test]
3196    fn safety_backup_job_name_is_deterministic() {
3197        let source = KanidmRestoreSource {
3198            local: Some(super::KanidmRestoreLocalSource {
3199                file_name: "backup.json".to_string(),
3200            }),
3201            backup_ref: None,
3202        };
3203        let restore = make_restore(source, None);
3204        let name1 = super::safety_backup_job_name(&restore);
3205        let name2 = super::safety_backup_job_name(&restore);
3206        assert_eq!(name1, name2);
3207        assert_eq!(name1, "test-restore-safety-backup");
3208    }
3209
3210    #[test]
3211    fn source_prep_job_name_is_deterministic() {
3212        let source = KanidmRestoreSource {
3213            local: None,
3214            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3215                name: "backup-1".to_string(),
3216            }),
3217        };
3218        let restore = make_restore(source, None);
3219        let name = super::source_prep_job_name(&restore);
3220        assert_eq!(name, "test-restore-source-prep");
3221    }
3222
3223    #[test]
3224    fn break_glass_condition_is_recorded_in_status() {
3225        let mut status = KanidmRestoreStatus::default();
3226        let condition = super::restore_condition(
3227            &status.conditions,
3228            CONDITION_BREAK_GLASS,
3229            CONDITION_TRUE,
3230            "BreakGlassOverride",
3231            "Safety backup skipped",
3232            Some(1),
3233        );
3234        status.conditions.push(condition);
3235        assert!(status.conditions.iter().any(|c| {
3236            c.type_ == CONDITION_BREAK_GLASS
3237                && c.status == CONDITION_TRUE
3238                && c.reason == "BreakGlassOverride"
3239        }));
3240    }
3241
3242    #[test]
3243    fn replica_count_entry_serialization() {
3244        let entry = ReplicaCountEntry {
3245            group: "primary".to_string(),
3246            replicas: 3,
3247        };
3248        let json = serde_json::to_string(&entry).unwrap();
3249        assert!(json.contains("\"group\":\"primary\""));
3250        assert!(json.contains("\"replicas\":3"));
3251    }
3252
3253    #[test]
3254    fn local_source_does_not_require_repository_ref() {
3255        let source = KanidmRestoreSource {
3256            local: Some(super::KanidmRestoreLocalSource {
3257                file_name: "backup.json".to_string(),
3258            }),
3259            backup_ref: None,
3260        };
3261        let safety = SafetyBackupConfig {
3262            repository_ref: None,
3263            skip: false,
3264        };
3265        let restore = make_restore(source, Some(safety));
3266        assert!(validate_safety_backup_config(&restore).is_ok());
3267    }
3268
3269    #[test]
3270    fn skip_without_break_glass_annotations_is_rejected() {
3271        let source = KanidmRestoreSource {
3272            local: Some(super::KanidmRestoreLocalSource {
3273                file_name: "backup.json".to_string(),
3274            }),
3275            backup_ref: None,
3276        };
3277        let safety = SafetyBackupConfig {
3278            repository_ref: None,
3279            skip: true,
3280        };
3281        let restore = make_restore(source, Some(safety));
3282        assert!(validate_safety_backup_config(&restore).is_err());
3283    }
3284
3285    #[test]
3286    fn has_accepted_condition_true() {
3287        let conditions = vec![Condition {
3288            type_: "Ready".to_string(),
3289            status: "True".to_string(),
3290            reason: "Accepted".to_string(),
3291            message: String::new(),
3292            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3293                k8s_openapi::jiff::Timestamp::now(),
3294            ),
3295            observed_generation: None,
3296        }];
3297        assert!(has_accepted_condition(&conditions));
3298    }
3299
3300    #[test]
3301    fn has_accepted_condition_false_when_not_accepted_reason() {
3302        let conditions = vec![Condition {
3303            type_: "Ready".to_string(),
3304            status: "True".to_string(),
3305            reason: "SomeOtherReason".to_string(),
3306            message: String::new(),
3307            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3308                k8s_openapi::jiff::Timestamp::now(),
3309            ),
3310            observed_generation: None,
3311        }];
3312        assert!(!has_accepted_condition(&conditions));
3313    }
3314
3315    #[test]
3316    fn has_accepted_condition_false_when_status_false() {
3317        let conditions = vec![Condition {
3318            type_: "Ready".to_string(),
3319            status: "False".to_string(),
3320            reason: "Accepted".to_string(),
3321            message: String::new(),
3322            last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3323                k8s_openapi::jiff::Timestamp::now(),
3324            ),
3325            observed_generation: None,
3326        }];
3327        assert!(!has_accepted_condition(&conditions));
3328    }
3329
3330    #[test]
3331    fn has_accepted_condition_empty() {
3332        assert!(!has_accepted_condition(&[]));
3333    }
3334
3335    #[test]
3336    fn safety_backup_upload_operation_doc_is_valid() {
3337        let source = KanidmRestoreSource {
3338            local: None,
3339            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3340                name: "backup-1".to_string(),
3341            }),
3342        };
3343        let mut restore = make_restore(source, None);
3344        restore.metadata.namespace = Some("test-ns".to_string());
3345        let target = super::super::crd::Kanidm::default();
3346        let doc_str = super::build_safety_upload_operation_doc(
3347            &restore,
3348            &target,
3349            "test-backup-id",
3350            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3351            "my-bucket",
3352            "prod",
3353            "https://s3.example.com",
3354            "us-east-1",
3355            false,
3356            false,
3357            None,
3358            None,
3359        )
3360        .unwrap();
3361        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3362        assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3363        assert_eq!(parsed["kind"], "OperationDocument");
3364        assert_eq!(parsed["operation"], "upload");
3365        assert_eq!(parsed["bucket"], "my-bucket");
3366        assert_eq!(parsed["prefix"], "prod");
3367        assert_eq!(parsed["endpoint"], "https://s3.example.com");
3368        assert_eq!(parsed["region"], "us-east-1");
3369        assert_eq!(parsed["backupId"], "test-backup-id");
3370        assert_eq!(parsed["consistency"], "kanidm-offline");
3371        assert_eq!(parsed["reason"], "restore-safety");
3372        assert_eq!(
3373            parsed["payloadPath"],
3374            format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3375        );
3376        assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3377    }
3378
3379    #[test]
3380    fn download_operation_doc_uses_real_repository_data() {
3381        let source = KanidmRestoreSource {
3382            local: None,
3383            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3384                name: "backup-1".to_string(),
3385            }),
3386        };
3387        let restore = make_restore(source, None);
3388        let target = super::super::crd::Kanidm::default();
3389        let doc_str = super::build_download_operation_doc(
3390            &restore,
3391            &target,
3392            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3393            "019c7c76-f423-7a12-8f41-2bea7588a303",
3394            "real-bucket",
3395            "real-prefix",
3396            "https://real-endpoint.com",
3397            "eu-west-1",
3398            true,
3399            false,
3400            None,
3401            None,
3402        );
3403        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3404        assert_eq!(parsed["bucket"], "real-bucket");
3405        assert_eq!(parsed["prefix"], "real-prefix");
3406        assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3407        assert_eq!(parsed["region"], "eu-west-1");
3408        assert_eq!(parsed["forcePathStyle"], true);
3409        assert_eq!(
3410            parsed["manifestKey"],
3411            "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3412        );
3413        assert_eq!(
3414            parsed["expectedBackupId"],
3415            "019c7c76-f423-7a12-8f41-2bea7588a303"
3416        );
3417    }
3418
3419    #[test]
3420    fn safety_backup_ref_is_set_after_verification() {
3421        let status = KanidmRestoreStatus {
3422            phase: KanidmRestorePhase::SafetyBackup,
3423            safety_backup_job_name: Some("test-safety-backup".to_string()),
3424            safety_backup_ref: Some("safety-test-restore".to_string()),
3425            safety_backup_expected_backup_id: Some(
3426                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3427            ),
3428            safety_backup_manifest_key: Some(
3429                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3430            ),
3431            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3432            ..Default::default()
3433        };
3434        assert!(status.safety_backup_ref.is_some());
3435        assert!(
3436            status
3437                .safety_backup_ref
3438                .as_ref()
3439                .unwrap()
3440                .starts_with("safety-")
3441        );
3442        assert!(status.safety_backup_expected_backup_id.is_some());
3443        assert!(status.safety_backup_manifest_key.is_some());
3444        assert!(status.safety_backup_payload_sha256.is_some());
3445    }
3446
3447    #[test]
3448    fn database_mutation_not_started_without_safety_backup_ref() {
3449        let status = KanidmRestoreStatus {
3450            phase: KanidmRestorePhase::PreparingSource,
3451            safety_backup_ref: None,
3452            database_mutation_started: false,
3453            ..Default::default()
3454        };
3455        assert!(!status.database_mutation_started);
3456        assert!(status.safety_backup_ref.is_none());
3457    }
3458
3459    #[test]
3460    fn pre_boundary_failure_does_not_set_mutation_started() {
3461        let status = KanidmRestoreStatus {
3462            phase: KanidmRestorePhase::Failed,
3463            message: Some(
3464                "safety backup job failed; target restored to original state".to_string(),
3465            ),
3466            database_mutation_started: false,
3467            ..Default::default()
3468        };
3469        assert!(!status.database_mutation_started);
3470    }
3471
3472    #[test]
3473    fn shared_volume_constants_are_distinct() {
3474        assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3475        assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3476    }
3477
3478    #[test]
3479    fn validate_backup_compatibility_matching_versions_passes() {
3480        let source = KanidmRestoreSource {
3481            local: None,
3482            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3483                name: "backup-1".to_string(),
3484            }),
3485        };
3486        let restore = make_restore(source, None);
3487        let target = kanidm_with_version("1.10.4");
3488        let backup = kaniop_backup_core::crd::KanidmBackup {
3489            metadata: kube::api::ObjectMeta::default(),
3490            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3491                backup_id: "id".to_string(),
3492                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3493                    name: "k".to_string(),
3494                    uid: "uid".to_string(),
3495                },
3496                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3497                    name: "repo".to_string(),
3498                },
3499                manifest_key: "key".to_string(),
3500            },
3501            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3502                kanidm_version: Some("1.10.4".to_string()),
3503                ..Default::default()
3504            }),
3505        };
3506        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3507    }
3508
3509    #[test]
3510    fn validate_backup_compatibility_version_mismatch_fails() {
3511        let source = KanidmRestoreSource {
3512            local: None,
3513            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3514                name: "backup-1".to_string(),
3515            }),
3516        };
3517        let restore = make_restore(source, None);
3518        let target = kanidm_with_version("1.11.0");
3519        let backup = kaniop_backup_core::crd::KanidmBackup {
3520            metadata: kube::api::ObjectMeta::default(),
3521            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3522                backup_id: "id".to_string(),
3523                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3524                    name: "k".to_string(),
3525                    uid: "uid".to_string(),
3526                },
3527                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3528                    name: "repo".to_string(),
3529                },
3530                manifest_key: "key".to_string(),
3531            },
3532            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3533                kanidm_version: Some("1.10.4".to_string()),
3534                ..Default::default()
3535            }),
3536        };
3537        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3538    }
3539
3540    #[test]
3541    fn validate_backup_compatibility_no_backup_version_passes() {
3542        let source = KanidmRestoreSource {
3543            local: None,
3544            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3545                name: "backup-1".to_string(),
3546            }),
3547        };
3548        let restore = make_restore(source, None);
3549        let target = super::super::crd::Kanidm::default();
3550        let backup = kaniop_backup_core::crd::KanidmBackup {
3551            metadata: kube::api::ObjectMeta::default(),
3552            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3553                backup_id: "id".to_string(),
3554                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3555                    name: "k".to_string(),
3556                    uid: "uid".to_string(),
3557                },
3558                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3559                    name: "repo".to_string(),
3560                },
3561                manifest_key: "key".to_string(),
3562            },
3563            status: None,
3564        };
3565        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3566    }
3567
3568    #[test]
3569    fn validate_backup_compatibility_image_digest_mismatch_fails() {
3570        let source = KanidmRestoreSource {
3571            local: None,
3572            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3573                name: "backup-1".to_string(),
3574            }),
3575        };
3576        let mut restore = make_restore(source, None);
3577        restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3578        let target = super::super::crd::Kanidm::default();
3579        let backup = kaniop_backup_core::crd::KanidmBackup {
3580            metadata: kube::api::ObjectMeta::default(),
3581            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3582                backup_id: "id".to_string(),
3583                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3584                    name: "k".to_string(),
3585                    uid: "uid".to_string(),
3586                },
3587                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3588                    name: "repo".to_string(),
3589                },
3590                manifest_key: "key".to_string(),
3591            },
3592            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3593                image_digest: Some("sha256:originaldigest".to_string()),
3594                ..Default::default()
3595            }),
3596        };
3597        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3598    }
3599
3600    #[test]
3601    fn validate_backup_compatibility_image_digest_match_passes() {
3602        let source = KanidmRestoreSource {
3603            local: None,
3604            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3605                name: "backup-1".to_string(),
3606            }),
3607        };
3608        let mut restore = make_restore(source, None);
3609        restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3610        let target = super::super::crd::Kanidm::default();
3611        let backup = kaniop_backup_core::crd::KanidmBackup {
3612            metadata: kube::api::ObjectMeta::default(),
3613            spec: kaniop_backup_core::crd::KanidmBackupSpec {
3614                backup_id: "id".to_string(),
3615                kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3616                    name: "k".to_string(),
3617                    uid: "uid".to_string(),
3618                },
3619                repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3620                    name: "repo".to_string(),
3621                },
3622                manifest_key: "key".to_string(),
3623            },
3624            status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3625                image_digest: Some("sha256:abc123".to_string()),
3626                ..Default::default()
3627            }),
3628        };
3629        assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3630    }
3631
3632    #[test]
3633    fn safety_backup_id_is_deterministic() {
3634        let source = KanidmRestoreSource {
3635            local: None,
3636            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3637                name: "backup-1".to_string(),
3638            }),
3639        };
3640        let restore = make_restore(source, None);
3641        let id1 = super::compute_safety_backup_id(&restore);
3642        let id2 = super::compute_safety_backup_id(&restore);
3643        assert_eq!(id1, id2);
3644        assert!(uuid::Uuid::parse_str(&id1).is_ok());
3645    }
3646
3647    #[test]
3648    fn safety_backup_id_differs_for_different_restores() {
3649        let source1 = KanidmRestoreSource {
3650            local: None,
3651            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3652                name: "backup-1".to_string(),
3653            }),
3654        };
3655        let restore1 = make_restore(source1, None);
3656        let mut restore2 = make_restore(
3657            KanidmRestoreSource {
3658                local: None,
3659                backup_ref: Some(super::KanidmRestoreBackupRefSource {
3660                    name: "backup-2".to_string(),
3661                }),
3662            },
3663            None,
3664        );
3665        restore2.metadata.name = Some("different-restore".to_string());
3666        let id1 = super::compute_safety_backup_id(&restore1);
3667        let id2 = super::compute_safety_backup_id(&restore2);
3668        assert_ne!(id1, id2);
3669    }
3670
3671    #[test]
3672    fn safety_manifest_key_is_deterministic() {
3673        let source = KanidmRestoreSource {
3674            local: None,
3675            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3676                name: "backup-1".to_string(),
3677            }),
3678        };
3679        let restore = make_restore(source, None);
3680        let key1 = super::compute_safety_manifest_key(&restore);
3681        let key2 = super::compute_safety_manifest_key(&restore);
3682        assert_eq!(key1, key2);
3683        assert!(key1.contains("manifest.json"));
3684        assert!(key1.starts_with("v1/tenants/"));
3685    }
3686
3687    #[test]
3688    fn safety_backup_id_survives_restart() {
3689        let source = KanidmRestoreSource {
3690            local: None,
3691            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3692                name: "backup-1".to_string(),
3693            }),
3694        };
3695        let mut restore = make_restore(source, None);
3696        let original_id = super::compute_safety_backup_id(&restore);
3697        restore.status = Some(KanidmRestoreStatus {
3698            phase: KanidmRestorePhase::SafetyBackup,
3699            safety_backup_expected_backup_id: Some(original_id.clone()),
3700            ..Default::default()
3701        });
3702        let recomputed_id = super::compute_safety_backup_id(&restore);
3703        assert_eq!(original_id, recomputed_id);
3704    }
3705
3706    #[test]
3707    fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3708        let source = KanidmRestoreSource {
3709            local: None,
3710            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3711                name: "backup-1".to_string(),
3712            }),
3713        };
3714        let restore = make_restore(source, None);
3715        assert!(super::requires_safety_backup(&restore));
3716        let status_without_ref = KanidmRestoreStatus {
3717            phase: KanidmRestorePhase::RestoringPrimary,
3718            safety_backup_ref: None,
3719            database_mutation_started: false,
3720            ..Default::default()
3721        };
3722        assert!(status_without_ref.safety_backup_ref.is_none());
3723        assert!(!status_without_ref.database_mutation_started);
3724    }
3725
3726    #[test]
3727    fn mutation_boundary_allows_progression_when_safety_ref_set() {
3728        let status_with_ref = KanidmRestoreStatus {
3729            phase: KanidmRestorePhase::RestoringPrimary,
3730            safety_backup_ref: Some("safety-test-restore".to_string()),
3731            safety_backup_manifest_key: Some(
3732                "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3733            ),
3734            safety_backup_payload_sha256: Some("abc123".to_string()),
3735            database_mutation_started: false,
3736            ..Default::default()
3737        };
3738        assert!(status_with_ref.safety_backup_ref.is_some());
3739        assert!(!status_with_ref.database_mutation_started);
3740    }
3741
3742    #[test]
3743    fn result_document_verification_rejects_missing_backup_id() {
3744        let result_json = r#"{
3745            "apiVersion": "backup.kaniop.rs/v1alpha1",
3746            "kind": "ResultDocument",
3747            "operation": "upload",
3748            "success": true,
3749            "exitCode": "success",
3750            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3751            "payloadSha256": "abc123"
3752        }"#;
3753        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3754        assert!(result.backup_id.is_none());
3755    }
3756
3757    #[test]
3758    fn result_document_verification_rejects_missing_manifest_key() {
3759        let result_json = r#"{
3760            "apiVersion": "backup.kaniop.rs/v1alpha1",
3761            "kind": "ResultDocument",
3762            "operation": "upload",
3763            "success": true,
3764            "exitCode": "success",
3765            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3766            "payloadSha256": "abc123"
3767        }"#;
3768        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3769        assert!(result.manifest_key.is_none());
3770    }
3771
3772    #[test]
3773    fn result_document_verification_rejects_missing_payload_sha256() {
3774        let result_json = r#"{
3775            "apiVersion": "backup.kaniop.rs/v1alpha1",
3776            "kind": "ResultDocument",
3777            "operation": "upload",
3778            "success": true,
3779            "exitCode": "success",
3780            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3781            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3782        }"#;
3783        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3784        assert!(result.payload_sha256.is_none());
3785    }
3786
3787    #[test]
3788    fn result_document_verification_rejects_failure_exit_code() {
3789        let result_json = r#"{
3790            "apiVersion": "backup.kaniop.rs/v1alpha1",
3791            "kind": "ResultDocument",
3792            "operation": "upload",
3793            "success": false,
3794            "exitCode": "integrity",
3795            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3796            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3797            "payloadSha256": "abc123"
3798        }"#;
3799        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3800        assert!(!result.success);
3801        assert_eq!(
3802            result.exit_code,
3803            kaniop_backup_core::result::ExitCode::Integrity
3804        );
3805    }
3806
3807    #[test]
3808    fn result_document_verification_rejects_wrong_operation() {
3809        let result_json = r#"{
3810            "apiVersion": "backup.kaniop.rs/v1alpha1",
3811            "kind": "ResultDocument",
3812            "operation": "download",
3813            "success": true,
3814            "exitCode": "success",
3815            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3816            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3817            "payloadSha256": "abc123"
3818        }"#;
3819        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3820        assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3821    }
3822
3823    #[test]
3824    fn result_document_verification_accepts_valid_document() {
3825        let result_json = r#"{
3826            "apiVersion": "backup.kaniop.rs/v1alpha1",
3827            "kind": "ResultDocument",
3828            "operation": "upload",
3829            "success": true,
3830            "exitCode": "success",
3831            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3832            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3833            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3834        }"#;
3835        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3836        assert!(result.success);
3837        assert_eq!(
3838            result.exit_code,
3839            kaniop_backup_core::result::ExitCode::Success
3840        );
3841        assert_eq!(result.operation, "upload");
3842        assert!(result.backup_id.is_some());
3843        assert!(result.manifest_key.is_some());
3844        assert!(result.payload_sha256.is_some());
3845    }
3846
3847    #[test]
3848    fn termination_message_path_constant_is_valid() {
3849        assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3850        assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3851    }
3852
3853    #[test]
3854    fn status_tracks_safety_backup_evidence() {
3855        let status = KanidmRestoreStatus {
3856            phase: KanidmRestorePhase::PreparingSource,
3857            safety_backup_ref: Some("safety-test-restore".to_string()),
3858            safety_backup_expected_backup_id: Some(
3859                "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3860            ),
3861            safety_backup_manifest_key: Some(
3862                "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3863            ),
3864            safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3865            ..Default::default()
3866        };
3867        assert!(status.safety_backup_ref.is_some());
3868        assert!(status.safety_backup_expected_backup_id.is_some());
3869        assert!(status.safety_backup_manifest_key.is_some());
3870        assert!(status.safety_backup_payload_sha256.is_some());
3871    }
3872
3873    #[test]
3874    fn fail_closed_semantics_preserved_on_result_parse_failure() {
3875        let invalid_json = "not valid json";
3876        let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3877        assert!(result.is_err());
3878    }
3879
3880    #[test]
3881    fn fail_closed_semantics_preserved_on_oversized_result() {
3882        let oversized_json = format!(
3883            "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3884            kaniop_backup_core::result::RESULT_DOC_VERSION,
3885            "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3886        );
3887        let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3888        assert!(result.is_err());
3889    }
3890
3891    #[test]
3892    fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3893        let source = KanidmRestoreSource {
3894            local: None,
3895            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3896                name: "backup-1".to_string(),
3897            }),
3898        };
3899        let mut restore = make_restore(source, None);
3900        restore.metadata.namespace = Some("test-ns".to_string());
3901        let target = super::super::crd::Kanidm::default();
3902        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3903        let doc_str = super::build_safety_upload_operation_doc(
3904            &restore,
3905            &target,
3906            "test-backup-id",
3907            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3908            "my-bucket",
3909            "prod",
3910            "https://s3.example.com",
3911            "us-east-1",
3912            false,
3913            false,
3914            Some(&ca_path),
3915            None,
3916        )
3917        .unwrap();
3918        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3919        assert_eq!(parsed["caBundlePath"], ca_path);
3920    }
3921
3922    #[test]
3923    fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3924        let source = KanidmRestoreSource {
3925            local: None,
3926            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3927                name: "backup-1".to_string(),
3928            }),
3929        };
3930        let mut restore = make_restore(source, None);
3931        restore.metadata.namespace = Some("test-ns".to_string());
3932        let target = super::super::crd::Kanidm::default();
3933        let doc_str = super::build_safety_upload_operation_doc(
3934            &restore,
3935            &target,
3936            "test-backup-id",
3937            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3938            "my-bucket",
3939            "prod",
3940            "https://s3.example.com",
3941            "us-east-1",
3942            false,
3943            false,
3944            None,
3945            None,
3946        )
3947        .unwrap();
3948        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3949        assert!(parsed["caBundlePath"].is_null());
3950    }
3951
3952    #[test]
3953    fn download_operation_doc_includes_ca_bundle_path_when_set() {
3954        let source = KanidmRestoreSource {
3955            local: None,
3956            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3957                name: "backup-1".to_string(),
3958            }),
3959        };
3960        let restore = make_restore(source, None);
3961        let target = super::super::crd::Kanidm::default();
3962        let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3963        let doc_str = super::build_download_operation_doc(
3964            &restore,
3965            &target,
3966            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3967            "019c7c76-f423-7a12-8f41-2bea7588a303",
3968            "real-bucket",
3969            "real-prefix",
3970            "https://real-endpoint.com",
3971            "eu-west-1",
3972            true,
3973            false,
3974            Some(&ca_path),
3975            None,
3976        );
3977        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3978        assert_eq!(parsed["caBundlePath"], ca_path);
3979    }
3980
3981    #[test]
3982    fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3983        let source = KanidmRestoreSource {
3984            local: None,
3985            backup_ref: Some(super::KanidmRestoreBackupRefSource {
3986                name: "backup-1".to_string(),
3987            }),
3988        };
3989        let restore = make_restore(source, None);
3990        let target = super::super::crd::Kanidm::default();
3991        let doc_str = super::build_download_operation_doc(
3992            &restore,
3993            &target,
3994            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3995            "019c7c76-f423-7a12-8f41-2bea7588a303",
3996            "real-bucket",
3997            "real-prefix",
3998            "https://real-endpoint.com",
3999            "eu-west-1",
4000            true,
4001            false,
4002            None,
4003            None,
4004        );
4005        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4006        assert!(parsed["caBundlePath"].is_null());
4007    }
4008
4009    #[test]
4010    fn source_prep_result_operation_is_download() {
4011        assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4012        assert_ne!(
4013            super::SOURCE_PREP_RESULT_OPERATION,
4014            super::SAFETY_BACKUP_RESULT_OPERATION
4015        );
4016    }
4017
4018    #[test]
4019    fn download_result_document_verification_accepts_valid_document() {
4020        let result_json = r#"{
4021            "apiVersion": "backup.kaniop.rs/v1alpha1",
4022            "kind": "ResultDocument",
4023            "operation": "download",
4024            "success": true,
4025            "exitCode": "success",
4026            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4027            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4028            "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4029        }"#;
4030        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4031        assert!(result.success);
4032        assert_eq!(
4033            result.exit_code,
4034            kaniop_backup_core::result::ExitCode::Success
4035        );
4036        assert_eq!(result.operation, "download");
4037        assert_eq!(
4038            result.backup_id.as_deref(),
4039            Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4040        );
4041        assert!(result.manifest_key.is_some());
4042        assert!(result.payload_sha256.is_some());
4043    }
4044
4045    #[test]
4046    fn download_result_document_rejects_wrong_operation() {
4047        let result_json = r#"{
4048            "apiVersion": "backup.kaniop.rs/v1alpha1",
4049            "kind": "ResultDocument",
4050            "operation": "upload",
4051            "success": true,
4052            "exitCode": "success",
4053            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4054            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4055            "payloadSha256": "abc123"
4056        }"#;
4057        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4058        assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4059    }
4060
4061    #[test]
4062    fn download_result_document_rejects_failure_success_field() {
4063        let result_json = r#"{
4064            "apiVersion": "backup.kaniop.rs/v1alpha1",
4065            "kind": "ResultDocument",
4066            "operation": "download",
4067            "success": false,
4068            "exitCode": "retryable",
4069            "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4070            "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4071            "payloadSha256": "abc123"
4072        }"#;
4073        let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4074        assert!(!result.success);
4075        assert_ne!(
4076            result.exit_code,
4077            kaniop_backup_core::result::ExitCode::Success
4078        );
4079    }
4080
4081    #[test]
4082    fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4083        let status = KanidmRestoreStatus {
4084            phase: KanidmRestorePhase::PreparingSource,
4085            database_mutation_started: false,
4086            ..Default::default()
4087        };
4088        assert!(!status.database_mutation_started);
4089        assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4090    }
4091
4092    #[test]
4093    fn backup_job_volume_size_default_is_10gi() {
4094        let qty = crate::controller::backup_job_volume_size();
4095        assert_eq!(qty, Quantity("10Gi".to_string()));
4096    }
4097
4098    #[test]
4099    fn encryption_env_vars_present_when_client_side_key_ref_set() {
4100        use kaniop_backup_core::auth::build_encryption_env_vars;
4101        use kaniop_backup_core::crd::SecretRef;
4102
4103        let key_ref = SecretRef {
4104            name: "kek-secret".to_string(),
4105        };
4106        let env_vars = build_encryption_env_vars(Some(&key_ref));
4107        assert_eq!(env_vars.len(), 1);
4108        assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4109        let secret_ref = env_vars[0]
4110            .value_from
4111            .as_ref()
4112            .unwrap()
4113            .secret_key_ref
4114            .as_ref()
4115            .unwrap();
4116        assert_eq!(secret_ref.name, "kek-secret");
4117        assert_eq!(secret_ref.key, "encryption-key");
4118    }
4119
4120    #[test]
4121    fn encryption_env_vars_absent_when_no_key_ref() {
4122        use kaniop_backup_core::auth::build_encryption_env_vars;
4123
4124        let env_vars = build_encryption_env_vars(None);
4125        assert!(
4126            env_vars.is_empty(),
4127            "encryption env vars must be empty when key_ref is None"
4128        );
4129    }
4130
4131    #[test]
4132    fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4133        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4134
4135        let source = KanidmRestoreSource {
4136            local: None,
4137            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4138                name: "backup-1".to_string(),
4139            }),
4140        };
4141        let mut restore = make_restore(source, None);
4142        restore.metadata.namespace = Some("test-ns".to_string());
4143        let target = super::super::crd::Kanidm::default();
4144        let encryption = RepositoryEncryption {
4145            mode: EncryptionMode::ClientSide,
4146            key_id: None,
4147            key_ref: Some(SecretRef {
4148                name: "kek-secret".to_string(),
4149            }),
4150        };
4151        let doc_str = super::build_safety_upload_operation_doc(
4152            &restore,
4153            &target,
4154            "test-backup-id",
4155            "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4156            "my-bucket",
4157            "prod",
4158            "https://s3.example.com",
4159            "us-east-1",
4160            false,
4161            false,
4162            None,
4163            Some(&encryption),
4164        )
4165        .unwrap();
4166        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4167        assert_eq!(parsed["encryptionMode"], "clientSide");
4168        assert!(parsed["encryptionKeyId"].is_null());
4169    }
4170
4171    #[test]
4172    fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4173        use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4174
4175        let source = KanidmRestoreSource {
4176            local: None,
4177            backup_ref: Some(super::KanidmRestoreBackupRefSource {
4178                name: "backup-1".to_string(),
4179            }),
4180        };
4181        let restore = make_restore(source, None);
4182        let target = super::super::crd::Kanidm::default();
4183        let encryption = RepositoryEncryption {
4184            mode: EncryptionMode::ProviderKms,
4185            key_id: Some("alias/kaniop-backups".to_string()),
4186            key_ref: None,
4187        };
4188        let doc_str = super::build_download_operation_doc(
4189            &restore,
4190            &target,
4191            "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4192            "019c7c76-f423-7a12-8f41-2bea7588a303",
4193            "real-bucket",
4194            "real-prefix",
4195            "https://real-endpoint.com",
4196            "eu-west-1",
4197            true,
4198            false,
4199            None,
4200            Some(&encryption),
4201        );
4202        let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4203        assert_eq!(parsed["encryptionMode"], "providerKms");
4204        assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4205    }
4206}