1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78 feature = "schemars",
79 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82 category = "kaniop",
83 group = "kaniop.rs",
84 version = "v1beta1",
85 kind = "KanidmRestore",
86 plural = "kanidmrestores",
87 singular = "kanidmrestore",
88 shortname = "idmrestore",
89 namespaced,
90 status = "KanidmRestoreStatus",
91 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97 pub target_ref: KanidmRestoreTargetRef,
98 pub source: KanidmRestoreSource,
99 pub restore_image: String,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108 pub name: String,
109 pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115 feature = "schemars",
116 schemars(extend("x-kubernetes-validations" = [
117 {
118 "message": "exactly one of local or backupRef must be set",
119 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120 }
121 ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125 #[serde(skip_serializing_if = "Option::is_none")]
127 pub local: Option<KanidmRestoreLocalSource>,
128 #[serde(skip_serializing_if = "Option::is_none")]
130 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137 pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145 pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153 #[serde(skip_serializing_if = "Option::is_none")]
155 pub repository_ref: Option<SafetyBackupRepositoryRef>,
156 #[serde(default)]
158 pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165 pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171 #[default]
172 Pending,
173 Validating,
174 Quiescing,
175 SafetyBackup,
176 PreparingSource,
177 RestoringPrimary,
178 Verifying,
179 RebuildingReplicas,
180 Resuming,
181 Completed,
182 Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_generation: Option<i64>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_target_uid: Option<String>,
193 #[serde(default)]
194 pub phase: KanidmRestorePhase,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub restore_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub verify_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub safety_backup_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub source_prep_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_ref: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_expected_backup_id: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_manifest_key: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_payload_sha256: Option<String>,
211 #[serde(default)]
212 pub replicas_cleared: bool,
213 #[serde(default)]
214 pub certificates_cleared: bool,
215 #[serde(default)]
216 pub database_mutation_started: bool,
217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
218 pub original_replicas: Vec<ReplicaCountEntry>,
219 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220 pub phase_timestamps: std::collections::BTreeMap<String, String>,
221 #[serde(default, skip_serializing_if = "Vec::is_empty")]
222 #[cfg_attr(
223 feature = "schemars",
224 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225 )]
226 pub conditions: Vec<Condition>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235 pub group: String,
236 pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241 attempts: Counter<u64>,
242 outcomes: Counter<u64>,
243 duration_seconds: Histogram<f64>,
244 start_timestamp_seconds: Gauge<i64>,
245 break_glass_total: Counter<u64>,
246 disaster_recovery_total: Counter<u64>,
247 safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251 fn new() -> Self {
252 let meter = global::meter("kaniop");
253 Self {
254 attempts: meter
255 .u64_counter("restore_attempts")
256 .with_description("Number of Kanidm restore attempts started")
257 .build(),
258 outcomes: meter
259 .u64_counter("restore_outcomes")
260 .with_description("Number of terminal Kanidm restore outcomes")
261 .build(),
262 duration_seconds: meter
263 .f64_histogram("restore_duration_seconds")
264 .with_description("Kanidm restore duration from object creation to terminal phase")
265 .with_unit("s")
266 .build(),
267 start_timestamp_seconds: meter
268 .i64_gauge("restore_start_timestamp_seconds")
269 .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270 .with_unit("s")
271 .build(),
272 break_glass_total: meter
273 .u64_counter("restore_break_glass")
274 .with_description("Number of break-glass safety backup overrides")
275 .build(),
276 disaster_recovery_total: meter
277 .u64_counter("restore_disaster_recovery")
278 .with_description("Number of approved disaster-recovery source identity overrides")
279 .build(),
280 safety_backup_duration_seconds: meter
281 .f64_histogram("restore_safety_backup_duration_seconds")
282 .with_description("Duration of pre-restore safety backup creation")
283 .with_unit("s")
284 .build(),
285 }
286 }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290 let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291 let active = restore.metadata.deletion_timestamp.is_none()
292 && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293 let timestamp = if active {
294 restore
295 .metadata
296 .creation_timestamp
297 .as_ref()
298 .map(|created| created.0.as_second())
299 .unwrap_or_else(|| Timestamp::now().as_second())
300 } else {
301 0
302 };
303 let attributes = [
304 KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305 KeyValue::new("restore", restore.name_any()),
306 ];
307 metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312 client: Client,
313 recorder: Recorder,
314 metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319 match phase {
320 KanidmRestorePhase::Pending => {
321 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322 Ok(Action::requeue(REQUEUE))
323 }
324 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325 Ok((target, disaster_recovery_source)) => {
326 mark_restoring(&restore, &target, &ctx).await?;
327 if let Some(source) = disaster_recovery_source.as_ref() {
328 record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329 }
330 let mut status = restore.status.clone().unwrap_or_default();
331 if let Some(source) = disaster_recovery_source.as_ref() {
332 let target_uid = target.uid().unwrap_or_default();
333 let condition = restore_condition(
334 &status.conditions,
335 CONDITION_DISASTER_RECOVERY,
336 CONDITION_TRUE,
337 "ApprovedSourceIdentityMismatch",
338 &format!(
339 "Approved disaster recovery from source {}/{} into target {}/{}",
340 source.name,
341 source.uid,
342 target.name_any(),
343 target_uid
344 ),
345 restore.metadata.generation,
346 );
347 status
348 .conditions
349 .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350 status.conditions.push(condition);
351 }
352 status.observed_target_uid = target.uid();
353 status.original_replicas = capture_original_replicas(&target);
354 status.phase = KanidmRestorePhase::Quiescing;
355 status.message = None;
356 patch_status(&restore, &ctx, status).await?;
357 Ok(Action::requeue(REQUEUE))
358 }
359 Err(error) => {
360 set_phase(
361 &restore,
362 &ctx,
363 KanidmRestorePhase::Failed,
364 Some(error.to_string()),
365 )
366 .await?;
367 Ok(Action::requeue(Duration::from_secs(300)))
368 }
369 },
370 KanidmRestorePhase::Quiescing => {
371 let target = get_target(&restore, &ctx).await?;
372 scale_all(&target, &ctx, 0).await?;
373 if target_pods_stopped(&target, &ctx).await?
374 && target_volumes_detached(&target, &ctx).await?
375 {
376 let next_phase = if requires_safety_backup(&restore) {
377 KanidmRestorePhase::SafetyBackup
378 } else {
379 record_break_glass(&restore, &ctx).await?;
380 KanidmRestorePhase::PreparingSource
381 };
382 if next_phase == KanidmRestorePhase::PreparingSource {
383 let refreshed = Api::<KanidmRestore>::namespaced(
384 ctx.client.clone(),
385 restore.namespace().as_deref().unwrap_or_default(),
386 )
387 .get(&restore.name_any())
388 .await
389 .map_err(|error| {
390 Error::kube_error(
391 "get",
392 "KanidmRestore",
393 restore.namespace().as_deref().unwrap_or_default(),
394 restore.name_any(),
395 error,
396 )
397 })?;
398 set_phase(&refreshed, &ctx, next_phase, None).await?;
399 } else {
400 set_phase(&restore, &ctx, next_phase, None).await?;
401 }
402 }
403 Ok(Action::requeue(REQUEUE))
404 }
405 KanidmRestorePhase::SafetyBackup => {
406 let target = get_target(&restore, &ctx).await?;
407 let name = safety_backup_job_name(&restore);
408 let expected_backup_id = compute_safety_backup_id(&restore)?;
409 let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410 ensure_safety_backup_job(
411 &restore,
412 &target,
413 &ctx,
414 &name,
415 &expected_backup_id,
416 &expected_manifest_key,
417 )
418 .await?;
419 let mut status = restore.status.clone().unwrap_or_default();
420 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421 match job_state(&restore, &ctx, &name).await? {
422 JobState::Complete => {
423 let result =
424 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425 match result {
426 Ok(verified) => {
427 status.safety_backup_job_name = Some(name);
428 status.safety_backup_ref =
429 Some(format!("safety-{}", restore.name_any()));
430 status.safety_backup_manifest_key = Some(verified.manifest_key);
431 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432 record_safety_backup_duration(&restore, &ctx).await;
433 status.phase = KanidmRestorePhase::PreparingSource;
434 status.message = None;
435 patch_status(&restore, &ctx, status).await?;
436 }
437 Err(error) => {
438 resume_before_mutation(&restore, &ctx).await?;
439 set_phase(
440 &restore,
441 &ctx,
442 KanidmRestorePhase::Failed,
443 Some(format!("safety backup result verification failed: {error}")),
444 )
445 .await?;
446 }
447 }
448 }
449 JobState::Failed => {
450 resume_before_mutation(&restore, &ctx).await?;
451 set_phase(
452 &restore,
453 &ctx,
454 KanidmRestorePhase::Failed,
455 Some(
456 "safety backup job failed; target restored to original state"
457 .to_string(),
458 ),
459 )
460 .await?;
461 }
462 JobState::Running => {
463 patch_status(&restore, &ctx, status).await?;
464 }
465 }
466 Ok(Action::requeue(REQUEUE))
467 }
468 KanidmRestorePhase::PreparingSource => {
469 if is_remote_source(&restore) {
470 let target = get_target(&restore, &ctx).await?;
471 let name = source_prep_job_name(&restore);
472 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473 match job_state(&restore, &ctx, &name).await? {
474 JobState::Complete => {
475 let backup_id = restore
476 .spec
477 .source
478 .backup_ref
479 .as_ref()
480 .map(|b| b.name.clone())
481 .unwrap_or_default();
482 let backup = {
483 let ns = restore.namespace().unwrap();
484 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485 .get(&backup_id)
486 .await
487 .map_err(|e| {
488 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489 })?
490 };
491 let expected_backup_id = &backup.spec.backup_id;
492 let result =
493 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494 .await;
495 match result {
496 Ok(verified) => {
497 if let Some(expected_sha256) = backup
498 .status
499 .as_ref()
500 .and_then(|s| s.payload_sha256.as_deref())
501 {
502 if verified.payload_sha256 != expected_sha256 {
503 resume_before_mutation(&restore, &ctx).await?;
504 set_phase(
505 &restore,
506 &ctx,
507 KanidmRestorePhase::Failed,
508 Some(format!(
509 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510 verified.payload_sha256
511 )),
512 )
513 .await?;
514 return Ok(Action::requeue(REQUEUE));
515 }
516 }
517 let mut status = restore.status.clone().unwrap_or_default();
518 status.source_prep_job_name = Some(name);
519 status.phase = KanidmRestorePhase::RestoringPrimary;
520 status.message = None;
521 patch_status(&restore, &ctx, status).await?;
522 }
523 Err(error) => {
524 resume_before_mutation(&restore, &ctx).await?;
525 set_phase(
526 &restore,
527 &ctx,
528 KanidmRestorePhase::Failed,
529 Some(format!(
530 "source preparation result verification failed: {error}"
531 )),
532 )
533 .await?;
534 }
535 }
536 }
537 JobState::Failed => {
538 resume_before_mutation(&restore, &ctx).await?;
539 set_phase(
540 &restore,
541 &ctx,
542 KanidmRestorePhase::Failed,
543 Some(
544 "source preparation job failed; target restored to original state"
545 .to_string(),
546 ),
547 )
548 .await?;
549 }
550 JobState::Running => {}
551 }
552 } else {
553 let target = get_target(&restore, &ctx).await?;
554 let name = source_check_job_name(&restore);
555 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556 match job_state(&restore, &ctx, &name).await? {
557 JobState::Complete => {
558 let mut status = restore.status.clone().unwrap_or_default();
559 status.source_prep_job_name = Some(name);
560 status.phase = KanidmRestorePhase::RestoringPrimary;
561 status.message = None;
562 patch_status(&restore, &ctx, status).await?;
563 }
564 JobState::Failed => {
565 resume_before_mutation(&restore, &ctx).await?;
566 set_phase(
567 &restore,
568 &ctx,
569 KanidmRestorePhase::Failed,
570 Some(
571 "local backup file check failed; target restored to original state"
572 .to_string(),
573 ),
574 )
575 .await?;
576 }
577 JobState::Running => {}
578 }
579 }
580 Ok(Action::requeue(REQUEUE))
581 }
582 KanidmRestorePhase::RestoringPrimary => {
583 let target = get_target(&restore, &ctx).await?;
584 let status = restore.status.as_ref().cloned().unwrap_or_default();
585 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586 return Err(Error::MissingData(
587 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588 ));
589 }
590 if !status.database_mutation_started {
591 let mut status = status;
592 status.database_mutation_started = true;
593 status.message = Some(
594 "database mutation boundary persisted; starting primary restore".to_string(),
595 );
596 patch_status(&restore, &ctx, status).await?;
597 return Ok(Action::requeue(REQUEUE));
598 }
599 let name = restore_job_name(&restore);
600 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601 match job_state(&restore, &ctx, &name).await? {
602 JobState::Complete => {
603 let mut status = restore.status.clone().unwrap_or_default();
604 status.restore_job_name = Some(name);
605 status.phase = KanidmRestorePhase::Verifying;
606 status.message = None;
607 patch_status(&restore, &ctx, status).await?;
608 }
609 JobState::Failed => {
610 let status = restore.status.clone().unwrap_or_default();
611 let failed_status = restore_primary_failed_status(&status, &name);
612 patch_status(&restore, &ctx, failed_status).await?;
613 }
614 JobState::Running => {}
615 }
616 Ok(Action::requeue(REQUEUE))
617 }
618 KanidmRestorePhase::Verifying => {
619 let target = get_target(&restore, &ctx).await?;
620 let name = verify_job_name(&restore);
621 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622 match job_state(&restore, &ctx, &name).await? {
623 JobState::Complete => {
624 let mut status = restore.status.clone().unwrap_or_default();
625 status.verify_job_name = Some(name);
626 status.phase = KanidmRestorePhase::RebuildingReplicas;
627 status.message = None;
628 patch_status(&restore, &ctx, status).await?;
629 }
630 JobState::Failed => {
631 let status = restore.status.clone().unwrap_or_default();
632 let failed_status = verifying_failed_status(&status, &name);
633 patch_status(&restore, &ctx, failed_status).await?;
634 }
635 JobState::Running => {}
636 }
637 Ok(Action::requeue(REQUEUE))
638 }
639 KanidmRestorePhase::RebuildingReplicas => {
640 let target = get_target(&restore, &ctx).await?;
641 let mut status = restore.status.clone().unwrap_or_default();
642 if !status.replicas_cleared {
643 if delete_secondary_pvcs(&target, &ctx).await? {
644 status.replicas_cleared = true;
645 status.message = Some("secondary database state cleared".to_string());
646 patch_status(&restore, &ctx, status).await?;
647 }
648 return Ok(Action::requeue(REQUEUE));
649 }
650
651 scale_primary(&target, &ctx, 1).await?;
652 if !primary_ready(&target, &ctx).await? {
653 return Ok(Action::requeue(REQUEUE));
654 }
655 if !status.certificates_cleared {
656 delete_replica_cert_secrets(&target, &ctx).await?;
657 delete_admin_secret(&target, &ctx).await?;
658 status.certificates_cleared = true;
659 status.message = Some(
660 "replica certificates and admin secret cleared for regeneration".to_string(),
661 );
662 patch_status(&restore, &ctx, status).await?;
663 return Ok(Action::requeue(REQUEUE));
664 }
665 status.phase = KanidmRestorePhase::Resuming;
666 status.message = None;
667 patch_status(&restore, &ctx, status).await?;
668 Ok(Action::requeue(REQUEUE))
669 }
670 KanidmRestorePhase::Resuming => {
671 let target = get_target(&restore, &ctx).await?;
672 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673 clear_restoring(&restore, &target, &ctx).await?;
674 Ok(Action::requeue(Duration::from_secs(3600)))
675 }
676 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677 Ok(Action::requeue(Duration::from_secs(3600)))
678 }
679 }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684 name: String,
685 uid: String,
686}
687
688async fn validate(
689 restore: &KanidmRestore,
690 ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692 let target = get_target(restore, ctx).await?;
693 let actual_uid = target
694 .uid()
695 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696 if actual_uid != restore.spec.target_ref.uid {
697 return Err(Error::MissingData(format!(
698 "target UID mismatch: expected {}, got {}",
699 restore.spec.target_ref.uid, actual_uid
700 )));
701 }
702 validate_source(restore)?;
703 let disaster_recovery_source = if is_remote_source(restore) {
704 validate_backup_ref(restore, &target, ctx).await?
705 } else {
706 None
707 };
708 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709 {
710 return Err(Error::MissingData(format!(
711 "restoreImage must be the target's pinned Kanidm image (target image is {})",
712 target.spec.image
713 )));
714 }
715 let storage = target
716 .spec
717 .storage
718 .as_ref()
719 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720 if storage.empty_dir.is_some()
721 || storage.ephemeral.is_some()
722 || storage.volume_claim_template.is_none()
723 {
724 return Err(Error::MissingData(
725 "restore requires PVC-backed Kanidm storage".to_string(),
726 ));
727 }
728 let primaries = target
729 .spec
730 .replica_groups
731 .iter()
732 .filter(|rg| rg.primary_node)
733 .count();
734 if primaries != 1 {
735 return Err(Error::MissingData(
736 "backup/restore requires exactly one primary replica group".to_string(),
737 ));
738 }
739 let ns = restore.namespace().unwrap();
740 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741 .list(&ListParams::default())
742 .await
743 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744 if restores.items.iter().any(|other| {
745 other.name_any() != restore.name_any()
746 && other.spec.target_ref.name == restore.spec.target_ref.name
747 && !matches!(
748 other.status.as_ref().map(|s| s.phase),
749 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750 )
751 }) {
752 return Err(Error::MissingData(
753 "another active restore targets this Kanidm".to_string(),
754 ));
755 }
756 validate_safety_backup_config(restore)?;
757 Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761 !name.is_empty()
762 && name != "."
763 && name != ".."
764 && !name.contains('/')
765 && !name.contains('\\')
766 && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770 image == "kanidm/server:latest"
771 || image.ends_with(":latest")
772 || (!image.contains('@')
773 && !image
774 .rsplit('/')
775 .next()
776 .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780 let ns = restore
781 .namespace()
782 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784 .get(&restore.spec.target_ref.name)
785 .await
786 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790 restore: &KanidmRestore,
791 target: &Kanidm,
792 ctx: &RestoreContext,
793) -> Result<()> {
794 let ns = target.namespace().unwrap();
795 let uid = restore
796 .uid()
797 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799 .patch(
800 &target.name_any(),
801 &PatchParams::default(),
802 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803 )
804 .await
805 .map(|_| ())
806 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810 restore: &KanidmRestore,
811 target: &Kanidm,
812 ctx: &RestoreContext,
813) -> Result<()> {
814 let ns = target.namespace().unwrap();
815 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816 return Ok(());
817 }
818 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819 .patch(
820 &target.name_any(),
821 &PatchParams::default(),
822 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823 )
824 .await
825 .map(|_| ())
826 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830 restore: &KanidmRestore,
831 ctx: &RestoreContext,
832 mut status: KanidmRestoreStatus,
833) -> Result<()> {
834 let ns = restore.namespace().unwrap();
835 let previous_phase = restore.status.as_ref().map(|current| current.phase);
836 let next_phase = status.phase;
837 status.observed_generation = restore.metadata.generation;
838 update_restore_conditions(&mut status, restore.metadata.generation);
839
840 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841 .patch_status(
842 &restore.name_any(),
843 &PatchParams::default(),
844 &Patch::Merge(json!({"status": &status})),
845 )
846 .await
847 .map_err(|e| {
848 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849 })?;
850
851 if previous_phase != Some(next_phase) {
852 record_restore_transition(
853 restore,
854 ctx,
855 previous_phase,
856 next_phase,
857 status.message.as_deref(),
858 )
859 .await;
860 }
861 Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865 let previous = status.conditions.clone();
866 let phase = status.phase;
867 let phase_reason = format!("{phase:?}");
868 let message = status
869 .message
870 .clone()
871 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872 let terminal_success = phase == KanidmRestorePhase::Completed;
873 let terminal_failure = phase == KanidmRestorePhase::Failed;
874 let progressing = !terminal_success && !terminal_failure;
875
876 let sticky_conditions: Vec<Condition> = previous
877 .iter()
878 .filter(|condition| {
879 condition.type_ == CONDITION_BREAK_GLASS
880 || condition.type_ == CONDITION_DISASTER_RECOVERY
881 })
882 .cloned()
883 .collect();
884 status.conditions = vec![
885 restore_condition(
886 &previous,
887 CONDITION_PROGRESSING,
888 if progressing {
889 CONDITION_TRUE
890 } else {
891 CONDITION_FALSE
892 },
893 &phase_reason,
894 &message,
895 generation,
896 ),
897 restore_condition(
898 &previous,
899 CONDITION_READY,
900 if terminal_success {
901 CONDITION_TRUE
902 } else {
903 CONDITION_FALSE
904 },
905 if terminal_success {
906 "RestoreCompleted"
907 } else {
908 "RestoreNotCompleted"
909 },
910 if terminal_success {
911 "Kanidm restore completed successfully."
912 } else {
913 "Kanidm restore has not completed successfully."
914 },
915 generation,
916 ),
917 restore_condition(
918 &previous,
919 CONDITION_FAILED,
920 if terminal_failure {
921 CONDITION_TRUE
922 } else {
923 CONDITION_FALSE
924 },
925 if terminal_failure {
926 "RestoreFailed"
927 } else {
928 "NoRestoreFailure"
929 },
930 if terminal_failure {
931 status
932 .message
933 .as_deref()
934 .unwrap_or("Kanidm restore failed.")
935 } else {
936 "No terminal restore failure has been recorded."
937 },
938 generation,
939 ),
940 ];
941 status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945 previous: &[Condition],
946 condition_type: &str,
947 condition_status: &str,
948 reason: &str,
949 message: &str,
950 generation: Option<i64>,
951) -> Condition {
952 let last_transition_time = previous
953 .iter()
954 .find(|condition| {
955 condition.type_ == condition_type
956 && condition.status == condition_status
957 && condition.reason == reason
958 })
959 .map(|condition| condition.last_transition_time.clone())
960 .unwrap_or_else(|| Time(Timestamp::now()));
961 Condition {
962 type_: condition_type.to_string(),
963 status: condition_status.to_string(),
964 reason: reason.to_string(),
965 message: message.to_string(),
966 last_transition_time,
967 observed_generation: generation,
968 }
969}
970
971async fn record_restore_transition(
972 restore: &KanidmRestore,
973 ctx: &RestoreContext,
974 previous_phase: Option<KanidmRestorePhase>,
975 phase: KanidmRestorePhase,
976 message: Option<&str>,
977) {
978 if phase == KanidmRestorePhase::Validating {
979 ctx.metrics.attempts.add(1, &[]);
980 }
981
982 let result = match phase {
983 KanidmRestorePhase::Completed => Some("success"),
984 KanidmRestorePhase::Failed => Some("failure"),
985 _ => None,
986 };
987 if let Some(result) = result {
988 let attributes = [
989 KeyValue::new("result", result),
990 KeyValue::new("phase", format!("{phase:?}")),
991 ];
992 ctx.metrics.outcomes.add(1, &attributes);
993 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995 ctx.metrics.duration_seconds.record(elapsed, &attributes);
996 }
997 }
998
999 let reason = if phase == KanidmRestorePhase::Failed {
1000 "RestoreFailed"
1001 } else {
1002 "RestorePhaseChanged"
1003 };
1004 let note = message.map(str::to_string).or_else(|| {
1005 Some(format!(
1006 "Kanidm restore phase changed from {} to {phase:?}.",
1007 previous_phase
1008 .map(|previous| format!("{previous:?}"))
1009 .unwrap_or_else(|| "None".to_string())
1010 ))
1011 });
1012 if let Err(error) = ctx
1013 .recorder
1014 .publish(
1015 &Event {
1016 type_: if phase == KanidmRestorePhase::Failed {
1017 EventType::Warning
1018 } else {
1019 EventType::Normal
1020 },
1021 reason: reason.to_string(),
1022 note,
1023 action: "Restore".to_string(),
1024 secondary: None,
1025 },
1026 &restore.object_ref(&()),
1027 )
1028 .await
1029 {
1030 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031 }
1032}
1033
1034async fn set_phase(
1035 restore: &KanidmRestore,
1036 ctx: &RestoreContext,
1037 phase: KanidmRestorePhase,
1038 message: Option<String>,
1039) -> Result<()> {
1040 let mut status = restore.status.clone().unwrap_or_default();
1041 status.phase = phase;
1042 status.message = message;
1043 patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047 restore: &KanidmRestore,
1048 ctx: &RestoreContext,
1049 message: &str,
1050) -> Result<()> {
1051 set_phase(
1052 restore,
1053 ctx,
1054 KanidmRestorePhase::Failed,
1055 Some(message.to_string()),
1056 )
1057 .await
1058}
1059
1060fn restore_primary_failed_status(
1061 existing: &KanidmRestoreStatus,
1062 job_name: &str,
1063) -> KanidmRestoreStatus {
1064 let mut status = existing.clone();
1065 status.restore_job_name = Some(job_name.to_string());
1066 status.phase = KanidmRestorePhase::Failed;
1067 status.message = Some("database restore job failed".to_string());
1068 status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072 let mut status = existing.clone();
1073 status.verify_job_name = Some(job_name.to_string());
1074 status.phase = KanidmRestorePhase::Failed;
1075 status.message = Some("database verification failed".to_string());
1076 status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080 target
1081 .spec
1082 .replica_groups
1083 .iter()
1084 .find(|rg| rg.primary_node)
1085 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089 let ns = target.namespace().unwrap();
1090 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091 for rg in &target.spec.replica_groups {
1092 let name = target.statefulset_name(&rg.name);
1093 api.patch(
1094 &name,
1095 &PatchParams::default(),
1096 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097 )
1098 .await
1099 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100 }
1101 Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105 let ns = target.namespace().unwrap();
1106 let rg = primary_group(target)?;
1107 let name = target.statefulset_name(&rg.name);
1108 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109 .patch(
1110 &name,
1111 &PatchParams::default(),
1112 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113 )
1114 .await
1115 .map(|_| ())
1116 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120 let ns = target.namespace().unwrap();
1121 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122 for rg in &target.spec.replica_groups {
1123 let name = target.statefulset_name(&rg.name);
1124 api.patch(
1125 &name,
1126 &PatchParams::default(),
1127 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128 )
1129 .await
1130 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131 }
1132 Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136 let ns = target.namespace().unwrap();
1137 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139 .await
1140 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141 Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145 let ns = target.namespace().unwrap();
1146 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147 let mut pv_names = BTreeSet::new();
1148 let primary_name = primary_pvc_name(target)?;
1149 let primary_pvc = pvc_api
1150 .get(&primary_name)
1151 .await
1152 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153 let primary_pv = primary_pvc
1154 .spec
1155 .as_ref()
1156 .and_then(|spec| spec.volume_name.clone())
1157 .ok_or_else(|| {
1158 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159 })?;
1160 pv_names.insert(primary_pv);
1161
1162 for rg in &target.spec.replica_groups {
1163 let sts = target.statefulset_name(&rg.name);
1164 for ordinal in 0..rg.replicas {
1165 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166 if name == primary_name {
1167 continue;
1168 }
1169 if let Some(pvc) = pvc_api
1170 .get_opt(&name)
1171 .await
1172 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174 {
1175 pv_names.insert(pv_name);
1176 }
1177 }
1178 }
1179
1180 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181 .list(&ListParams::default())
1182 .await
1183 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184 Ok(!attachments.items.iter().any(|attachment| {
1185 attachment
1186 .spec
1187 .source
1188 .persistent_volume_name
1189 .as_ref()
1190 .is_some_and(|pv_name| pv_names.contains(pv_name))
1191 }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195 let ns = target.namespace().unwrap();
1196 let name = target.statefulset_name(&primary_group(target)?.name);
1197 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198 .get(&name)
1199 .await
1200 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201 Ok(sts
1202 .status
1203 .as_ref()
1204 .and_then(|s| s.ready_replicas)
1205 .unwrap_or(0)
1206 >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210 let rg = primary_group(target)?;
1211 Ok(format!(
1212 "{DATA_VOLUME}-{}-0",
1213 target.statefulset_name(&rg.name)
1214 ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218 let ns = target.namespace().unwrap();
1219 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220 let mut all_absent = true;
1221 for rg in &target.spec.replica_groups {
1222 let sts = target.statefulset_name(&rg.name);
1223 for ordinal in 0..rg.replicas {
1224 if rg.primary_node && ordinal == 0 {
1225 continue;
1226 }
1227 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228 match api
1229 .get_opt(&name)
1230 .await
1231 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232 {
1233 None => {}
1234 Some(pvc) => {
1235 all_absent = false;
1236 if pvc.metadata.deletion_timestamp.is_none() {
1237 match api.delete(&name, &DeleteParams::default()).await {
1238 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239 Err(kube::Error::Api(status)) if status.code == 404 => {}
1240 Err(error) => {
1241 return Err(Error::kube_error(
1242 "delete",
1243 "PersistentVolumeClaim",
1244 &ns,
1245 &name,
1246 error,
1247 ));
1248 }
1249 }
1250 }
1251 }
1252 }
1253 }
1254 }
1255 Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259 let ns = target.namespace().unwrap();
1260 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261 let label_selector = format!(
1262 "{}={},{}={}",
1263 CLUSTER_LABEL,
1264 target.name_any(),
1265 SECRET_TYPE_LABEL,
1266 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267 );
1268 let secrets = api
1269 .list(&ListParams::default().labels(&label_selector))
1270 .await
1271 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272 let mut all_absent = secrets.items.is_empty();
1273 for secret in secrets.items {
1274 let name = secret.name_any();
1275 if secret.metadata.deletion_timestamp.is_none() {
1276 match api.delete(&name, &DeleteParams::default()).await {
1277 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278 Err(kube::Error::Api(status)) if status.code == 404 => {}
1279 Err(error) => {
1280 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281 }
1282 }
1283 all_absent = false;
1284 }
1285 }
1286 Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290 let ns = target.namespace().unwrap();
1291 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292 let name = format!("{}-admin-passwords", target.name_any());
1293 match api
1294 .get_opt(&name)
1295 .await
1296 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297 {
1298 None => Ok(true),
1299 Some(secret) => {
1300 if secret.metadata.deletion_timestamp.is_some() {
1301 return Ok(false);
1302 }
1303 match api.delete(&name, &DeleteParams::default()).await {
1304 Ok(_) => {
1305 debug!(secret = %name, "deleting stale admin secret");
1306 Ok(false)
1307 }
1308 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310 }
1311 }
1312 }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316 format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319 format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323 let origin = target
1324 .spec
1325 .origin
1326 .clone()
1327 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328 vec![
1329 EnvVar {
1330 name: "KANIDM_DB_PATH".to_string(),
1331 value: Some("/data/kanidm.db".to_string()),
1332 ..Default::default()
1333 },
1334 EnvVar {
1335 name: "KANIDM_DOMAIN".to_string(),
1336 value: Some(target.spec.domain.clone()),
1337 ..Default::default()
1338 },
1339 EnvVar {
1340 name: "KANIDM_ORIGIN".to_string(),
1341 value: Some(origin),
1342 ..Default::default()
1343 },
1344 EnvVar {
1345 name: "KANIDM_TLS_CHAIN".to_string(),
1346 value: Some(format!("{TLS_PATH}/tls.crt")),
1347 ..Default::default()
1348 },
1349 EnvVar {
1350 name: "KANIDM_TLS_KEY".to_string(),
1351 value: Some(format!("{TLS_PATH}/tls.key")),
1352 ..Default::default()
1353 },
1354 ]
1355}
1356
1357async fn ensure_database_job(
1358 restore: &KanidmRestore,
1359 target: &Kanidm,
1360 ctx: &RestoreContext,
1361 name: &str,
1362 verify: bool,
1363) -> Result<()> {
1364 let ns = restore.namespace().unwrap();
1365 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366 if api
1367 .get_opt(name)
1368 .await
1369 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370 .is_some()
1371 {
1372 return Ok(());
1373 }
1374 let mut command = vec![
1375 "kanidmd".to_string(),
1376 "database".to_string(),
1377 if verify { "backup" } else { "restore" }.to_string(),
1378 ];
1379 if verify {
1380 command.push("/verify/verification.json.gz".to_string());
1381 } else {
1382 if is_remote_source(restore) {
1383 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384 } else {
1385 let file_name = restore
1386 .spec
1387 .source
1388 .local
1389 .as_ref()
1390 .map(|l| l.file_name.clone())
1391 .unwrap_or_default();
1392 command.push(format!("{BACKUP_PATH}/{file_name}"));
1393 }
1394 }
1395 let job = Job {
1396 metadata: ObjectMeta {
1397 name: Some(name.to_string()),
1398 namespace: Some(ns.clone()),
1399 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400 ..Default::default()
1401 },
1402 spec: Some(JobSpec {
1403 backoff_limit: Some(0),
1404 template: PodTemplateSpec {
1405 metadata: Some(ObjectMeta::default()),
1406 spec: Some(PodSpec {
1407 automount_service_account_token: Some(false),
1408 restart_policy: Some("Never".to_string()),
1409 security_context: Some(source_job_pod_security_context(target)),
1410 containers: vec![Container {
1411 name: if verify { "verify" } else { "restore" }.to_string(),
1412 image: Some(restore.spec.restore_image.clone()),
1413 command: Some(command),
1414 env: Some(build_kanidm_db_env_vars(target)),
1415 security_context: Some(kanidm_job_security_context()),
1416 resources: Some(job_resource_requirements()),
1417 volume_mounts: Some(vec![
1418 VolumeMount {
1419 name: DATA_VOLUME.to_string(),
1420 mount_path: DATA_PATH.to_string(),
1421 ..Default::default()
1422 },
1423 VolumeMount {
1424 name: TLS_VOLUME.to_string(),
1425 mount_path: TLS_PATH.to_string(),
1426 read_only: Some(true),
1427 ..Default::default()
1428 },
1429 VolumeMount {
1430 name: "verification-output".to_string(),
1431 mount_path: "/verify".to_string(),
1432 ..Default::default()
1433 },
1434 ]),
1435 ..Default::default()
1436 }],
1437 volumes: Some(vec![
1438 Volume {
1439 name: DATA_VOLUME.to_string(),
1440 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441 claim_name: primary_pvc_name(target)?,
1442 read_only: Some(false),
1443 }),
1444 ..Default::default()
1445 },
1446 Volume {
1447 name: TLS_VOLUME.to_string(),
1448 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449 secret_name: Some(target.effective_tls_secret_name()),
1450 ..Default::default()
1451 }),
1452 ..Default::default()
1453 },
1454 Volume {
1455 name: "verification-output".to_string(),
1456 empty_dir: Some(Default::default()),
1457 ..Default::default()
1458 },
1459 ]),
1460 ..Default::default()
1461 }),
1462 },
1463 ..Default::default()
1464 }),
1465 ..Default::default()
1466 };
1467 api.create(&PostParams::default(), &job)
1468 .await
1469 .map(|_| ())
1470 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474 Running,
1475 Complete,
1476 Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480 let ns = restore.namespace().unwrap();
1481 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482 .get(name)
1483 .await
1484 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486 return Ok(JobState::Failed);
1487 }
1488 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489 return Ok(JobState::Complete);
1490 }
1491 Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495 restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499 let safety = restore.spec.safety_backup.as_ref();
1500 let skip = safety.is_some_and(|s| s.skip);
1501 !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505 let has_local = restore.spec.source.local.is_some();
1506 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507 if has_local && has_backup_ref {
1508 return Err(Error::MissingData(
1509 "source.local and source.backupRef are mutually exclusive".to_string(),
1510 ));
1511 }
1512 if !has_local && !has_backup_ref {
1513 return Err(Error::MissingData(
1514 "source must specify either local or backupRef".to_string(),
1515 ));
1516 }
1517 if let Some(local) = &restore.spec.source.local {
1518 if !safe_basename(&local.file_name) {
1519 return Err(Error::MissingData(
1520 "restore source fileName must be a safe basename".to_string(),
1521 ));
1522 }
1523 }
1524 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525 if backup_ref.name.is_empty() {
1526 return Err(Error::MissingData(
1527 "source.backupRef.name must not be empty".to_string(),
1528 ));
1529 }
1530 }
1531 Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535 restore
1536 .metadata
1537 .annotations
1538 .as_ref()
1539 .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540 .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544 let annotations = restore.metadata.annotations.as_ref();
1545 let reason = annotations
1546 .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547 .map(String::as_str)
1548 .filter(|value| !value.trim().is_empty());
1549 let approved_by = annotations
1550 .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551 .map(String::as_str)
1552 .filter(|value| !value.trim().is_empty());
1553
1554 if reason.is_none() || approved_by.is_none() {
1555 return Err(Error::MissingData(format!(
1556 "disaster-recovery restore requires non-empty {} and {} annotations",
1557 BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558 )));
1559 }
1560 Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564 restore: &KanidmRestore,
1565 target: &Kanidm,
1566 ctx: &RestoreContext,
1567 source: &DisasterRecoverySource,
1568) {
1569 ctx.metrics.disaster_recovery_total.add(1, &[]);
1570 let reason = restore
1571 .metadata
1572 .annotations
1573 .as_ref()
1574 .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575 .cloned()
1576 .unwrap_or_default();
1577 let approved_by = restore
1578 .metadata
1579 .annotations
1580 .as_ref()
1581 .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582 .cloned()
1583 .unwrap_or_default();
1584 let target_uid = target.uid().unwrap_or_default();
1585 let note = format!(
1586 "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587 source.name,
1588 source.uid,
1589 target.name_any(),
1590 target_uid,
1591 reason,
1592 approved_by
1593 );
1594 if let Err(error) = ctx
1595 .recorder
1596 .publish(
1597 &Event {
1598 type_: EventType::Warning,
1599 reason: "DisasterRecoveryOverride".to_string(),
1600 note: Some(note.clone()),
1601 action: "Restore".to_string(),
1602 secondary: None,
1603 },
1604 &restore.object_ref(&()),
1605 )
1606 .await
1607 {
1608 warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609 }
1610 warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614 let safety = restore.spec.safety_backup.as_ref();
1615 let skip = safety.is_some_and(|s| s.skip);
1616 if skip {
1617 let annotations = restore.metadata.annotations.as_ref();
1618 let reason = annotations
1619 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620 .map(|s| s.trim())
1621 .unwrap_or("");
1622 let approver = annotations
1623 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624 .map(|s| s.trim())
1625 .unwrap_or("");
1626 if reason.is_empty() {
1627 return Err(Error::MissingData(format!(
1628 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629 )));
1630 }
1631 if approver.is_empty() {
1632 return Err(Error::MissingData(format!(
1633 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634 )));
1635 }
1636 }
1637 if !skip {
1638 let has_repo = safety
1639 .and_then(|s| s.repository_ref.as_ref())
1640 .is_some_and(|r| !r.name.is_empty());
1641 if !has_repo {
1642 return Err(Error::MissingData(
1643 "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644 .to_string(),
1645 ));
1646 }
1647 }
1648 Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652 target
1653 .spec
1654 .replica_groups
1655 .iter()
1656 .map(|rg| ReplicaCountEntry {
1657 group: rg.name.clone(),
1658 replicas: rg.replicas,
1659 })
1660 .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664 if let Ok(target) = get_target(restore, ctx).await {
1665 let owns_maintenance =
1666 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667 if owns_maintenance {
1668 scale_desired(&target, ctx).await?;
1669 clear_restoring(restore, &target, ctx).await?;
1670 }
1671 }
1672 Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676 ctx.metrics.break_glass_total.add(1, &[]);
1677 let reason = restore
1678 .metadata
1679 .annotations
1680 .as_ref()
1681 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682 .cloned()
1683 .unwrap_or_default();
1684 let approver = restore
1685 .metadata
1686 .annotations
1687 .as_ref()
1688 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689 .cloned()
1690 .unwrap_or_default();
1691 let mut status = restore.status.clone().unwrap_or_default();
1692 let break_glass_condition = restore_condition(
1693 &status.conditions,
1694 CONDITION_BREAK_GLASS,
1695 CONDITION_TRUE,
1696 "BreakGlassOverride",
1697 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698 restore.metadata.generation,
1699 );
1700 status
1701 .conditions
1702 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703 status.conditions.push(break_glass_condition);
1704 patch_status(restore, ctx, status).await?;
1705 if let Err(error) = ctx
1706 .recorder
1707 .publish(
1708 &Event {
1709 type_: EventType::Warning,
1710 reason: "SafetyBackupSkipped".to_string(),
1711 note: Some(format!(
1712 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713 )),
1714 action: "Restore".to_string(),
1715 secondary: None,
1716 },
1717 &restore.object_ref(&()),
1718 )
1719 .await
1720 {
1721 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722 }
1723 warn!(
1724 restore = %restore.name_any(),
1725 reason = %reason,
1726 approver = %approver,
1727 "break-glass safety backup override activated"
1728 );
1729 Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733 let status = restore.status.as_ref().cloned().unwrap_or_default();
1734 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735 if let Ok(start_ts) = start.parse::<Timestamp>() {
1736 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737 ctx.metrics
1738 .safety_backup_duration_seconds
1739 .record(elapsed, &[]);
1740 }
1741 }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745 format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749 format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753 format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757 restore: &KanidmRestore,
1758 target: &Kanidm,
1759 ctx: &RestoreContext,
1760 name: &str,
1761) -> Result<()> {
1762 let ns = restore.namespace().unwrap();
1763 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764 if api
1765 .get_opt(name)
1766 .await
1767 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768 .is_some()
1769 {
1770 return Ok(());
1771 }
1772 let file_name = restore
1773 .spec
1774 .source
1775 .local
1776 .as_ref()
1777 .map(|l| l.file_name.clone())
1778 .unwrap_or_default();
1779 let operation_doc = serde_json::json!({
1780 "apiVersion": "backup.kaniop.rs/v1alpha1",
1781 "kind": "OperationDocument",
1782 "operation": "check",
1783 "path": format!("{BACKUP_PATH}/{file_name}"),
1784 "resultPath": "/run/kaniop-result/result.json",
1785 "format": "kanidmJsonGzip",
1786 })
1787 .to_string();
1788 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790 let data_mover_image = data_mover_image();
1791 let job = Job {
1792 metadata: ObjectMeta {
1793 name: Some(name.to_string()),
1794 namespace: Some(ns.clone()),
1795 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796 ..Default::default()
1797 },
1798 spec: Some(JobSpec {
1799 backoff_limit: Some(0),
1800 template: PodTemplateSpec {
1801 metadata: Some(ObjectMeta::default()),
1802 spec: Some(PodSpec {
1803 automount_service_account_token: Some(false),
1804 restart_policy: Some("Never".to_string()),
1805 security_context: Some(source_job_pod_security_context(target)),
1806 containers: vec![Container {
1807 name: "source-check".to_string(),
1808 image: Some(data_mover_image),
1809 command: Some(vec![
1810 "/bin/kaniop-data-mover".to_string(),
1811 "check".to_string(),
1812 "--operation-doc".to_string(),
1813 "/run/kaniop/operation.json".to_string(),
1814 ]),
1815 security_context: Some(hardened_security_context()),
1816 resources: Some(job_resource_requirements()),
1817 env: Some(vec![EnvVar {
1818 name: "RUST_LOG".to_string(),
1819 value: Some("info".to_string()),
1820 ..Default::default()
1821 }]),
1822 volume_mounts: Some(vec![
1823 VolumeMount {
1824 name: DATA_VOLUME.to_string(),
1825 mount_path: DATA_PATH.to_string(),
1826 read_only: Some(true),
1827 ..Default::default()
1828 },
1829 VolumeMount {
1830 name: "operation".to_string(),
1831 mount_path: "/run/kaniop".to_string(),
1832 read_only: Some(true),
1833 ..Default::default()
1834 },
1835 VolumeMount {
1836 name: "result".to_string(),
1837 mount_path: "/run/kaniop-result".to_string(),
1838 ..Default::default()
1839 },
1840 ]),
1841 ..Default::default()
1842 }],
1843 volumes: Some(vec![
1844 Volume {
1845 name: DATA_VOLUME.to_string(),
1846 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847 claim_name: primary_pvc_name(target)?,
1848 read_only: Some(true),
1849 }),
1850 ..Default::default()
1851 },
1852 Volume {
1853 name: "operation".to_string(),
1854 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855 name: operation_cm_name,
1856 ..Default::default()
1857 }),
1858 ..Default::default()
1859 },
1860 Volume {
1861 name: "result".to_string(),
1862 empty_dir: Some(Default::default()),
1863 ..Default::default()
1864 },
1865 ]),
1866 ..Default::default()
1867 }),
1868 },
1869 ..Default::default()
1870 }),
1871 ..Default::default()
1872 };
1873 api.create(&PostParams::default(), &job)
1874 .await
1875 .map(|_| ())
1876 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882 let ns = restore.namespace().unwrap_or_default();
1883 let uid = restore.uid().ok_or_else(|| {
1884 Error::MissingData(
1885 "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886 .to_string(),
1887 )
1888 })?;
1889 let namespace_uuid =
1890 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892 Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1896 let ns = restore.namespace().unwrap_or_default();
1897 let kanidm_uid = &restore.spec.target_ref.uid;
1898 let backup_id = compute_safety_backup_id(restore)?;
1899 Ok(format!(
1900 "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1901 ))
1902}
1903
1904struct VerifiedSafetyBackup {
1905 manifest_key: String,
1906 payload_sha256: String,
1907}
1908
1909async fn read_safety_backup_result(
1910 restore: &KanidmRestore,
1911 ctx: &RestoreContext,
1912 job_name: &str,
1913 expected_backup_id: &str,
1914) -> Result<VerifiedSafetyBackup> {
1915 let ns = restore.namespace().unwrap();
1916 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1917 let pod_list = pods
1918 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1919 .await
1920 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1921 let pod = pod_list.items.first().ok_or_else(|| {
1922 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1923 })?;
1924 let pod_name = pod.name_any();
1925 let container_status = pod
1926 .status
1927 .as_ref()
1928 .and_then(|s| s.container_statuses.as_ref())
1929 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1930 .ok_or_else(|| {
1931 Error::MissingData(format!(
1932 "safety-upload container status not found in pod {pod_name}"
1933 ))
1934 })?;
1935 let termination_message = container_status
1936 .state
1937 .as_ref()
1938 .and_then(|state| state.terminated.as_ref())
1939 .and_then(|t| t.message.as_ref())
1940 .ok_or_else(|| {
1941 Error::MissingData(format!(
1942 "termination message not found in pod {pod_name}; result transport failed"
1943 ))
1944 })?;
1945 if termination_message.is_empty() {
1946 return Err(Error::MissingData(format!(
1947 "termination message is empty in pod {pod_name}; result document was not written"
1948 )));
1949 }
1950 let result_doc = parse_result_document(termination_message).map_err(|e| {
1951 Error::ParseError(format!(
1952 "failed to parse result document from pod {pod_name}: {e}"
1953 ))
1954 })?;
1955 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1956 return Err(Error::ParseError(format!(
1957 "unsupported result document version: {}",
1958 result_doc.api_version
1959 )));
1960 }
1961 if result_doc.kind != "ResultDocument" {
1962 return Err(Error::ParseError(format!(
1963 "invalid result document kind: {}",
1964 result_doc.kind
1965 )));
1966 }
1967 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1968 return Err(Error::ParseError(format!(
1969 "result document operation mismatch: expected '{}', got '{}'",
1970 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1971 )));
1972 }
1973 if !result_doc.success {
1974 return Err(Error::ParseError(format!(
1975 "safety backup reported failure with exit code {:?}",
1976 result_doc.exit_code
1977 )));
1978 }
1979 if result_doc.exit_code != ExitCode::Success {
1980 return Err(Error::ParseError(format!(
1981 "safety backup exit code is not Success: {:?}",
1982 result_doc.exit_code
1983 )));
1984 }
1985 let doc_backup_id = result_doc
1986 .backup_id
1987 .as_deref()
1988 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1989 if doc_backup_id != expected_backup_id {
1990 return Err(Error::ParseError(format!(
1991 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1992 )));
1993 }
1994 let manifest_key = result_doc
1995 .manifest_key
1996 .clone()
1997 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1998 if manifest_key.is_empty() {
1999 return Err(Error::ParseError(
2000 "result document manifestKey is empty".to_string(),
2001 ));
2002 }
2003 let payload_sha256 = result_doc
2004 .payload_sha256
2005 .clone()
2006 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2007 if payload_sha256.is_empty() {
2008 return Err(Error::ParseError(
2009 "result document payloadSha256 is empty".to_string(),
2010 ));
2011 }
2012 info!(
2013 restore = %restore.name_any(),
2014 backup_id = %expected_backup_id,
2015 manifest_key = %manifest_key,
2016 "safety backup result verified"
2017 );
2018 Ok(VerifiedSafetyBackup {
2019 manifest_key,
2020 payload_sha256,
2021 })
2022}
2023
2024struct VerifiedSourcePrepResult {
2025 #[allow(dead_code)]
2026 manifest_key: String,
2027 payload_sha256: String,
2028}
2029
2030async fn read_source_prep_result(
2031 restore: &KanidmRestore,
2032 ctx: &RestoreContext,
2033 job_name: &str,
2034 expected_backup_id: &str,
2035) -> Result<VerifiedSourcePrepResult> {
2036 let ns = restore.namespace().unwrap();
2037 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2038 let pod_list = pods
2039 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2040 .await
2041 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2042 let pod = pod_list.items.first().ok_or_else(|| {
2043 Error::MissingData(format!("no pod found for source prep job {job_name}"))
2044 })?;
2045 let pod_name = pod.name_any();
2046 let container_status = pod
2047 .status
2048 .as_ref()
2049 .and_then(|s| s.container_statuses.as_ref())
2050 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2051 .ok_or_else(|| {
2052 Error::MissingData(format!(
2053 "source-prep container status not found in pod {pod_name}"
2054 ))
2055 })?;
2056 let termination_message = container_status
2057 .state
2058 .as_ref()
2059 .and_then(|state| state.terminated.as_ref())
2060 .and_then(|t| t.message.as_ref())
2061 .ok_or_else(|| {
2062 Error::MissingData(format!(
2063 "termination message not found in pod {pod_name}; result transport failed"
2064 ))
2065 })?;
2066 if termination_message.is_empty() {
2067 return Err(Error::MissingData(format!(
2068 "termination message is empty in pod {pod_name}; result document was not written"
2069 )));
2070 }
2071 let result_doc = parse_result_document(termination_message).map_err(|e| {
2072 Error::ParseError(format!(
2073 "failed to parse result document from pod {pod_name}: {e}"
2074 ))
2075 })?;
2076 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2077 return Err(Error::ParseError(format!(
2078 "unsupported result document version: {}",
2079 result_doc.api_version
2080 )));
2081 }
2082 if result_doc.kind != "ResultDocument" {
2083 return Err(Error::ParseError(format!(
2084 "invalid result document kind: {}",
2085 result_doc.kind
2086 )));
2087 }
2088 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2089 return Err(Error::ParseError(format!(
2090 "result document operation mismatch: expected '{}', got '{}'",
2091 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2092 )));
2093 }
2094 if !result_doc.success {
2095 return Err(Error::ParseError(format!(
2096 "source prep reported failure with exit code {:?}",
2097 result_doc.exit_code
2098 )));
2099 }
2100 if result_doc.exit_code != ExitCode::Success {
2101 return Err(Error::ParseError(format!(
2102 "source prep exit code is not Success: {:?}",
2103 result_doc.exit_code
2104 )));
2105 }
2106 let doc_backup_id = result_doc
2107 .backup_id
2108 .as_deref()
2109 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2110 if doc_backup_id != expected_backup_id {
2111 return Err(Error::ParseError(format!(
2112 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2113 )));
2114 }
2115 let manifest_key = result_doc
2116 .manifest_key
2117 .clone()
2118 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2119 if manifest_key.is_empty() {
2120 return Err(Error::ParseError(
2121 "result document manifestKey is empty".to_string(),
2122 ));
2123 }
2124 let payload_sha256 = result_doc
2125 .payload_sha256
2126 .clone()
2127 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2128 if payload_sha256.is_empty() {
2129 return Err(Error::ParseError(
2130 "result document payloadSha256 is empty".to_string(),
2131 ));
2132 }
2133 info!(
2134 restore = %restore.name_any(),
2135 backup_id = %expected_backup_id,
2136 manifest_key = %manifest_key,
2137 payload_sha256 = %payload_sha256,
2138 "source prep result verified"
2139 );
2140 Ok(VerifiedSourcePrepResult {
2141 manifest_key,
2142 payload_sha256,
2143 })
2144}
2145
2146fn source_job_pod_security_context(
2147 target: &Kanidm,
2148) -> k8s_openapi::api::core::v1::PodSecurityContext {
2149 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2150 security_context.seccomp_profile = Some(SeccompProfile {
2151 type_: "RuntimeDefault".to_string(),
2152 ..Default::default()
2153 });
2154 security_context
2155}
2156
2157fn hardened_security_context() -> SecurityContext {
2158 SecurityContext {
2159 allow_privilege_escalation: Some(false),
2160 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2161 drop: Some(vec!["ALL".to_string()]),
2162 ..Default::default()
2163 }),
2164 read_only_root_filesystem: Some(true),
2165 run_as_non_root: Some(true),
2166 seccomp_profile: Some(SeccompProfile {
2167 type_: "RuntimeDefault".to_string(),
2168 ..Default::default()
2169 }),
2170 ..Default::default()
2171 }
2172}
2173
2174fn kanidm_job_security_context() -> SecurityContext {
2175 SecurityContext {
2176 allow_privilege_escalation: Some(false),
2177 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2178 drop: Some(vec!["ALL".to_string()]),
2179 ..Default::default()
2180 }),
2181 seccomp_profile: Some(SeccompProfile {
2182 type_: "RuntimeDefault".to_string(),
2183 ..Default::default()
2184 }),
2185 ..Default::default()
2186 }
2187}
2188
2189fn job_resource_requirements() -> ResourceRequirements {
2190 ResourceRequirements {
2191 requests: Some(std::collections::BTreeMap::from([
2192 ("cpu".to_string(), Quantity("100m".to_string())),
2193 ("memory".to_string(), Quantity("128Mi".to_string())),
2194 ])),
2195 limits: Some(std::collections::BTreeMap::from([
2196 ("cpu".to_string(), Quantity("2".to_string())),
2197 ("memory".to_string(), Quantity("2Gi".to_string())),
2198 ])),
2199 ..Default::default()
2200 }
2201}
2202
2203async fn ensure_safety_backup_job(
2204 restore: &KanidmRestore,
2205 target: &Kanidm,
2206 ctx: &RestoreContext,
2207 name: &str,
2208 backup_id: &str,
2209 manifest_key: &str,
2210) -> Result<()> {
2211 let ns = restore.namespace().unwrap();
2212 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2213 if api
2214 .get_opt(name)
2215 .await
2216 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2217 .is_some()
2218 {
2219 return Ok(());
2220 }
2221 let safety = restore
2222 .spec
2223 .safety_backup
2224 .as_ref()
2225 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2226 let repo_ref = safety
2227 .repository_ref
2228 .as_ref()
2229 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2230 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2231 .get(&repo_ref.name)
2232 .await
2233 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2234 let _namespace_uid = &ns;
2235 let _kanidm_uid = &restore.spec.target_ref.uid;
2236 let endpoint = &repo.spec.s3.endpoint;
2237 let region = &repo.spec.s3.region;
2238 let operation_doc = build_safety_upload_operation_doc(
2239 restore,
2240 target,
2241 backup_id,
2242 manifest_key,
2243 &repo.spec.s3.bucket,
2244 &repo.spec.s3.prefix,
2245 endpoint,
2246 region,
2247 repo.spec.s3.force_path_style,
2248 repo.spec.s3.insecure,
2249 repo.spec
2250 .s3
2251 .ca_bundle_ref
2252 .as_deref()
2253 .map(|_| ca_bundle_path())
2254 .as_deref(),
2255 repo.spec.encryption.as_ref(),
2256 )?;
2257 let operation_cm_name = format!("{name}-op");
2258 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2259 let auth_method = &repo.spec.authentication.writer;
2260 let data_mover_image = data_mover_image();
2261 let init_command = vec![
2262 "kanidmd".to_string(),
2263 "database".to_string(),
2264 "backup".to_string(),
2265 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2266 ];
2267 let upload_script = format!(
2268 r#"set -eu
2269/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2270RESULT_FILE="/run/kaniop-result/result.json"
2271TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2272if [ ! -f "$RESULT_FILE" ]; then
2273 echo "ERROR: result document not found at $RESULT_FILE"
2274 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2275 exit 1
2276fi
2277if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2278 echo "ERROR: safety backup upload reported failure"
2279 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2280 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2281 exit 1
2282fi
2283dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2284echo "safety backup upload completed successfully"
2285"#
2286 );
2287 let job = Job {
2288 metadata: ObjectMeta {
2289 name: Some(name.to_string()),
2290 namespace: Some(ns.clone()),
2291 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2292 ..Default::default()
2293 },
2294 spec: Some(JobSpec {
2295 backoff_limit: Some(0),
2296 template: PodTemplateSpec {
2297 metadata: Some(ObjectMeta::default()),
2298 spec: Some(PodSpec {
2299 automount_service_account_token: Some(false),
2300 restart_policy: Some("Never".to_string()),
2301 security_context: Some(source_job_pod_security_context(target)),
2302 init_containers: Some(vec![Container {
2303 name: "safety-backup".to_string(),
2304 image: Some(restore.spec.restore_image.clone()),
2305 command: Some(init_command),
2306 security_context: Some(SecurityContext {
2307 allow_privilege_escalation: Some(false),
2308 capabilities: Some(Capabilities {
2309 drop: Some(vec!["ALL".to_string()]),
2310 ..Default::default()
2311 }),
2312 read_only_root_filesystem: Some(true),
2313 seccomp_profile: Some(SeccompProfile {
2314 type_: "RuntimeDefault".to_string(),
2315 ..Default::default()
2316 }),
2317 ..Default::default()
2318 }),
2319 resources: Some(job_resource_requirements()),
2320 env: Some(build_kanidm_db_env_vars(target)),
2321 volume_mounts: Some(vec![
2322 VolumeMount {
2323 name: DATA_VOLUME.to_string(),
2324 mount_path: DATA_PATH.to_string(),
2325 read_only: Some(false),
2326 ..Default::default()
2327 },
2328 VolumeMount {
2329 name: TLS_VOLUME.to_string(),
2330 mount_path: TLS_PATH.to_string(),
2331 read_only: Some(true),
2332 ..Default::default()
2333 },
2334 VolumeMount {
2335 name: SHARED_VOLUME.to_string(),
2336 mount_path: SHARED_VOL_PATH.to_string(),
2337 ..Default::default()
2338 },
2339 ]),
2340 ..Default::default()
2341 }]),
2342 containers: vec![Container {
2343 name: "safety-upload".to_string(),
2344 image: Some(data_mover_image),
2345 command: Some(vec!["/bin/sh".to_string()]),
2346 args: Some(vec!["-c".to_string(), upload_script]),
2347 security_context: Some(hardened_security_context()),
2348 resources: Some(job_resource_requirements()),
2349 env: {
2350 let mut env_vars =
2351 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2352 env_vars.push(EnvVar {
2353 name: "RUST_LOG".to_string(),
2354 value: Some("info".to_string()),
2355 ..Default::default()
2356 });
2357 if repo.spec.s3.ca_bundle_ref.is_some() {
2358 env_vars.push(ca_bundle_env_var());
2359 }
2360 env_vars.extend(build_encryption_env_vars(
2361 repo.spec
2362 .encryption
2363 .as_ref()
2364 .and_then(|e| e.key_ref.as_ref()),
2365 ));
2366 Some(env_vars)
2367 },
2368 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2369 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2370 volume_mounts: {
2371 let mut mounts = vec![
2372 VolumeMount {
2373 name: SHARED_VOLUME.to_string(),
2374 mount_path: SHARED_VOL_PATH.to_string(),
2375 read_only: Some(true),
2376 ..Default::default()
2377 },
2378 VolumeMount {
2379 name: "operation".to_string(),
2380 mount_path: "/run/kaniop".to_string(),
2381 read_only: Some(true),
2382 ..Default::default()
2383 },
2384 VolumeMount {
2385 name: "result".to_string(),
2386 mount_path: "/run/kaniop-result".to_string(),
2387 ..Default::default()
2388 },
2389 ];
2390 mounts.extend(build_auth_volume_mounts(auth_method));
2391 if repo.spec.s3.ca_bundle_ref.is_some() {
2392 mounts.push(build_ca_bundle_volume_mount());
2393 }
2394 Some(mounts)
2395 },
2396 ..Default::default()
2397 }],
2398 volumes: {
2399 let mut vols = vec![
2400 Volume {
2401 name: DATA_VOLUME.to_string(),
2402 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2403 claim_name: primary_pvc_name(target)?,
2404 read_only: Some(false),
2405 }),
2406 ..Default::default()
2407 },
2408 Volume {
2409 name: TLS_VOLUME.to_string(),
2410 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2411 secret_name: Some(target.effective_tls_secret_name()),
2412 ..Default::default()
2413 }),
2414 ..Default::default()
2415 },
2416 Volume {
2417 name: SHARED_VOLUME.to_string(),
2418 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2419 size_limit: Some(crate::controller::backup_job_volume_size()),
2420 ..Default::default()
2421 }),
2422 ..Default::default()
2423 },
2424 Volume {
2425 name: "operation".to_string(),
2426 config_map: Some(ConfigMapVolumeSource {
2427 name: operation_cm_name,
2428 ..Default::default()
2429 }),
2430 ..Default::default()
2431 },
2432 Volume {
2433 name: "result".to_string(),
2434 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2435 size_limit: Some(Quantity("16Mi".to_string())),
2436 ..Default::default()
2437 }),
2438 ..Default::default()
2439 },
2440 ];
2441 vols.extend(build_auth_volumes(auth_method));
2442 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2443 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2444 }
2445 Some(vols)
2446 },
2447 ..Default::default()
2448 }),
2449 },
2450 ..Default::default()
2451 }),
2452 ..Default::default()
2453 };
2454 api.create(&PostParams::default(), &job)
2455 .await
2456 .map(|_| ())
2457 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2458}
2459
2460async fn ensure_source_prep_job(
2461 restore: &KanidmRestore,
2462 target: &Kanidm,
2463 ctx: &RestoreContext,
2464 name: &str,
2465) -> Result<()> {
2466 let ns = restore.namespace().unwrap();
2467 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2468 if api
2469 .get_opt(name)
2470 .await
2471 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2472 .is_some()
2473 {
2474 return Ok(());
2475 }
2476 let backup_name = restore
2477 .spec
2478 .source
2479 .backup_ref
2480 .as_ref()
2481 .map(|b| b.name.clone())
2482 .unwrap_or_default();
2483 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2484 .get(&backup_name)
2485 .await
2486 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2487 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2488 .get(&backup.spec.repository_ref.name)
2489 .await
2490 .map_err(|e| {
2491 Error::kube_error(
2492 "get",
2493 "KanidmBackupRepository",
2494 &ns,
2495 &backup.spec.repository_ref.name,
2496 e,
2497 )
2498 })?;
2499 let endpoint = &repo.spec.s3.endpoint;
2500 let region = &repo.spec.s3.region;
2501 let operation_doc = build_download_operation_doc(
2502 target,
2503 &backup.spec.manifest_key,
2504 &backup.spec.backup_id,
2505 &backup.spec.kanidm_ref.uid,
2506 &repo.spec.s3.bucket,
2507 &repo.spec.s3.prefix,
2508 endpoint,
2509 region,
2510 repo.spec.s3.force_path_style,
2511 repo.spec.s3.insecure,
2512 repo.spec
2513 .s3
2514 .ca_bundle_ref
2515 .as_deref()
2516 .map(|_| ca_bundle_path())
2517 .as_deref(),
2518 repo.spec.encryption.as_ref(),
2519 );
2520 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2521 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2522 let auth_method = &repo.spec.authentication.reader;
2523 let data_mover_image = data_mover_image();
2524 let download_script = format!(
2525 r#"set -eu
2526/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2527RESULT_FILE="/run/kaniop-result/result.json"
2528TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2529if [ ! -f "$RESULT_FILE" ]; then
2530 echo "ERROR: result document not found at $RESULT_FILE"
2531 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2532 exit 1
2533fi
2534if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2535 echo "ERROR: source preparation download reported failure"
2536 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2537 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2538 exit 1
2539fi
2540dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2541echo "source preparation download completed successfully"
2542"#
2543 );
2544 let job = Job {
2545 metadata: ObjectMeta {
2546 name: Some(name.to_string()),
2547 namespace: Some(ns.clone()),
2548 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2549 ..Default::default()
2550 },
2551 spec: Some(JobSpec {
2552 backoff_limit: Some(0),
2553 template: PodTemplateSpec {
2554 metadata: Some(ObjectMeta::default()),
2555 spec: Some(PodSpec {
2556 automount_service_account_token: Some(false),
2557 restart_policy: Some("Never".to_string()),
2558 security_context: Some(source_job_pod_security_context(target)),
2559 containers: vec![Container {
2560 name: "source-prep".to_string(),
2561 image: Some(data_mover_image),
2562 command: Some(vec!["/bin/sh".to_string()]),
2563 args: Some(vec!["-c".to_string(), download_script]),
2564 security_context: Some(hardened_security_context()),
2565 resources: Some(job_resource_requirements()),
2566 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2567 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2568 env: {
2569 let mut env_vars = build_auth_env_vars(
2570 auth_method,
2571 &backup.spec.repository_ref.name,
2572 AuthRole::Reader,
2573 );
2574 env_vars.push(EnvVar {
2575 name: "RUST_LOG".to_string(),
2576 value: Some("info".to_string()),
2577 ..Default::default()
2578 });
2579 if repo.spec.s3.ca_bundle_ref.is_some() {
2580 env_vars.push(ca_bundle_env_var());
2581 }
2582 env_vars.extend(build_encryption_env_vars(
2583 repo.spec
2584 .encryption
2585 .as_ref()
2586 .and_then(|e| e.key_ref.as_ref()),
2587 ));
2588 Some(env_vars)
2589 },
2590 volume_mounts: {
2591 let mut mounts = vec![
2592 VolumeMount {
2593 name: DATA_VOLUME.to_string(),
2594 mount_path: DATA_PATH.to_string(),
2595 ..Default::default()
2596 },
2597 VolumeMount {
2598 name: "operation".to_string(),
2599 mount_path: "/run/kaniop".to_string(),
2600 read_only: Some(true),
2601 ..Default::default()
2602 },
2603 VolumeMount {
2604 name: "result".to_string(),
2605 mount_path: "/run/kaniop-result".to_string(),
2606 ..Default::default()
2607 },
2608 ];
2609 mounts.extend(build_auth_volume_mounts(auth_method));
2610 if repo.spec.s3.ca_bundle_ref.is_some() {
2611 mounts.push(build_ca_bundle_volume_mount());
2612 }
2613 Some(mounts)
2614 },
2615 ..Default::default()
2616 }],
2617 volumes: {
2618 let mut vols = vec![
2619 Volume {
2620 name: DATA_VOLUME.to_string(),
2621 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2622 claim_name: primary_pvc_name(target)?,
2623 read_only: Some(false),
2624 }),
2625 ..Default::default()
2626 },
2627 Volume {
2628 name: "operation".to_string(),
2629 config_map: Some(ConfigMapVolumeSource {
2630 name: operation_cm_name,
2631 ..Default::default()
2632 }),
2633 ..Default::default()
2634 },
2635 Volume {
2636 name: "result".to_string(),
2637 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2638 size_limit: Some(Quantity("16Mi".to_string())),
2639 ..Default::default()
2640 }),
2641 ..Default::default()
2642 },
2643 ];
2644 vols.extend(build_auth_volumes(auth_method));
2645 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2646 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2647 }
2648 Some(vols)
2649 },
2650 ..Default::default()
2651 }),
2652 },
2653 ..Default::default()
2654 }),
2655 ..Default::default()
2656 };
2657 api.create(&PostParams::default(), &job)
2658 .await
2659 .map(|_| ())
2660 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2661}
2662
2663#[allow(clippy::too_many_arguments)]
2664fn build_safety_upload_operation_doc(
2665 restore: &KanidmRestore,
2666 target: &Kanidm,
2667 backup_id: &str,
2668 manifest_key: &str,
2669 bucket: &str,
2670 prefix: &str,
2671 endpoint: &str,
2672 region: &str,
2673 force_path_style: bool,
2674 insecure: bool,
2675 ca_bundle_path: Option<&str>,
2676 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2677) -> Result<String> {
2678 let enc_mode =
2679 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2680 let enc_key_id = encryption
2681 .and_then(|e| e.key_id.as_ref())
2682 .map(|s| serde_json::Value::String(s.clone()));
2683 let op = UploadOperation {
2684 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2685 bucket: bucket.to_string(),
2686 prefix: prefix.to_string(),
2687 endpoint: endpoint.to_string(),
2688 region: region.to_string(),
2689 force_path_style,
2690 insecure,
2691 ca_bundle_path: ca_bundle_path.map(str::to_string),
2692 backup_id: backup_id.to_string(),
2693 namespace_uid: restore.namespace().unwrap_or_default(),
2694 kanidm_uid: restore.spec.target_ref.uid.clone(),
2695 kanidm_name: restore.spec.target_ref.name.clone(),
2696 domain: target.spec.domain.clone(),
2697 kanidm_version: target
2698 .status
2699 .as_ref()
2700 .and_then(|s| s.version.as_ref())
2701 .map(|v| v.image_tag.clone())
2702 .unwrap_or_default(),
2703 image_digest: None,
2704 consistency: "kanidm-offline".to_string(),
2705 reason: "restore-safety".to_string(),
2706 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2707 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2708 result_path: "/run/kaniop-result/result.json".to_string(),
2709 max_concurrent_parts: 4,
2710 max_retries: 3,
2711 };
2712 let _ = manifest_key;
2713 let doc = OperationDocument {
2714 api_version: OPERATION_DOC_VERSION.to_string(),
2715 kind: "OperationDocument".to_string(),
2716 spec: OperationSpec::Upload(op),
2717 };
2718 serde_json::to_string(&doc)
2719 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2720}
2721
2722async fn validate_backup_ref(
2723 restore: &KanidmRestore,
2724 target: &Kanidm,
2725 ctx: &RestoreContext,
2726) -> Result<Option<DisasterRecoverySource>> {
2727 let backup_ref = restore
2728 .spec
2729 .source
2730 .backup_ref
2731 .as_ref()
2732 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2733 let ns = restore.namespace().unwrap();
2734 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2735 .get(&backup_ref.name)
2736 .await
2737 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2738 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2739 return Err(Error::MissingData(format!(
2740 "KanidmBackup '{}' is not Ready (phase: {:?})",
2741 backup_ref.name,
2742 backup.status.as_ref().map(|s| &s.phase)
2743 )));
2744 }
2745 let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2746 || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2747 let disaster_recovery_source = if source_identity_mismatch {
2748 if !disaster_recovery_requested(restore) {
2749 return Err(Error::MissingData(format!(
2750 "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2751 backup.spec.kanidm_ref.name,
2752 backup.spec.kanidm_ref.uid,
2753 restore.spec.target_ref.name,
2754 restore.spec.target_ref.uid,
2755 DISASTER_RECOVERY_ANNOTATION,
2756 BREAK_GLASS_REASON_ANNOTATION,
2757 BREAK_GLASS_APPROVED_BY_ANNOTATION,
2758 )));
2759 }
2760 validate_disaster_recovery_approval(restore)?;
2761 Some(DisasterRecoverySource {
2762 name: backup.spec.kanidm_ref.name.clone(),
2763 uid: backup.spec.kanidm_ref.uid.clone(),
2764 })
2765 } else {
2766 None
2767 };
2768 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2769 .get(&backup.spec.repository_ref.name)
2770 .await
2771 .map_err(|e| {
2772 Error::kube_error(
2773 "get",
2774 "KanidmBackupRepository",
2775 &ns,
2776 &backup.spec.repository_ref.name,
2777 e,
2778 )
2779 })?;
2780 if !has_accepted_condition(
2781 &repo
2782 .status
2783 .as_ref()
2784 .map(|s| &s.conditions)
2785 .cloned()
2786 .unwrap_or_default(),
2787 ) {
2788 return Err(Error::MissingData(format!(
2789 "KanidmBackupRepository '{}' configuration has not been accepted",
2790 backup.spec.repository_ref.name
2791 )));
2792 }
2793 validate_backup_compatibility(restore, target, &backup)?;
2794 Ok(disaster_recovery_source)
2795}
2796
2797fn has_accepted_condition(conditions: &[Condition]) -> bool {
2798 conditions
2799 .iter()
2800 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2801}
2802
2803fn validate_backup_compatibility(
2804 restore: &KanidmRestore,
2805 target: &Kanidm,
2806 backup: &KanidmBackup,
2807) -> Result<()> {
2808 let status = backup.status.as_ref();
2809 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2810 let target_version = target
2811 .status
2812 .as_ref()
2813 .and_then(|s| s.version.as_ref())
2814 .map(|v| v.image_tag.as_str());
2815 if let Some(tv) = target_version {
2816 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2817 return Err(Error::MissingData(format!(
2818 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2819 )));
2820 }
2821 }
2822 }
2823 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2824 if !backup_digest.is_empty()
2825 && restore.spec.restore_image.contains('@')
2826 && !restore.spec.restore_image.contains(backup_digest)
2827 {
2828 return Err(Error::MissingData(format!(
2829 "restore image digest does not match backup image digest '{backup_digest}'"
2830 )));
2831 }
2832 }
2833 Ok(())
2834}
2835
2836#[allow(clippy::too_many_arguments)]
2837fn build_download_operation_doc(
2838 target: &Kanidm,
2839 manifest_key: &str,
2840 expected_backup_id: &str,
2841 expected_source_uid: &str,
2842 bucket: &str,
2843 prefix: &str,
2844 endpoint: &str,
2845 region: &str,
2846 force_path_style: bool,
2847 insecure: bool,
2848 ca_bundle_path: Option<&str>,
2849 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2850) -> String {
2851 let enc_mode =
2852 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2853 let enc_key_id = encryption
2854 .and_then(|e| e.key_id.as_ref())
2855 .map(|s| serde_json::Value::String(s.clone()));
2856 serde_json::json!({
2857 "apiVersion": "backup.kaniop.rs/v1alpha1",
2858 "kind": "OperationDocument",
2859 "operation": "download",
2860 "manifestKey": manifest_key,
2861 "bucket": bucket,
2862 "prefix": prefix,
2863 "endpoint": endpoint,
2864 "region": region,
2865 "forcePathStyle": force_path_style,
2866 "insecure": insecure,
2867 "caBundlePath": ca_bundle_path,
2868 "expectedBackupId": expected_backup_id,
2869 "expectedKanidmUid": expected_source_uid,
2870 "expectedDomain": target.spec.domain,
2871 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2872 "resultPath": "/run/kaniop-result/result.json",
2873 "maxRetries": 3,
2874 "encryptionMode": enc_mode,
2875 "encryptionKeyId": enc_key_id,
2876 })
2877 .to_string()
2878}
2879
2880async fn ensure_operation_configmap(
2881 restore: &KanidmRestore,
2882 name: &str,
2883 data: &str,
2884 ns: &str,
2885 ctx: &RestoreContext,
2886) -> Result<()> {
2887 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2888 if api
2889 .get_opt(name)
2890 .await
2891 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2892 .is_some()
2893 {
2894 return Ok(());
2895 }
2896 let cm = ConfigMap {
2897 metadata: ObjectMeta {
2898 name: Some(name.to_string()),
2899 namespace: Some(ns.to_string()),
2900 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2901 ..Default::default()
2902 },
2903 data: Some(std::collections::BTreeMap::from([(
2904 "operation.json".to_string(),
2905 data.to_string(),
2906 )])),
2907 ..Default::default()
2908 };
2909 api.create(&PostParams::default(), &cm)
2910 .await
2911 .map(|_| ())
2912 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2913}
2914
2915#[cfg(test)]
2916mod tests {
2917 use super::{
2918 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2919 CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2920 DISASTER_RECOVERY_ANNOTATION,
2921 KanidmRestore, KanidmRestorePhase,
2922 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2923 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2924 disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2925 is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2926 restore_job_name, safe_basename, validate_disaster_recovery_approval,
2927 validate_safety_backup_config, validate_source, verify_job_name,
2928 };
2929 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2930 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2931 use k8s_openapi::jiff::Timestamp;
2932 use kube::api::ObjectMeta as ApiObjectMeta;
2933 use std::collections::BTreeMap;
2934
2935 fn make_restore(
2936 source: KanidmRestoreSource,
2937 safety: Option<SafetyBackupConfig>,
2938 ) -> KanidmRestore {
2939 KanidmRestore {
2940 metadata: ApiObjectMeta {
2941 name: Some("test-restore".to_string()),
2942 namespace: Some("default".to_string()),
2943 uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2944 ..Default::default()
2945 },
2946 spec: KanidmRestoreSpec {
2947 target_ref: KanidmRestoreTargetRef {
2948 name: "corp-idm".to_string(),
2949 uid: "test-uid-123".to_string(),
2950 },
2951 source,
2952 restore_image: "kanidm/server@sha256:abc".to_string(),
2953 safety_backup: safety,
2954 },
2955 status: None,
2956 }
2957 }
2958
2959 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2960 super::super::crd::Kanidm {
2961 status: Some(super::super::crd::KanidmStatus {
2962 available_replicas: 1,
2963 replicas: 1,
2964 unavailable_replicas: 0,
2965 updated_replicas: 1,
2966 replica_statuses: vec![],
2967 replica_column: "1/1".to_string(),
2968 secret_name: None,
2969 version: Some(super::super::crd::KanidmVersionStatus {
2970 image_tag: image_tag.to_string(),
2971 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2972 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2973 }),
2974 domain_appearance_image: None,
2975 mail_sender: None,
2976 conditions: None,
2977 }),
2978 ..Default::default()
2979 }
2980 }
2981
2982 #[test]
2983 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2984 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2985 }
2986
2987 #[test]
2988 fn completed_restore_sets_ready_condition() {
2989 let mut status = KanidmRestoreStatus {
2990 phase: KanidmRestorePhase::Completed,
2991 ..Default::default()
2992 };
2993 super::update_restore_conditions(&mut status, Some(7));
2994 assert!(status.conditions.iter().any(|condition| {
2995 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2996 }));
2997 assert!(!status.conditions.iter().any(|condition| {
2998 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2999 }));
3000 }
3001
3002 #[test]
3003 fn disaster_recovery_condition_survives_phase_updates() {
3004 let dr_condition = Condition {
3005 type_: CONDITION_DISASTER_RECOVERY.to_string(),
3006 status: CONDITION_TRUE.to_string(),
3007 reason: "ApprovedSourceIdentityMismatch".to_string(),
3008 message: "approved DR".to_string(),
3009 last_transition_time: Time(Timestamp::now()),
3010 observed_generation: Some(7),
3011 };
3012 let mut status = KanidmRestoreStatus {
3013 phase: KanidmRestorePhase::Quiescing,
3014 conditions: vec![dr_condition.clone()],
3015 ..Default::default()
3016 };
3017
3018 super::update_restore_conditions(&mut status, Some(8));
3019
3020 assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3021 }
3022
3023 #[test]
3024 fn failed_restore_sets_failed_condition() {
3025 let mut status = KanidmRestoreStatus {
3026 phase: KanidmRestorePhase::Failed,
3027 message: Some("verification failed".to_string()),
3028 ..Default::default()
3029 };
3030 super::update_restore_conditions(&mut status, Some(8));
3031 assert!(status.conditions.iter().any(|condition| {
3032 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3033 }));
3034 }
3035
3036 #[test]
3037 fn restore_filename_is_confined_to_backup_directory() {
3038 assert!(safe_basename("backup.json.gz"));
3039 assert!(!safe_basename("../backup.json.gz"));
3040 assert!(!safe_basename("a/b"));
3041 assert!(!safe_basename(""));
3042 }
3043
3044 #[test]
3045 fn restore_image_rejects_latest_and_unpinned_names() {
3046 assert!(mutable_image("kanidm/server:latest"));
3047 assert!(mutable_image("kanidm/server"));
3048 assert!(!mutable_image("kanidm/server:1.10.0"));
3049 assert!(!mutable_image("kanidm/server@sha256:abc"));
3050 }
3051
3052 #[test]
3053 fn source_validation_rejects_both_local_and_backup_ref() {
3054 let source = KanidmRestoreSource {
3055 local: Some(super::KanidmRestoreLocalSource {
3056 file_name: "backup.json".to_string(),
3057 }),
3058 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3059 name: "some-backup".to_string(),
3060 }),
3061 };
3062 let restore = make_restore(source, None);
3063 assert!(validate_source(&restore).is_err());
3064 }
3065
3066 #[test]
3067 fn source_validation_rejects_empty_source() {
3068 let source = KanidmRestoreSource {
3069 local: None,
3070 backup_ref: None,
3071 };
3072 let restore = make_restore(source, None);
3073 assert!(validate_source(&restore).is_err());
3074 }
3075
3076 #[test]
3077 fn source_validation_accepts_local_source() {
3078 let source = KanidmRestoreSource {
3079 local: Some(super::KanidmRestoreLocalSource {
3080 file_name: "backup.json".to_string(),
3081 }),
3082 backup_ref: None,
3083 };
3084 let restore = make_restore(source, None);
3085 assert!(validate_source(&restore).is_ok());
3086 }
3087
3088 #[test]
3089 fn source_validation_accepts_backup_ref_source() {
3090 let source = KanidmRestoreSource {
3091 local: None,
3092 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3093 name: "corp-idm-019c7c76".to_string(),
3094 }),
3095 };
3096 let restore = make_restore(source, None);
3097 assert!(validate_source(&restore).is_ok());
3098 }
3099
3100 #[test]
3101 fn source_validation_rejects_empty_backup_ref_name() {
3102 let source = KanidmRestoreSource {
3103 local: None,
3104 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3105 name: "".to_string(),
3106 }),
3107 };
3108 let restore = make_restore(source, None);
3109 assert!(validate_source(&restore).is_err());
3110 }
3111
3112 #[test]
3113 fn is_remote_source_true_for_backup_ref() {
3114 let source = KanidmRestoreSource {
3115 local: None,
3116 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3117 name: "backup-1".to_string(),
3118 }),
3119 };
3120 let restore = make_restore(source, None);
3121 assert!(is_remote_source(&restore));
3122 }
3123
3124 #[test]
3125 fn is_remote_source_false_for_local() {
3126 let source = KanidmRestoreSource {
3127 local: Some(super::KanidmRestoreLocalSource {
3128 file_name: "backup.json".to_string(),
3129 }),
3130 backup_ref: None,
3131 };
3132 let restore = make_restore(source, None);
3133 assert!(!is_remote_source(&restore));
3134 }
3135
3136 #[test]
3137 fn requires_safety_backup_default_true() {
3138 let source = KanidmRestoreSource {
3139 local: None,
3140 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3141 name: "backup-1".to_string(),
3142 }),
3143 };
3144 let restore = make_restore(source, None);
3145 assert!(requires_safety_backup(&restore));
3146 }
3147
3148 #[test]
3149 fn requires_safety_backup_false_when_skip() {
3150 let source = KanidmRestoreSource {
3151 local: None,
3152 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3153 name: "backup-1".to_string(),
3154 }),
3155 };
3156 let safety = SafetyBackupConfig {
3157 repository_ref: Some(SafetyBackupRepositoryRef {
3158 name: "offsite".to_string(),
3159 }),
3160 skip: true,
3161 };
3162 let restore = make_restore(source, Some(safety));
3163 assert!(!requires_safety_backup(&restore));
3164 }
3165
3166 #[test]
3167 fn break_glass_validation_rejects_skip_without_reason() {
3168 let source = KanidmRestoreSource {
3169 local: None,
3170 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3171 name: "backup-1".to_string(),
3172 }),
3173 };
3174 let safety = SafetyBackupConfig {
3175 repository_ref: Some(SafetyBackupRepositoryRef {
3176 name: "offsite".to_string(),
3177 }),
3178 skip: true,
3179 };
3180 let mut restore = make_restore(source, Some(safety));
3181 restore.metadata.annotations = Some(BTreeMap::from([(
3182 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3183 "commander@example.com".to_string(),
3184 )]));
3185 assert!(validate_safety_backup_config(&restore).is_err());
3186 }
3187
3188 #[test]
3189 fn break_glass_validation_rejects_skip_without_approver() {
3190 let source = KanidmRestoreSource {
3191 local: None,
3192 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3193 name: "backup-1".to_string(),
3194 }),
3195 };
3196 let safety = SafetyBackupConfig {
3197 repository_ref: Some(SafetyBackupRepositoryRef {
3198 name: "offsite".to_string(),
3199 }),
3200 skip: true,
3201 };
3202 let mut restore = make_restore(source, Some(safety));
3203 restore.metadata.annotations = Some(BTreeMap::from([(
3204 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3205 "PVC is unreadable".to_string(),
3206 )]));
3207 assert!(validate_safety_backup_config(&restore).is_err());
3208 }
3209
3210 #[test]
3211 fn break_glass_validation_accepts_valid_annotations() {
3212 let source = KanidmRestoreSource {
3213 local: None,
3214 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3215 name: "backup-1".to_string(),
3216 }),
3217 };
3218 let safety = SafetyBackupConfig {
3219 repository_ref: Some(SafetyBackupRepositoryRef {
3220 name: "offsite".to_string(),
3221 }),
3222 skip: true,
3223 };
3224 let mut restore = make_restore(source, Some(safety));
3225 restore.metadata.annotations = Some(BTreeMap::from([
3226 (
3227 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3228 "PVC is unreadable".to_string(),
3229 ),
3230 (
3231 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3232 "commander@example.com".to_string(),
3233 ),
3234 ]));
3235 assert!(validate_safety_backup_config(&restore).is_ok());
3236 }
3237
3238 #[test]
3239 fn disaster_recovery_requires_explicit_approval_annotations() {
3240 let source = KanidmRestoreSource {
3241 local: None,
3242 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3243 name: "backup-1".to_string(),
3244 }),
3245 };
3246 let mut restore = make_restore(source, None);
3247 restore.metadata.annotations = Some(BTreeMap::from([(
3248 DISASTER_RECOVERY_ANNOTATION.to_string(),
3249 "true".to_string(),
3250 )]));
3251 assert!(disaster_recovery_requested(&restore));
3252 assert!(validate_disaster_recovery_approval(&restore).is_err());
3253 }
3254
3255 #[test]
3256 fn disaster_recovery_accepts_reason_and_approver() {
3257 let source = KanidmRestoreSource {
3258 local: None,
3259 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3260 name: "backup-1".to_string(),
3261 }),
3262 };
3263 let mut restore = make_restore(source, None);
3264 restore.metadata.annotations = Some(BTreeMap::from([
3265 (
3266 DISASTER_RECOVERY_ANNOTATION.to_string(),
3267 "true".to_string(),
3268 ),
3269 (
3270 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3271 "original cluster lost".to_string(),
3272 ),
3273 (
3274 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3275 "incident-commander".to_string(),
3276 ),
3277 ]));
3278 assert!(disaster_recovery_requested(&restore));
3279 assert!(validate_disaster_recovery_approval(&restore).is_ok());
3280 }
3281
3282 #[test]
3283 fn remote_source_requires_repository_ref_when_not_skipping() {
3284 let source = KanidmRestoreSource {
3285 local: None,
3286 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3287 name: "backup-1".to_string(),
3288 }),
3289 };
3290 let safety = SafetyBackupConfig {
3291 repository_ref: None,
3292 skip: false,
3293 };
3294 let restore = make_restore(source, Some(safety));
3295 assert!(validate_safety_backup_config(&restore).is_err());
3296 }
3297
3298 #[test]
3299 fn remote_source_accepts_repository_ref() {
3300 let source = KanidmRestoreSource {
3301 local: None,
3302 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3303 name: "backup-1".to_string(),
3304 }),
3305 };
3306 let safety = SafetyBackupConfig {
3307 repository_ref: Some(SafetyBackupRepositoryRef {
3308 name: "offsite".to_string(),
3309 }),
3310 skip: false,
3311 };
3312 let restore = make_restore(source, Some(safety));
3313 assert!(validate_safety_backup_config(&restore).is_ok());
3314 }
3315
3316 #[test]
3317 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3318 let phases = vec![
3319 KanidmRestorePhase::Pending,
3320 KanidmRestorePhase::Validating,
3321 KanidmRestorePhase::Quiescing,
3322 KanidmRestorePhase::SafetyBackup,
3323 KanidmRestorePhase::PreparingSource,
3324 KanidmRestorePhase::RestoringPrimary,
3325 KanidmRestorePhase::Verifying,
3326 KanidmRestorePhase::RebuildingReplicas,
3327 KanidmRestorePhase::Resuming,
3328 KanidmRestorePhase::Completed,
3329 ];
3330 assert_eq!(phases.len(), 10);
3331 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3332 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3333 }
3334
3335 #[test]
3336 fn status_default_has_empty_phase_timestamps() {
3337 let status = KanidmRestoreStatus::default();
3338 assert!(status.phase_timestamps.is_empty());
3339 }
3340
3341 #[test]
3342 fn status_default_has_empty_original_replicas() {
3343 let status = KanidmRestoreStatus::default();
3344 assert!(status.original_replicas.is_empty());
3345 }
3346
3347 #[test]
3348 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3349 let mut target = super::super::crd::Kanidm::default();
3350 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3351 run_as_non_root: Some(false),
3352 run_as_user: Some(389),
3353 run_as_group: Some(389),
3354 fs_group: Some(389),
3355 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3356 supplemental_groups: Some(vec![390, 391]),
3357 ..Default::default()
3358 });
3359
3360 let ctx = super::source_job_pod_security_context(&target);
3361
3362 assert_eq!(ctx.run_as_non_root, Some(false));
3363 assert_eq!(ctx.run_as_user, Some(389));
3364 assert_eq!(ctx.run_as_group, Some(389));
3365 assert_eq!(ctx.fs_group, Some(389));
3366 assert_eq!(
3367 ctx.fs_group_change_policy.as_deref(),
3368 Some("OnRootMismatch")
3369 );
3370 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3371 assert_eq!(
3372 ctx.seccomp_profile
3373 .as_ref()
3374 .map(|profile| profile.type_.as_str()),
3375 Some("RuntimeDefault")
3376 );
3377 }
3378
3379 #[test]
3380 fn hardened_security_context_drops_all_capabilities() {
3381 let ctx = hardened_security_context();
3382 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3383 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3384 assert_eq!(ctx.run_as_non_root, Some(true));
3385 let caps = ctx.capabilities.unwrap();
3386 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3387 assert_eq!(
3388 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3389 Some("RuntimeDefault")
3390 );
3391 }
3392
3393 #[test]
3394 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3395 let ctx = kanidm_job_security_context();
3396 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3397 assert!(ctx.run_as_non_root.is_none());
3398 assert!(ctx.read_only_root_filesystem.is_none());
3399 let caps = ctx.capabilities.unwrap();
3400 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3401 assert_eq!(
3402 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3403 Some("RuntimeDefault")
3404 );
3405 }
3406
3407 #[test]
3408 fn safety_backup_job_name_is_deterministic() {
3409 let source = KanidmRestoreSource {
3410 local: Some(super::KanidmRestoreLocalSource {
3411 file_name: "backup.json".to_string(),
3412 }),
3413 backup_ref: None,
3414 };
3415 let restore = make_restore(source, None);
3416 let name1 = super::safety_backup_job_name(&restore);
3417 let name2 = super::safety_backup_job_name(&restore);
3418 assert_eq!(name1, name2);
3419 assert_eq!(name1, "test-restore-safety-backup");
3420 }
3421
3422 #[test]
3423 fn source_prep_job_name_is_deterministic() {
3424 let source = KanidmRestoreSource {
3425 local: None,
3426 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3427 name: "backup-1".to_string(),
3428 }),
3429 };
3430 let restore = make_restore(source, None);
3431 let name = super::source_prep_job_name(&restore);
3432 assert_eq!(name, "test-restore-source-prep");
3433 }
3434
3435 #[test]
3436 fn break_glass_condition_is_recorded_in_status() {
3437 let mut status = KanidmRestoreStatus::default();
3438 let condition = super::restore_condition(
3439 &status.conditions,
3440 CONDITION_BREAK_GLASS,
3441 CONDITION_TRUE,
3442 "BreakGlassOverride",
3443 "Safety backup skipped",
3444 Some(1),
3445 );
3446 status.conditions.push(condition);
3447 assert!(status.conditions.iter().any(|c| {
3448 c.type_ == CONDITION_BREAK_GLASS
3449 && c.status == CONDITION_TRUE
3450 && c.reason == "BreakGlassOverride"
3451 }));
3452 }
3453
3454 #[test]
3455 fn replica_count_entry_serialization() {
3456 let entry = ReplicaCountEntry {
3457 group: "primary".to_string(),
3458 replicas: 3,
3459 };
3460 let json = serde_json::to_string(&entry).unwrap();
3461 assert!(json.contains("\"group\":\"primary\""));
3462 assert!(json.contains("\"replicas\":3"));
3463 }
3464
3465 #[test]
3466 fn local_source_requires_repository_ref_when_not_skipping() {
3467 let source = KanidmRestoreSource {
3468 local: Some(super::KanidmRestoreLocalSource {
3469 file_name: "backup.json".to_string(),
3470 }),
3471 backup_ref: None,
3472 };
3473 let safety = SafetyBackupConfig {
3474 repository_ref: None,
3475 skip: false,
3476 };
3477 let restore = make_restore(source, Some(safety));
3478 assert!(validate_safety_backup_config(&restore).is_err());
3479 }
3480
3481 #[test]
3482 fn local_source_accepts_repository_ref_when_not_skipping() {
3483 let source = KanidmRestoreSource {
3484 local: Some(super::KanidmRestoreLocalSource {
3485 file_name: "backup.json".to_string(),
3486 }),
3487 backup_ref: None,
3488 };
3489 let safety = SafetyBackupConfig {
3490 repository_ref: Some(SafetyBackupRepositoryRef {
3491 name: "offsite".to_string(),
3492 }),
3493 skip: false,
3494 };
3495 let restore = make_restore(source, Some(safety));
3496 assert!(validate_safety_backup_config(&restore).is_ok());
3497 }
3498
3499 #[test]
3500 fn skip_without_break_glass_annotations_is_rejected() {
3501 let source = KanidmRestoreSource {
3502 local: Some(super::KanidmRestoreLocalSource {
3503 file_name: "backup.json".to_string(),
3504 }),
3505 backup_ref: None,
3506 };
3507 let safety = SafetyBackupConfig {
3508 repository_ref: None,
3509 skip: true,
3510 };
3511 let restore = make_restore(source, Some(safety));
3512 assert!(validate_safety_backup_config(&restore).is_err());
3513 }
3514
3515 #[test]
3516 fn has_accepted_condition_true() {
3517 let conditions = vec![Condition {
3518 type_: "Ready".to_string(),
3519 status: "True".to_string(),
3520 reason: "Accepted".to_string(),
3521 message: String::new(),
3522 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3523 k8s_openapi::jiff::Timestamp::now(),
3524 ),
3525 observed_generation: None,
3526 }];
3527 assert!(has_accepted_condition(&conditions));
3528 }
3529
3530 #[test]
3531 fn has_accepted_condition_false_when_not_accepted_reason() {
3532 let conditions = vec![Condition {
3533 type_: "Ready".to_string(),
3534 status: "True".to_string(),
3535 reason: "SomeOtherReason".to_string(),
3536 message: String::new(),
3537 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3538 k8s_openapi::jiff::Timestamp::now(),
3539 ),
3540 observed_generation: None,
3541 }];
3542 assert!(!has_accepted_condition(&conditions));
3543 }
3544
3545 #[test]
3546 fn has_accepted_condition_false_when_status_false() {
3547 let conditions = vec![Condition {
3548 type_: "Ready".to_string(),
3549 status: "False".to_string(),
3550 reason: "Accepted".to_string(),
3551 message: String::new(),
3552 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3553 k8s_openapi::jiff::Timestamp::now(),
3554 ),
3555 observed_generation: None,
3556 }];
3557 assert!(!has_accepted_condition(&conditions));
3558 }
3559
3560 #[test]
3561 fn has_accepted_condition_empty() {
3562 assert!(!has_accepted_condition(&[]));
3563 }
3564
3565 #[test]
3566 fn safety_backup_upload_operation_doc_is_valid() {
3567 let source = KanidmRestoreSource {
3568 local: None,
3569 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3570 name: "backup-1".to_string(),
3571 }),
3572 };
3573 let mut restore = make_restore(source, None);
3574 restore.metadata.namespace = Some("test-ns".to_string());
3575 let target = super::super::crd::Kanidm::default();
3576 let doc_str = super::build_safety_upload_operation_doc(
3577 &restore,
3578 &target,
3579 "test-backup-id",
3580 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3581 "my-bucket",
3582 "prod",
3583 "https://s3.example.com",
3584 "us-east-1",
3585 false,
3586 false,
3587 None,
3588 None,
3589 )
3590 .unwrap();
3591 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3592 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3593 assert_eq!(parsed["kind"], "OperationDocument");
3594 assert_eq!(parsed["operation"], "upload");
3595 assert_eq!(parsed["bucket"], "my-bucket");
3596 assert_eq!(parsed["prefix"], "prod");
3597 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3598 assert_eq!(parsed["region"], "us-east-1");
3599 assert_eq!(parsed["backupId"], "test-backup-id");
3600 assert_eq!(parsed["consistency"], "kanidm-offline");
3601 assert_eq!(parsed["reason"], "restore-safety");
3602 assert_eq!(
3603 parsed["payloadPath"],
3604 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3605 );
3606 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3607 }
3608
3609 #[test]
3610 fn download_operation_doc_uses_real_repository_data() {
3611 let source = KanidmRestoreSource {
3612 local: None,
3613 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3614 name: "backup-1".to_string(),
3615 }),
3616 };
3617 let _restore = make_restore(source, None);
3618 let target = super::super::crd::Kanidm::default();
3619 let doc_str = super::build_download_operation_doc(
3620 &target,
3621 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3622 "019c7c76-f423-7a12-8f41-2bea7588a303",
3623 "source-uid-456",
3624 "real-bucket",
3625 "real-prefix",
3626 "https://real-endpoint.com",
3627 "eu-west-1",
3628 true,
3629 false,
3630 None,
3631 None,
3632 );
3633 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3634 assert_eq!(parsed["bucket"], "real-bucket");
3635 assert_eq!(parsed["prefix"], "real-prefix");
3636 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3637 assert_eq!(parsed["region"], "eu-west-1");
3638 assert_eq!(parsed["forcePathStyle"], true);
3639 assert_eq!(
3640 parsed["manifestKey"],
3641 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3642 );
3643 assert_eq!(
3644 parsed["expectedBackupId"],
3645 "019c7c76-f423-7a12-8f41-2bea7588a303"
3646 );
3647 assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3648 }
3649
3650 #[test]
3651 fn safety_backup_ref_is_set_after_verification() {
3652 let status = KanidmRestoreStatus {
3653 phase: KanidmRestorePhase::SafetyBackup,
3654 safety_backup_job_name: Some("test-safety-backup".to_string()),
3655 safety_backup_ref: Some("safety-test-restore".to_string()),
3656 safety_backup_expected_backup_id: Some(
3657 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3658 ),
3659 safety_backup_manifest_key: Some(
3660 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3661 ),
3662 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3663 ..Default::default()
3664 };
3665 assert!(status.safety_backup_ref.is_some());
3666 assert!(
3667 status
3668 .safety_backup_ref
3669 .as_ref()
3670 .unwrap()
3671 .starts_with("safety-")
3672 );
3673 assert!(status.safety_backup_expected_backup_id.is_some());
3674 assert!(status.safety_backup_manifest_key.is_some());
3675 assert!(status.safety_backup_payload_sha256.is_some());
3676 }
3677
3678 #[test]
3679 fn database_mutation_not_started_without_safety_backup_ref() {
3680 let status = KanidmRestoreStatus {
3681 phase: KanidmRestorePhase::PreparingSource,
3682 safety_backup_ref: None,
3683 database_mutation_started: false,
3684 ..Default::default()
3685 };
3686 assert!(!status.database_mutation_started);
3687 assert!(status.safety_backup_ref.is_none());
3688 }
3689
3690 #[test]
3691 fn pre_boundary_failure_does_not_set_mutation_started() {
3692 let status = KanidmRestoreStatus {
3693 phase: KanidmRestorePhase::Failed,
3694 message: Some(
3695 "safety backup job failed; target restored to original state".to_string(),
3696 ),
3697 database_mutation_started: false,
3698 ..Default::default()
3699 };
3700 assert!(!status.database_mutation_started);
3701 }
3702
3703 #[test]
3704 fn shared_volume_constants_are_distinct() {
3705 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3706 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3707 }
3708
3709 #[test]
3710 fn validate_backup_compatibility_matching_versions_passes() {
3711 let source = KanidmRestoreSource {
3712 local: None,
3713 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3714 name: "backup-1".to_string(),
3715 }),
3716 };
3717 let restore = make_restore(source, None);
3718 let target = kanidm_with_version("1.10.4");
3719 let backup = kaniop_backup_core::crd::KanidmBackup {
3720 metadata: kube::api::ObjectMeta::default(),
3721 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3722 backup_id: "id".to_string(),
3723 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3724 name: "k".to_string(),
3725 uid: "uid".to_string(),
3726 },
3727 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3728 name: "repo".to_string(),
3729 },
3730 manifest_key: "key".to_string(),
3731 },
3732 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3733 kanidm_version: Some("1.10.4".to_string()),
3734 ..Default::default()
3735 }),
3736 };
3737 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3738 }
3739
3740 #[test]
3741 fn validate_backup_compatibility_version_mismatch_fails() {
3742 let source = KanidmRestoreSource {
3743 local: None,
3744 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3745 name: "backup-1".to_string(),
3746 }),
3747 };
3748 let restore = make_restore(source, None);
3749 let target = kanidm_with_version("1.11.0");
3750 let backup = kaniop_backup_core::crd::KanidmBackup {
3751 metadata: kube::api::ObjectMeta::default(),
3752 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3753 backup_id: "id".to_string(),
3754 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3755 name: "k".to_string(),
3756 uid: "uid".to_string(),
3757 },
3758 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3759 name: "repo".to_string(),
3760 },
3761 manifest_key: "key".to_string(),
3762 },
3763 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3764 kanidm_version: Some("1.10.4".to_string()),
3765 ..Default::default()
3766 }),
3767 };
3768 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3769 }
3770
3771 #[test]
3772 fn validate_backup_compatibility_no_backup_version_passes() {
3773 let source = KanidmRestoreSource {
3774 local: None,
3775 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3776 name: "backup-1".to_string(),
3777 }),
3778 };
3779 let restore = make_restore(source, None);
3780 let target = super::super::crd::Kanidm::default();
3781 let backup = kaniop_backup_core::crd::KanidmBackup {
3782 metadata: kube::api::ObjectMeta::default(),
3783 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3784 backup_id: "id".to_string(),
3785 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3786 name: "k".to_string(),
3787 uid: "uid".to_string(),
3788 },
3789 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3790 name: "repo".to_string(),
3791 },
3792 manifest_key: "key".to_string(),
3793 },
3794 status: None,
3795 };
3796 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3797 }
3798
3799 #[test]
3800 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3801 let source = KanidmRestoreSource {
3802 local: None,
3803 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3804 name: "backup-1".to_string(),
3805 }),
3806 };
3807 let mut restore = make_restore(source, None);
3808 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3809 let target = super::super::crd::Kanidm::default();
3810 let backup = kaniop_backup_core::crd::KanidmBackup {
3811 metadata: kube::api::ObjectMeta::default(),
3812 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3813 backup_id: "id".to_string(),
3814 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3815 name: "k".to_string(),
3816 uid: "uid".to_string(),
3817 },
3818 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3819 name: "repo".to_string(),
3820 },
3821 manifest_key: "key".to_string(),
3822 },
3823 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3824 image_digest: Some("sha256:originaldigest".to_string()),
3825 ..Default::default()
3826 }),
3827 };
3828 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3829 }
3830
3831 #[test]
3832 fn validate_backup_compatibility_image_digest_match_passes() {
3833 let source = KanidmRestoreSource {
3834 local: None,
3835 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3836 name: "backup-1".to_string(),
3837 }),
3838 };
3839 let mut restore = make_restore(source, None);
3840 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3841 let target = super::super::crd::Kanidm::default();
3842 let backup = kaniop_backup_core::crd::KanidmBackup {
3843 metadata: kube::api::ObjectMeta::default(),
3844 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3845 backup_id: "id".to_string(),
3846 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3847 name: "k".to_string(),
3848 uid: "uid".to_string(),
3849 },
3850 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3851 name: "repo".to_string(),
3852 },
3853 manifest_key: "key".to_string(),
3854 },
3855 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3856 image_digest: Some("sha256:abc123".to_string()),
3857 ..Default::default()
3858 }),
3859 };
3860 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3861 }
3862
3863 #[test]
3864 fn safety_backup_id_is_deterministic() {
3865 let source = KanidmRestoreSource {
3866 local: None,
3867 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3868 name: "backup-1".to_string(),
3869 }),
3870 };
3871 let restore = make_restore(source, None);
3872 let id1 = super::compute_safety_backup_id(&restore).unwrap();
3873 let id2 = super::compute_safety_backup_id(&restore).unwrap();
3874 assert_eq!(id1, id2);
3875 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3876 }
3877
3878 #[test]
3879 fn safety_manifest_key_is_deterministic() {
3880 let source = KanidmRestoreSource {
3881 local: None,
3882 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3883 name: "backup-1".to_string(),
3884 }),
3885 };
3886 let restore = make_restore(source, None);
3887 let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3888 let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3889 assert_eq!(key1, key2);
3890 assert!(key1.contains("manifest.json"));
3891 assert!(key1.starts_with("v1/tenants/"));
3892 }
3893
3894 #[test]
3895 fn safety_backup_id_survives_restart() {
3896 let source = KanidmRestoreSource {
3897 local: None,
3898 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3899 name: "backup-1".to_string(),
3900 }),
3901 };
3902 let mut restore = make_restore(source, None);
3903 let original_id = super::compute_safety_backup_id(&restore).unwrap();
3904 restore.status = Some(KanidmRestoreStatus {
3905 phase: KanidmRestorePhase::SafetyBackup,
3906 safety_backup_expected_backup_id: Some(original_id.clone()),
3907 ..Default::default()
3908 });
3909 let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3910 assert_eq!(original_id, recomputed_id);
3911 }
3912
3913 #[test]
3914 fn safety_backup_id_derived_from_uid_not_name() {
3915 let source = KanidmRestoreSource {
3916 local: None,
3917 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3918 name: "backup-1".to_string(),
3919 }),
3920 };
3921 let restore1 = make_restore(source.clone(), None);
3922 let mut restore2 = make_restore(source, None);
3923 restore2.metadata.name = Some("different-name-same-uid".to_string());
3924 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3925 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3926 assert_eq!(id1, id2);
3927 }
3928
3929 #[test]
3930 fn safety_backup_id_differs_for_different_uids() {
3931 let source = KanidmRestoreSource {
3932 local: None,
3933 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3934 name: "backup-1".to_string(),
3935 }),
3936 };
3937 let restore1 = make_restore(source.clone(), None);
3938 let mut restore2 = make_restore(source, None);
3939 restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3940 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3941 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3942 assert_ne!(id1, id2);
3943 }
3944
3945 #[test]
3946 fn safety_backup_id_errors_without_uid() {
3947 let source = KanidmRestoreSource {
3948 local: None,
3949 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3950 name: "backup-1".to_string(),
3951 }),
3952 };
3953 let mut restore = make_restore(source, None);
3954 restore.metadata.uid = None;
3955 assert!(super::compute_safety_backup_id(&restore).is_err());
3956 }
3957
3958 #[test]
3959 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3960 let source = KanidmRestoreSource {
3961 local: None,
3962 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3963 name: "backup-1".to_string(),
3964 }),
3965 };
3966 let restore = make_restore(source, None);
3967 assert!(super::requires_safety_backup(&restore));
3968 let status_without_ref = KanidmRestoreStatus {
3969 phase: KanidmRestorePhase::RestoringPrimary,
3970 safety_backup_ref: None,
3971 database_mutation_started: false,
3972 ..Default::default()
3973 };
3974 assert!(status_without_ref.safety_backup_ref.is_none());
3975 assert!(!status_without_ref.database_mutation_started);
3976 }
3977
3978 #[test]
3979 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3980 let status_with_ref = KanidmRestoreStatus {
3981 phase: KanidmRestorePhase::RestoringPrimary,
3982 safety_backup_ref: Some("safety-test-restore".to_string()),
3983 safety_backup_manifest_key: Some(
3984 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3985 ),
3986 safety_backup_payload_sha256: Some("abc123".to_string()),
3987 database_mutation_started: false,
3988 ..Default::default()
3989 };
3990 assert!(status_with_ref.safety_backup_ref.is_some());
3991 assert!(!status_with_ref.database_mutation_started);
3992 }
3993
3994 #[test]
3995 fn result_document_verification_rejects_missing_backup_id() {
3996 let result_json = r#"{
3997 "apiVersion": "backup.kaniop.rs/v1alpha1",
3998 "kind": "ResultDocument",
3999 "operation": "upload",
4000 "success": true,
4001 "exitCode": "success",
4002 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4003 "payloadSha256": "abc123"
4004 }"#;
4005 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4006 assert!(result.backup_id.is_none());
4007 }
4008
4009 #[test]
4010 fn result_document_verification_rejects_missing_manifest_key() {
4011 let result_json = r#"{
4012 "apiVersion": "backup.kaniop.rs/v1alpha1",
4013 "kind": "ResultDocument",
4014 "operation": "upload",
4015 "success": true,
4016 "exitCode": "success",
4017 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4018 "payloadSha256": "abc123"
4019 }"#;
4020 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021 assert!(result.manifest_key.is_none());
4022 }
4023
4024 #[test]
4025 fn result_document_verification_rejects_missing_payload_sha256() {
4026 let result_json = r#"{
4027 "apiVersion": "backup.kaniop.rs/v1alpha1",
4028 "kind": "ResultDocument",
4029 "operation": "upload",
4030 "success": true,
4031 "exitCode": "success",
4032 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4033 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4034 }"#;
4035 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4036 assert!(result.payload_sha256.is_none());
4037 }
4038
4039 #[test]
4040 fn result_document_verification_rejects_failure_exit_code() {
4041 let result_json = r#"{
4042 "apiVersion": "backup.kaniop.rs/v1alpha1",
4043 "kind": "ResultDocument",
4044 "operation": "upload",
4045 "success": false,
4046 "exitCode": "integrity",
4047 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4048 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4049 "payloadSha256": "abc123"
4050 }"#;
4051 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4052 assert!(!result.success);
4053 assert_eq!(
4054 result.exit_code,
4055 kaniop_backup_core::result::ExitCode::Integrity
4056 );
4057 }
4058
4059 #[test]
4060 fn result_document_verification_rejects_wrong_operation() {
4061 let result_json = r#"{
4062 "apiVersion": "backup.kaniop.rs/v1alpha1",
4063 "kind": "ResultDocument",
4064 "operation": "download",
4065 "success": true,
4066 "exitCode": "success",
4067 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4068 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4069 "payloadSha256": "abc123"
4070 }"#;
4071 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4072 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4073 }
4074
4075 #[test]
4076 fn result_document_verification_accepts_valid_document() {
4077 let result_json = r#"{
4078 "apiVersion": "backup.kaniop.rs/v1alpha1",
4079 "kind": "ResultDocument",
4080 "operation": "upload",
4081 "success": true,
4082 "exitCode": "success",
4083 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4084 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4085 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4086 }"#;
4087 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4088 assert!(result.success);
4089 assert_eq!(
4090 result.exit_code,
4091 kaniop_backup_core::result::ExitCode::Success
4092 );
4093 assert_eq!(result.operation, "upload");
4094 assert!(result.backup_id.is_some());
4095 assert!(result.manifest_key.is_some());
4096 assert!(result.payload_sha256.is_some());
4097 }
4098
4099 #[test]
4100 fn termination_message_path_constant_is_valid() {
4101 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4102 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4103 }
4104
4105 #[test]
4106 fn status_tracks_safety_backup_evidence() {
4107 let status = KanidmRestoreStatus {
4108 phase: KanidmRestorePhase::PreparingSource,
4109 safety_backup_ref: Some("safety-test-restore".to_string()),
4110 safety_backup_expected_backup_id: Some(
4111 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4112 ),
4113 safety_backup_manifest_key: Some(
4114 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4115 ),
4116 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4117 ..Default::default()
4118 };
4119 assert!(status.safety_backup_ref.is_some());
4120 assert!(status.safety_backup_expected_backup_id.is_some());
4121 assert!(status.safety_backup_manifest_key.is_some());
4122 assert!(status.safety_backup_payload_sha256.is_some());
4123 }
4124
4125 #[test]
4126 fn fail_closed_semantics_preserved_on_result_parse_failure() {
4127 let invalid_json = "not valid json";
4128 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4129 assert!(result.is_err());
4130 }
4131
4132 #[test]
4133 fn fail_closed_semantics_preserved_on_oversized_result() {
4134 let oversized_json = format!(
4135 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4136 kaniop_backup_core::result::RESULT_DOC_VERSION,
4137 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4138 );
4139 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4140 assert!(result.is_err());
4141 }
4142
4143 #[test]
4144 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4145 let source = KanidmRestoreSource {
4146 local: None,
4147 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4148 name: "backup-1".to_string(),
4149 }),
4150 };
4151 let mut restore = make_restore(source, None);
4152 restore.metadata.namespace = Some("test-ns".to_string());
4153 let target = super::super::crd::Kanidm::default();
4154 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4155 let doc_str = super::build_safety_upload_operation_doc(
4156 &restore,
4157 &target,
4158 "test-backup-id",
4159 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4160 "my-bucket",
4161 "prod",
4162 "https://s3.example.com",
4163 "us-east-1",
4164 false,
4165 false,
4166 Some(&ca_path),
4167 None,
4168 )
4169 .unwrap();
4170 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4171 assert_eq!(parsed["caBundlePath"], ca_path);
4172 }
4173
4174 #[test]
4175 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4176 let source = KanidmRestoreSource {
4177 local: None,
4178 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4179 name: "backup-1".to_string(),
4180 }),
4181 };
4182 let mut restore = make_restore(source, None);
4183 restore.metadata.namespace = Some("test-ns".to_string());
4184 let target = super::super::crd::Kanidm::default();
4185 let doc_str = super::build_safety_upload_operation_doc(
4186 &restore,
4187 &target,
4188 "test-backup-id",
4189 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4190 "my-bucket",
4191 "prod",
4192 "https://s3.example.com",
4193 "us-east-1",
4194 false,
4195 false,
4196 None,
4197 None,
4198 )
4199 .unwrap();
4200 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4201 assert!(parsed["caBundlePath"].is_null());
4202 }
4203
4204 #[test]
4205 fn download_operation_doc_includes_ca_bundle_path_when_set() {
4206 let source = KanidmRestoreSource {
4207 local: None,
4208 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4209 name: "backup-1".to_string(),
4210 }),
4211 };
4212 let _restore = make_restore(source, None);
4213 let target = super::super::crd::Kanidm::default();
4214 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4215 let doc_str = super::build_download_operation_doc(
4216 &target,
4217 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4218 "019c7c76-f423-7a12-8f41-2bea7588a303",
4219 "source-uid-456",
4220 "real-bucket",
4221 "real-prefix",
4222 "https://real-endpoint.com",
4223 "eu-west-1",
4224 true,
4225 false,
4226 Some(&ca_path),
4227 None,
4228 );
4229 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4230 assert_eq!(parsed["caBundlePath"], ca_path);
4231 }
4232
4233 #[test]
4234 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4235 let source = KanidmRestoreSource {
4236 local: None,
4237 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4238 name: "backup-1".to_string(),
4239 }),
4240 };
4241 let _restore = make_restore(source, None);
4242 let target = super::super::crd::Kanidm::default();
4243 let doc_str = super::build_download_operation_doc(
4244 &target,
4245 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4246 "019c7c76-f423-7a12-8f41-2bea7588a303",
4247 "source-uid-456",
4248 "real-bucket",
4249 "real-prefix",
4250 "https://real-endpoint.com",
4251 "eu-west-1",
4252 true,
4253 false,
4254 None,
4255 None,
4256 );
4257 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4258 assert!(parsed["caBundlePath"].is_null());
4259 }
4260
4261 #[test]
4262 fn source_prep_result_operation_is_download() {
4263 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4264 assert_ne!(
4265 super::SOURCE_PREP_RESULT_OPERATION,
4266 super::SAFETY_BACKUP_RESULT_OPERATION
4267 );
4268 }
4269
4270 #[test]
4271 fn download_result_document_verification_accepts_valid_document() {
4272 let result_json = r#"{
4273 "apiVersion": "backup.kaniop.rs/v1alpha1",
4274 "kind": "ResultDocument",
4275 "operation": "download",
4276 "success": true,
4277 "exitCode": "success",
4278 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4279 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4280 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4281 }"#;
4282 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4283 assert!(result.success);
4284 assert_eq!(
4285 result.exit_code,
4286 kaniop_backup_core::result::ExitCode::Success
4287 );
4288 assert_eq!(result.operation, "download");
4289 assert_eq!(
4290 result.backup_id.as_deref(),
4291 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4292 );
4293 assert!(result.manifest_key.is_some());
4294 assert!(result.payload_sha256.is_some());
4295 }
4296
4297 #[test]
4298 fn download_result_document_rejects_wrong_operation() {
4299 let result_json = r#"{
4300 "apiVersion": "backup.kaniop.rs/v1alpha1",
4301 "kind": "ResultDocument",
4302 "operation": "upload",
4303 "success": true,
4304 "exitCode": "success",
4305 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4306 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4307 "payloadSha256": "abc123"
4308 }"#;
4309 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4310 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4311 }
4312
4313 #[test]
4314 fn download_result_document_rejects_failure_success_field() {
4315 let result_json = r#"{
4316 "apiVersion": "backup.kaniop.rs/v1alpha1",
4317 "kind": "ResultDocument",
4318 "operation": "download",
4319 "success": false,
4320 "exitCode": "retryable",
4321 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4322 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4323 "payloadSha256": "abc123"
4324 }"#;
4325 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4326 assert!(!result.success);
4327 assert_ne!(
4328 result.exit_code,
4329 kaniop_backup_core::result::ExitCode::Success
4330 );
4331 }
4332
4333 #[test]
4334 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4335 let status = KanidmRestoreStatus {
4336 phase: KanidmRestorePhase::PreparingSource,
4337 database_mutation_started: false,
4338 ..Default::default()
4339 };
4340 assert!(!status.database_mutation_started);
4341 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4342 }
4343
4344 #[test]
4345 fn backup_job_volume_size_default_is_10gi() {
4346 let qty = crate::controller::backup_job_volume_size();
4347 assert_eq!(qty, Quantity("10Gi".to_string()));
4348 }
4349
4350 #[test]
4351 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4352 use kaniop_backup_core::auth::build_encryption_env_vars;
4353 use kaniop_backup_core::crd::SecretRef;
4354
4355 let key_ref = SecretRef {
4356 name: "kek-secret".to_string(),
4357 };
4358 let env_vars = build_encryption_env_vars(Some(&key_ref));
4359 assert_eq!(env_vars.len(), 1);
4360 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4361 let secret_ref = env_vars[0]
4362 .value_from
4363 .as_ref()
4364 .unwrap()
4365 .secret_key_ref
4366 .as_ref()
4367 .unwrap();
4368 assert_eq!(secret_ref.name, "kek-secret");
4369 assert_eq!(secret_ref.key, "encryption-key");
4370 }
4371
4372 #[test]
4373 fn encryption_env_vars_absent_when_no_key_ref() {
4374 use kaniop_backup_core::auth::build_encryption_env_vars;
4375
4376 let env_vars = build_encryption_env_vars(None);
4377 assert!(
4378 env_vars.is_empty(),
4379 "encryption env vars must be empty when key_ref is None"
4380 );
4381 }
4382
4383 #[test]
4384 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4385 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4386
4387 let source = KanidmRestoreSource {
4388 local: None,
4389 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4390 name: "backup-1".to_string(),
4391 }),
4392 };
4393 let mut restore = make_restore(source, None);
4394 restore.metadata.namespace = Some("test-ns".to_string());
4395 let target = super::super::crd::Kanidm::default();
4396 let encryption = RepositoryEncryption {
4397 mode: EncryptionMode::ClientSide,
4398 key_id: None,
4399 key_ref: Some(SecretRef {
4400 name: "kek-secret".to_string(),
4401 }),
4402 };
4403 let doc_str = super::build_safety_upload_operation_doc(
4404 &restore,
4405 &target,
4406 "test-backup-id",
4407 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4408 "my-bucket",
4409 "prod",
4410 "https://s3.example.com",
4411 "us-east-1",
4412 false,
4413 false,
4414 None,
4415 Some(&encryption),
4416 )
4417 .unwrap();
4418 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4419 assert_eq!(parsed["encryptionMode"], "clientSide");
4420 assert!(parsed["encryptionKeyId"].is_null());
4421 }
4422
4423 #[test]
4424 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4425 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4426
4427 let source = KanidmRestoreSource {
4428 local: None,
4429 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4430 name: "backup-1".to_string(),
4431 }),
4432 };
4433 let _restore = make_restore(source, None);
4434 let target = super::super::crd::Kanidm::default();
4435 let encryption = RepositoryEncryption {
4436 mode: EncryptionMode::ProviderKms,
4437 key_id: Some("alias/kaniop-backups".to_string()),
4438 key_ref: None,
4439 };
4440 let doc_str = super::build_download_operation_doc(
4441 &target,
4442 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4443 "019c7c76-f423-7a12-8f41-2bea7588a303",
4444 "source-uid-456",
4445 "real-bucket",
4446 "real-prefix",
4447 "https://real-endpoint.com",
4448 "eu-west-1",
4449 true,
4450 false,
4451 None,
4452 Some(&encryption),
4453 );
4454 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4455 assert_eq!(parsed["encryptionMode"], "providerKms");
4456 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4457 }
4458
4459 #[test]
4460 fn post_mutation_failure_records_job_names() {
4461 let existing = KanidmRestoreStatus {
4462 phase: KanidmRestorePhase::RestoringPrimary,
4463 database_mutation_started: true,
4464 ..Default::default()
4465 };
4466
4467 let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4468 assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4469 assert_eq!(
4470 restore_failed.restore_job_name.as_deref(),
4471 Some("my-restore-job")
4472 );
4473 assert!(restore_failed.verify_job_name.is_none());
4474 assert!(restore_failed.database_mutation_started);
4475 assert_eq!(
4476 restore_failed.message.as_deref(),
4477 Some("database restore job failed")
4478 );
4479
4480 let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4481 assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4482 assert_eq!(
4483 verify_failed.verify_job_name.as_deref(),
4484 Some("my-verify-job")
4485 );
4486 assert!(verify_failed.restore_job_name.is_none());
4487 assert!(verify_failed.database_mutation_started);
4488 assert_eq!(
4489 verify_failed.message.as_deref(),
4490 Some("database verification failed")
4491 );
4492 }
4493
4494 #[test]
4495 fn restore_and_verify_job_names_are_deterministic() {
4496 let source = KanidmRestoreSource {
4497 local: Some(super::KanidmRestoreLocalSource {
4498 file_name: "backup.json".to_string(),
4499 }),
4500 backup_ref: None,
4501 };
4502 let restore = make_restore(source, None);
4503 assert_eq!(restore_job_name(&restore), "test-restore-restore");
4504 assert_eq!(verify_job_name(&restore), "test-restore-verify");
4505 }
4506}