1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78 feature = "schemars",
79 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82 category = "kaniop",
83 group = "kaniop.rs",
84 version = "v1beta1",
85 kind = "KanidmRestore",
86 plural = "kanidmrestores",
87 singular = "kanidmrestore",
88 shortname = "idmrestore",
89 namespaced,
90 status = "KanidmRestoreStatus",
91 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97 pub target_ref: KanidmRestoreTargetRef,
98 pub source: KanidmRestoreSource,
99 pub restore_image: String,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108 pub name: String,
109 pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115 feature = "schemars",
116 schemars(extend("x-kubernetes-validations" = [
117 {
118 "message": "exactly one of local or backupRef must be set",
119 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120 }
121 ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125 #[serde(skip_serializing_if = "Option::is_none")]
127 pub local: Option<KanidmRestoreLocalSource>,
128 #[serde(skip_serializing_if = "Option::is_none")]
130 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137 pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145 pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153 #[serde(skip_serializing_if = "Option::is_none")]
155 pub repository_ref: Option<SafetyBackupRepositoryRef>,
156 #[serde(default)]
158 pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165 pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171 #[default]
172 Pending,
173 Validating,
174 Quiescing,
175 SafetyBackup,
176 PreparingSource,
177 RestoringPrimary,
178 Verifying,
179 RebuildingReplicas,
180 Resuming,
181 Completed,
182 Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_generation: Option<i64>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_target_uid: Option<String>,
193 #[serde(default)]
194 pub phase: KanidmRestorePhase,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub restore_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub verify_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub safety_backup_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub source_prep_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_ref: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_expected_backup_id: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_manifest_key: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_payload_sha256: Option<String>,
211 #[serde(default)]
212 pub replicas_cleared: bool,
213 #[serde(default)]
214 pub certificates_cleared: bool,
215 #[serde(default)]
216 pub database_mutation_started: bool,
217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
218 pub original_replicas: Vec<ReplicaCountEntry>,
219 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220 pub phase_timestamps: std::collections::BTreeMap<String, String>,
221 #[serde(default, skip_serializing_if = "Vec::is_empty")]
222 #[cfg_attr(
223 feature = "schemars",
224 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225 )]
226 pub conditions: Vec<Condition>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235 pub group: String,
236 pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241 attempts: Counter<u64>,
242 outcomes: Counter<u64>,
243 duration_seconds: Histogram<f64>,
244 start_timestamp_seconds: Gauge<i64>,
245 break_glass_total: Counter<u64>,
246 disaster_recovery_total: Counter<u64>,
247 safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251 fn new() -> Self {
252 let meter = global::meter("kaniop");
253 Self {
254 attempts: meter
255 .u64_counter("restore_attempts")
256 .with_description("Number of Kanidm restore attempts started")
257 .build(),
258 outcomes: meter
259 .u64_counter("restore_outcomes")
260 .with_description("Number of terminal Kanidm restore outcomes")
261 .build(),
262 duration_seconds: meter
263 .f64_histogram("restore_duration_seconds")
264 .with_description("Kanidm restore duration from object creation to terminal phase")
265 .with_unit("s")
266 .build(),
267 start_timestamp_seconds: meter
268 .i64_gauge("restore_start_timestamp_seconds")
269 .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270 .with_unit("s")
271 .build(),
272 break_glass_total: meter
273 .u64_counter("restore_break_glass")
274 .with_description("Number of break-glass safety backup overrides")
275 .build(),
276 disaster_recovery_total: meter
277 .u64_counter("restore_disaster_recovery")
278 .with_description("Number of approved disaster-recovery source identity overrides")
279 .build(),
280 safety_backup_duration_seconds: meter
281 .f64_histogram("restore_safety_backup_duration_seconds")
282 .with_description("Duration of pre-restore safety backup creation")
283 .with_unit("s")
284 .build(),
285 }
286 }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290 let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291 let active = restore.metadata.deletion_timestamp.is_none()
292 && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293 let timestamp = if active {
294 restore
295 .metadata
296 .creation_timestamp
297 .as_ref()
298 .map(|created| created.0.as_second())
299 .unwrap_or_else(|| Timestamp::now().as_second())
300 } else {
301 0
302 };
303 let attributes = [
304 KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305 KeyValue::new("restore", restore.name_any()),
306 ];
307 metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312 client: Client,
313 recorder: Recorder,
314 metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319 match phase {
320 KanidmRestorePhase::Pending => {
321 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322 Ok(Action::requeue(REQUEUE))
323 }
324 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325 Ok((target, disaster_recovery_source)) => {
326 mark_restoring(&restore, &target, &ctx).await?;
327 if let Some(source) = disaster_recovery_source.as_ref() {
328 record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329 }
330 let mut status = restore.status.clone().unwrap_or_default();
331 if let Some(source) = disaster_recovery_source.as_ref() {
332 let target_uid = target.uid().unwrap_or_default();
333 let condition = restore_condition(
334 &status.conditions,
335 CONDITION_DISASTER_RECOVERY,
336 CONDITION_TRUE,
337 "ApprovedSourceIdentityMismatch",
338 &format!(
339 "Approved disaster recovery from source {}/{} into target {}/{}",
340 source.name,
341 source.uid,
342 target.name_any(),
343 target_uid
344 ),
345 restore.metadata.generation,
346 );
347 status
348 .conditions
349 .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350 status.conditions.push(condition);
351 }
352 status.observed_target_uid = target.uid();
353 status.original_replicas = capture_original_replicas(&target);
354 status.phase = KanidmRestorePhase::Quiescing;
355 status.message = None;
356 patch_status(&restore, &ctx, status).await?;
357 Ok(Action::requeue(REQUEUE))
358 }
359 Err(error) => {
360 set_phase(
361 &restore,
362 &ctx,
363 KanidmRestorePhase::Failed,
364 Some(error.to_string()),
365 )
366 .await?;
367 Ok(Action::requeue(Duration::from_secs(300)))
368 }
369 },
370 KanidmRestorePhase::Quiescing => {
371 let target = get_target(&restore, &ctx).await?;
372 scale_all(&target, &ctx, 0).await?;
373 if target_pods_stopped(&target, &ctx).await?
374 && target_volumes_detached(&target, &ctx).await?
375 {
376 let next_phase = if requires_safety_backup(&restore) {
377 KanidmRestorePhase::SafetyBackup
378 } else {
379 record_break_glass(&restore, &ctx).await?;
380 KanidmRestorePhase::PreparingSource
381 };
382 if next_phase == KanidmRestorePhase::PreparingSource {
383 let refreshed = Api::<KanidmRestore>::namespaced(
384 ctx.client.clone(),
385 restore.namespace().as_deref().unwrap_or_default(),
386 )
387 .get(&restore.name_any())
388 .await
389 .map_err(|error| {
390 Error::kube_error(
391 "get",
392 "KanidmRestore",
393 restore.namespace().as_deref().unwrap_or_default(),
394 restore.name_any(),
395 error,
396 )
397 })?;
398 set_phase(&refreshed, &ctx, next_phase, None).await?;
399 } else {
400 set_phase(&restore, &ctx, next_phase, None).await?;
401 }
402 }
403 Ok(Action::requeue(REQUEUE))
404 }
405 KanidmRestorePhase::SafetyBackup => {
406 let target = get_target(&restore, &ctx).await?;
407 let name = safety_backup_job_name(&restore);
408 let expected_backup_id = compute_safety_backup_id(&restore)?;
409 let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410 ensure_safety_backup_job(
411 &restore,
412 &target,
413 &ctx,
414 &name,
415 &expected_backup_id,
416 &expected_manifest_key,
417 )
418 .await?;
419 let mut status = restore.status.clone().unwrap_or_default();
420 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421 match job_state(&restore, &ctx, &name).await? {
422 JobState::Complete => {
423 let result =
424 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425 match result {
426 Ok(verified) => {
427 status.safety_backup_job_name = Some(name);
428 status.safety_backup_ref =
429 Some(format!("safety-{}", restore.name_any()));
430 status.safety_backup_manifest_key = Some(verified.manifest_key);
431 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432 record_safety_backup_duration(&restore, &ctx).await;
433 status.phase = KanidmRestorePhase::PreparingSource;
434 status.message = None;
435 patch_status(&restore, &ctx, status).await?;
436 }
437 Err(error) => {
438 resume_before_mutation(&restore, &ctx).await?;
439 set_phase(
440 &restore,
441 &ctx,
442 KanidmRestorePhase::Failed,
443 Some(format!("safety backup result verification failed: {error}")),
444 )
445 .await?;
446 }
447 }
448 }
449 JobState::Failed => {
450 resume_before_mutation(&restore, &ctx).await?;
451 set_phase(
452 &restore,
453 &ctx,
454 KanidmRestorePhase::Failed,
455 Some(
456 "safety backup job failed; target restored to original state"
457 .to_string(),
458 ),
459 )
460 .await?;
461 }
462 JobState::Running => {
463 patch_status(&restore, &ctx, status).await?;
464 }
465 }
466 Ok(Action::requeue(REQUEUE))
467 }
468 KanidmRestorePhase::PreparingSource => {
469 if is_remote_source(&restore) {
470 let target = get_target(&restore, &ctx).await?;
471 let name = source_prep_job_name(&restore);
472 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473 match job_state(&restore, &ctx, &name).await? {
474 JobState::Complete => {
475 let backup_id = restore
476 .spec
477 .source
478 .backup_ref
479 .as_ref()
480 .map(|b| b.name.clone())
481 .unwrap_or_default();
482 let backup = {
483 let ns = restore.namespace().unwrap();
484 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485 .get(&backup_id)
486 .await
487 .map_err(|e| {
488 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489 })?
490 };
491 let expected_backup_id = &backup.spec.backup_id;
492 let result =
493 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494 .await;
495 match result {
496 Ok(verified) => {
497 if let Some(expected_sha256) = backup
498 .status
499 .as_ref()
500 .and_then(|s| s.payload_sha256.as_deref())
501 {
502 if verified.payload_sha256 != expected_sha256 {
503 resume_before_mutation(&restore, &ctx).await?;
504 set_phase(
505 &restore,
506 &ctx,
507 KanidmRestorePhase::Failed,
508 Some(format!(
509 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510 verified.payload_sha256
511 )),
512 )
513 .await?;
514 return Ok(Action::requeue(REQUEUE));
515 }
516 }
517 let mut status = restore.status.clone().unwrap_or_default();
518 status.source_prep_job_name = Some(name);
519 status.phase = KanidmRestorePhase::RestoringPrimary;
520 status.message = None;
521 patch_status(&restore, &ctx, status).await?;
522 }
523 Err(error) => {
524 resume_before_mutation(&restore, &ctx).await?;
525 set_phase(
526 &restore,
527 &ctx,
528 KanidmRestorePhase::Failed,
529 Some(format!(
530 "source preparation result verification failed: {error}"
531 )),
532 )
533 .await?;
534 }
535 }
536 }
537 JobState::Failed => {
538 resume_before_mutation(&restore, &ctx).await?;
539 set_phase(
540 &restore,
541 &ctx,
542 KanidmRestorePhase::Failed,
543 Some(
544 "source preparation job failed; target restored to original state"
545 .to_string(),
546 ),
547 )
548 .await?;
549 }
550 JobState::Running => {}
551 }
552 } else {
553 let target = get_target(&restore, &ctx).await?;
554 let name = source_check_job_name(&restore);
555 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556 match job_state(&restore, &ctx, &name).await? {
557 JobState::Complete => {
558 let mut status = restore.status.clone().unwrap_or_default();
559 status.source_prep_job_name = Some(name);
560 status.phase = KanidmRestorePhase::RestoringPrimary;
561 status.message = None;
562 patch_status(&restore, &ctx, status).await?;
563 }
564 JobState::Failed => {
565 resume_before_mutation(&restore, &ctx).await?;
566 set_phase(
567 &restore,
568 &ctx,
569 KanidmRestorePhase::Failed,
570 Some(
571 "local backup file check failed; target restored to original state"
572 .to_string(),
573 ),
574 )
575 .await?;
576 }
577 JobState::Running => {}
578 }
579 }
580 Ok(Action::requeue(REQUEUE))
581 }
582 KanidmRestorePhase::RestoringPrimary => {
583 let target = get_target(&restore, &ctx).await?;
584 let status = restore.status.as_ref().cloned().unwrap_or_default();
585 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586 return Err(Error::MissingData(
587 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588 ));
589 }
590 if !status.database_mutation_started {
591 let mut status = status;
592 status.database_mutation_started = true;
593 status.message = Some(
594 "database mutation boundary persisted; starting primary restore".to_string(),
595 );
596 patch_status(&restore, &ctx, status).await?;
597 return Ok(Action::requeue(REQUEUE));
598 }
599 let name = restore_job_name(&restore);
600 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601 match job_state(&restore, &ctx, &name).await? {
602 JobState::Complete => {
603 let mut status = restore.status.clone().unwrap_or_default();
604 status.restore_job_name = Some(name);
605 status.phase = KanidmRestorePhase::Verifying;
606 status.message = None;
607 patch_status(&restore, &ctx, status).await?;
608 }
609 JobState::Failed => {
610 let status = restore.status.clone().unwrap_or_default();
611 let failed_status = restore_primary_failed_status(&status, &name);
612 patch_status(&restore, &ctx, failed_status).await?;
613 }
614 JobState::Running => {}
615 }
616 Ok(Action::requeue(REQUEUE))
617 }
618 KanidmRestorePhase::Verifying => {
619 let target = get_target(&restore, &ctx).await?;
620 let name = verify_job_name(&restore);
621 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622 match job_state(&restore, &ctx, &name).await? {
623 JobState::Complete => {
624 let mut status = restore.status.clone().unwrap_or_default();
625 status.verify_job_name = Some(name);
626 status.phase = KanidmRestorePhase::RebuildingReplicas;
627 status.message = None;
628 patch_status(&restore, &ctx, status).await?;
629 }
630 JobState::Failed => {
631 let status = restore.status.clone().unwrap_or_default();
632 let failed_status = verifying_failed_status(&status, &name);
633 patch_status(&restore, &ctx, failed_status).await?;
634 }
635 JobState::Running => {}
636 }
637 Ok(Action::requeue(REQUEUE))
638 }
639 KanidmRestorePhase::RebuildingReplicas => {
640 let target = get_target(&restore, &ctx).await?;
641 let mut status = restore.status.clone().unwrap_or_default();
642 if !status.replicas_cleared {
643 if delete_secondary_pvcs(&target, &ctx).await? {
644 status.replicas_cleared = true;
645 status.message = Some("secondary database state cleared".to_string());
646 patch_status(&restore, &ctx, status).await?;
647 }
648 return Ok(Action::requeue(REQUEUE));
649 }
650
651 scale_primary(&target, &ctx, 1).await?;
652 if !primary_ready(&target, &ctx).await? {
653 return Ok(Action::requeue(REQUEUE));
654 }
655 if !status.certificates_cleared {
656 delete_replica_cert_secrets(&target, &ctx).await?;
657 delete_admin_secret(&target, &ctx).await?;
658 status.certificates_cleared = true;
659 status.message = Some(
660 "replica certificates and admin secret cleared for regeneration".to_string(),
661 );
662 patch_status(&restore, &ctx, status).await?;
663 return Ok(Action::requeue(REQUEUE));
664 }
665 status.phase = KanidmRestorePhase::Resuming;
666 status.message = None;
667 patch_status(&restore, &ctx, status).await?;
668 Ok(Action::requeue(REQUEUE))
669 }
670 KanidmRestorePhase::Resuming => {
671 let target = get_target(&restore, &ctx).await?;
672 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673 clear_restoring(&restore, &target, &ctx).await?;
674 Ok(Action::requeue(Duration::from_secs(3600)))
675 }
676 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677 Ok(Action::requeue(Duration::from_secs(3600)))
678 }
679 }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684 name: String,
685 uid: String,
686}
687
688async fn validate(
689 restore: &KanidmRestore,
690 ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692 let target = get_target(restore, ctx).await?;
693 let actual_uid = target
694 .uid()
695 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696 if actual_uid != restore.spec.target_ref.uid {
697 return Err(Error::MissingData(format!(
698 "target UID mismatch: expected {}, got {}",
699 restore.spec.target_ref.uid, actual_uid
700 )));
701 }
702 validate_source(restore)?;
703 let disaster_recovery_source = if is_remote_source(restore) {
704 validate_backup_ref(restore, &target, ctx).await?
705 } else {
706 None
707 };
708 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709 {
710 return Err(Error::MissingData(format!(
711 "restoreImage must be the target's pinned Kanidm image (target image is {})",
712 target.spec.image
713 )));
714 }
715 let storage = target
716 .spec
717 .storage
718 .as_ref()
719 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720 if storage.empty_dir.is_some()
721 || storage.ephemeral.is_some()
722 || storage.volume_claim_template.is_none()
723 {
724 return Err(Error::MissingData(
725 "restore requires PVC-backed Kanidm storage".to_string(),
726 ));
727 }
728 let primaries = target
729 .spec
730 .replica_groups
731 .iter()
732 .filter(|rg| rg.primary_node)
733 .count();
734 if primaries != 1 {
735 return Err(Error::MissingData(
736 "backup/restore requires exactly one primary replica group".to_string(),
737 ));
738 }
739 let ns = restore.namespace().unwrap();
740 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741 .list(&ListParams::default())
742 .await
743 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744 if restores.items.iter().any(|other| {
745 other.name_any() != restore.name_any()
746 && other.spec.target_ref.name == restore.spec.target_ref.name
747 && !matches!(
748 other.status.as_ref().map(|s| s.phase),
749 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750 )
751 }) {
752 return Err(Error::MissingData(
753 "another active restore targets this Kanidm".to_string(),
754 ));
755 }
756 validate_safety_backup_config(restore)?;
757 Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761 !name.is_empty()
762 && name != "."
763 && name != ".."
764 && !name.contains('/')
765 && !name.contains('\\')
766 && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770 image == "kanidm/server:latest"
771 || image.ends_with(":latest")
772 || (!image.contains('@')
773 && !image
774 .rsplit('/')
775 .next()
776 .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780 let ns = restore
781 .namespace()
782 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784 .get(&restore.spec.target_ref.name)
785 .await
786 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790 restore: &KanidmRestore,
791 target: &Kanidm,
792 ctx: &RestoreContext,
793) -> Result<()> {
794 let ns = target.namespace().unwrap();
795 let uid = restore
796 .uid()
797 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799 .patch(
800 &target.name_any(),
801 &PatchParams::default(),
802 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803 )
804 .await
805 .map(|_| ())
806 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810 restore: &KanidmRestore,
811 target: &Kanidm,
812 ctx: &RestoreContext,
813) -> Result<()> {
814 let ns = target.namespace().unwrap();
815 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816 return Ok(());
817 }
818 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819 .patch(
820 &target.name_any(),
821 &PatchParams::default(),
822 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823 )
824 .await
825 .map(|_| ())
826 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830 restore: &KanidmRestore,
831 ctx: &RestoreContext,
832 mut status: KanidmRestoreStatus,
833) -> Result<()> {
834 let ns = restore.namespace().unwrap();
835 let previous_phase = restore.status.as_ref().map(|current| current.phase);
836 let next_phase = status.phase;
837 status.observed_generation = restore.metadata.generation;
838 update_restore_conditions(&mut status, restore.metadata.generation);
839
840 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841 .patch_status(
842 &restore.name_any(),
843 &PatchParams::default(),
844 &Patch::Merge(json!({"status": &status})),
845 )
846 .await
847 .map_err(|e| {
848 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849 })?;
850
851 if previous_phase != Some(next_phase) {
852 record_restore_transition(
853 restore,
854 ctx,
855 previous_phase,
856 next_phase,
857 status.message.as_deref(),
858 )
859 .await;
860 }
861 Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865 let previous = status.conditions.clone();
866 let phase = status.phase;
867 let phase_reason = format!("{phase:?}");
868 let message = status
869 .message
870 .clone()
871 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872 let terminal_success = phase == KanidmRestorePhase::Completed;
873 let terminal_failure = phase == KanidmRestorePhase::Failed;
874 let progressing = !terminal_success && !terminal_failure;
875
876 let sticky_conditions: Vec<Condition> = previous
877 .iter()
878 .filter(|condition| {
879 condition.type_ == CONDITION_BREAK_GLASS
880 || condition.type_ == CONDITION_DISASTER_RECOVERY
881 })
882 .cloned()
883 .collect();
884 status.conditions = vec![
885 restore_condition(
886 &previous,
887 CONDITION_PROGRESSING,
888 if progressing {
889 CONDITION_TRUE
890 } else {
891 CONDITION_FALSE
892 },
893 &phase_reason,
894 &message,
895 generation,
896 ),
897 restore_condition(
898 &previous,
899 CONDITION_READY,
900 if terminal_success {
901 CONDITION_TRUE
902 } else {
903 CONDITION_FALSE
904 },
905 if terminal_success {
906 "RestoreCompleted"
907 } else {
908 "RestoreNotCompleted"
909 },
910 if terminal_success {
911 "Kanidm restore completed successfully."
912 } else {
913 "Kanidm restore has not completed successfully."
914 },
915 generation,
916 ),
917 restore_condition(
918 &previous,
919 CONDITION_FAILED,
920 if terminal_failure {
921 CONDITION_TRUE
922 } else {
923 CONDITION_FALSE
924 },
925 if terminal_failure {
926 "RestoreFailed"
927 } else {
928 "NoRestoreFailure"
929 },
930 if terminal_failure {
931 status
932 .message
933 .as_deref()
934 .unwrap_or("Kanidm restore failed.")
935 } else {
936 "No terminal restore failure has been recorded."
937 },
938 generation,
939 ),
940 ];
941 status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945 previous: &[Condition],
946 condition_type: &str,
947 condition_status: &str,
948 reason: &str,
949 message: &str,
950 generation: Option<i64>,
951) -> Condition {
952 let last_transition_time = previous
953 .iter()
954 .find(|condition| {
955 condition.type_ == condition_type
956 && condition.status == condition_status
957 && condition.reason == reason
958 })
959 .map(|condition| condition.last_transition_time.clone())
960 .unwrap_or_else(|| Time(Timestamp::now()));
961 Condition {
962 type_: condition_type.to_string(),
963 status: condition_status.to_string(),
964 reason: reason.to_string(),
965 message: message.to_string(),
966 last_transition_time,
967 observed_generation: generation,
968 }
969}
970
971async fn record_restore_transition(
972 restore: &KanidmRestore,
973 ctx: &RestoreContext,
974 previous_phase: Option<KanidmRestorePhase>,
975 phase: KanidmRestorePhase,
976 message: Option<&str>,
977) {
978 if phase == KanidmRestorePhase::Validating {
979 ctx.metrics.attempts.add(1, &[]);
980 }
981
982 let result = match phase {
983 KanidmRestorePhase::Completed => Some("success"),
984 KanidmRestorePhase::Failed => Some("failure"),
985 _ => None,
986 };
987 if let Some(result) = result {
988 let attributes = [
989 KeyValue::new("result", result),
990 KeyValue::new("phase", format!("{phase:?}")),
991 ];
992 ctx.metrics.outcomes.add(1, &attributes);
993 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995 ctx.metrics.duration_seconds.record(elapsed, &attributes);
996 }
997 }
998
999 let reason = if phase == KanidmRestorePhase::Failed {
1000 "RestoreFailed"
1001 } else {
1002 "RestorePhaseChanged"
1003 };
1004 let note = message.map(str::to_string).or_else(|| {
1005 Some(format!(
1006 "Kanidm restore phase changed from {} to {phase:?}.",
1007 previous_phase
1008 .map(|previous| format!("{previous:?}"))
1009 .unwrap_or_else(|| "None".to_string())
1010 ))
1011 });
1012 if let Err(error) = ctx
1013 .recorder
1014 .publish(
1015 &Event {
1016 type_: if phase == KanidmRestorePhase::Failed {
1017 EventType::Warning
1018 } else {
1019 EventType::Normal
1020 },
1021 reason: reason.to_string(),
1022 note,
1023 action: "Restore".to_string(),
1024 secondary: None,
1025 },
1026 &restore.object_ref(&()),
1027 )
1028 .await
1029 {
1030 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031 }
1032}
1033
1034async fn set_phase(
1035 restore: &KanidmRestore,
1036 ctx: &RestoreContext,
1037 phase: KanidmRestorePhase,
1038 message: Option<String>,
1039) -> Result<()> {
1040 let mut status = restore.status.clone().unwrap_or_default();
1041 status.phase = phase;
1042 status.message = message;
1043 patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047 restore: &KanidmRestore,
1048 ctx: &RestoreContext,
1049 message: &str,
1050) -> Result<()> {
1051 set_phase(
1052 restore,
1053 ctx,
1054 KanidmRestorePhase::Failed,
1055 Some(message.to_string()),
1056 )
1057 .await
1058}
1059
1060fn restore_primary_failed_status(
1061 existing: &KanidmRestoreStatus,
1062 job_name: &str,
1063) -> KanidmRestoreStatus {
1064 let mut status = existing.clone();
1065 status.restore_job_name = Some(job_name.to_string());
1066 status.phase = KanidmRestorePhase::Failed;
1067 status.message = Some("database restore job failed".to_string());
1068 status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072 let mut status = existing.clone();
1073 status.verify_job_name = Some(job_name.to_string());
1074 status.phase = KanidmRestorePhase::Failed;
1075 status.message = Some("database verification failed".to_string());
1076 status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080 target
1081 .spec
1082 .replica_groups
1083 .iter()
1084 .find(|rg| rg.primary_node)
1085 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089 let ns = target.namespace().unwrap();
1090 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091 for rg in &target.spec.replica_groups {
1092 let name = target.statefulset_name(&rg.name);
1093 api.patch(
1094 &name,
1095 &PatchParams::default(),
1096 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097 )
1098 .await
1099 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100 }
1101 Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105 let ns = target.namespace().unwrap();
1106 let rg = primary_group(target)?;
1107 let name = target.statefulset_name(&rg.name);
1108 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109 .patch(
1110 &name,
1111 &PatchParams::default(),
1112 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113 )
1114 .await
1115 .map(|_| ())
1116 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120 let ns = target.namespace().unwrap();
1121 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122 for rg in &target.spec.replica_groups {
1123 let name = target.statefulset_name(&rg.name);
1124 api.patch(
1125 &name,
1126 &PatchParams::default(),
1127 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128 )
1129 .await
1130 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131 }
1132 Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136 let ns = target.namespace().unwrap();
1137 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139 .await
1140 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141 Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145 let ns = target.namespace().unwrap();
1146 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147 let mut pv_names = BTreeSet::new();
1148 let primary_name = primary_pvc_name(target)?;
1149 let primary_pvc = pvc_api
1150 .get(&primary_name)
1151 .await
1152 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153 let primary_pv = primary_pvc
1154 .spec
1155 .as_ref()
1156 .and_then(|spec| spec.volume_name.clone())
1157 .ok_or_else(|| {
1158 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159 })?;
1160 pv_names.insert(primary_pv);
1161
1162 for rg in &target.spec.replica_groups {
1163 let sts = target.statefulset_name(&rg.name);
1164 for ordinal in 0..rg.replicas {
1165 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166 if name == primary_name {
1167 continue;
1168 }
1169 if let Some(pvc) = pvc_api
1170 .get_opt(&name)
1171 .await
1172 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174 {
1175 pv_names.insert(pv_name);
1176 }
1177 }
1178 }
1179
1180 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181 .list(&ListParams::default())
1182 .await
1183 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184 Ok(!attachments.items.iter().any(|attachment| {
1185 attachment
1186 .spec
1187 .source
1188 .persistent_volume_name
1189 .as_ref()
1190 .is_some_and(|pv_name| pv_names.contains(pv_name))
1191 }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195 let ns = target.namespace().unwrap();
1196 let name = target.statefulset_name(&primary_group(target)?.name);
1197 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198 .get(&name)
1199 .await
1200 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201 Ok(sts
1202 .status
1203 .as_ref()
1204 .and_then(|s| s.ready_replicas)
1205 .unwrap_or(0)
1206 >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210 let rg = primary_group(target)?;
1211 Ok(format!(
1212 "{DATA_VOLUME}-{}-0",
1213 target.statefulset_name(&rg.name)
1214 ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218 let ns = target.namespace().unwrap();
1219 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220 let mut all_absent = true;
1221 for rg in &target.spec.replica_groups {
1222 let sts = target.statefulset_name(&rg.name);
1223 for ordinal in 0..rg.replicas {
1224 if rg.primary_node && ordinal == 0 {
1225 continue;
1226 }
1227 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228 match api
1229 .get_opt(&name)
1230 .await
1231 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232 {
1233 None => {}
1234 Some(pvc) => {
1235 all_absent = false;
1236 if pvc.metadata.deletion_timestamp.is_none() {
1237 match api.delete(&name, &DeleteParams::default()).await {
1238 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239 Err(kube::Error::Api(status)) if status.code == 404 => {}
1240 Err(error) => {
1241 return Err(Error::kube_error(
1242 "delete",
1243 "PersistentVolumeClaim",
1244 &ns,
1245 &name,
1246 error,
1247 ));
1248 }
1249 }
1250 }
1251 }
1252 }
1253 }
1254 }
1255 Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259 let ns = target.namespace().unwrap();
1260 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261 let label_selector = format!(
1262 "{}={},{}={}",
1263 CLUSTER_LABEL,
1264 target.name_any(),
1265 SECRET_TYPE_LABEL,
1266 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267 );
1268 let secrets = api
1269 .list(&ListParams::default().labels(&label_selector))
1270 .await
1271 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272 let mut all_absent = secrets.items.is_empty();
1273 for secret in secrets.items {
1274 let name = secret.name_any();
1275 if secret.metadata.deletion_timestamp.is_none() {
1276 match api.delete(&name, &DeleteParams::default()).await {
1277 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278 Err(kube::Error::Api(status)) if status.code == 404 => {}
1279 Err(error) => {
1280 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281 }
1282 }
1283 all_absent = false;
1284 }
1285 }
1286 Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290 let ns = target.namespace().unwrap();
1291 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292 let name = format!("{}-admin-passwords", target.name_any());
1293 match api
1294 .get_opt(&name)
1295 .await
1296 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297 {
1298 None => Ok(true),
1299 Some(secret) => {
1300 if secret.metadata.deletion_timestamp.is_some() {
1301 return Ok(false);
1302 }
1303 match api.delete(&name, &DeleteParams::default()).await {
1304 Ok(_) => {
1305 debug!(secret = %name, "deleting stale admin secret");
1306 Ok(false)
1307 }
1308 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310 }
1311 }
1312 }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316 format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319 format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323 let origin = target
1324 .spec
1325 .origin
1326 .clone()
1327 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328 vec![
1329 EnvVar {
1330 name: "KANIDM_DB_PATH".to_string(),
1331 value: Some("/data/kanidm.db".to_string()),
1332 ..Default::default()
1333 },
1334 EnvVar {
1335 name: "KANIDM_DOMAIN".to_string(),
1336 value: Some(target.spec.domain.clone()),
1337 ..Default::default()
1338 },
1339 EnvVar {
1340 name: "KANIDM_ORIGIN".to_string(),
1341 value: Some(origin),
1342 ..Default::default()
1343 },
1344 EnvVar {
1345 name: "KANIDM_TLS_CHAIN".to_string(),
1346 value: Some(format!("{TLS_PATH}/tls.crt")),
1347 ..Default::default()
1348 },
1349 EnvVar {
1350 name: "KANIDM_TLS_KEY".to_string(),
1351 value: Some(format!("{TLS_PATH}/tls.key")),
1352 ..Default::default()
1353 },
1354 ]
1355}
1356
1357async fn ensure_database_job(
1358 restore: &KanidmRestore,
1359 target: &Kanidm,
1360 ctx: &RestoreContext,
1361 name: &str,
1362 verify: bool,
1363) -> Result<()> {
1364 let ns = restore.namespace().unwrap();
1365 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366 if api
1367 .get_opt(name)
1368 .await
1369 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370 .is_some()
1371 {
1372 return Ok(());
1373 }
1374 let mut command = vec![
1375 "kanidmd".to_string(),
1376 "database".to_string(),
1377 if verify { "backup" } else { "restore" }.to_string(),
1378 ];
1379 if verify {
1380 command.push("/verify/verification.json.gz".to_string());
1381 } else {
1382 if is_remote_source(restore) {
1383 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384 } else {
1385 let file_name = restore
1386 .spec
1387 .source
1388 .local
1389 .as_ref()
1390 .map(|l| l.file_name.clone())
1391 .unwrap_or_default();
1392 command.push(format!("{BACKUP_PATH}/{file_name}"));
1393 }
1394 }
1395 let job = Job {
1396 metadata: ObjectMeta {
1397 name: Some(name.to_string()),
1398 namespace: Some(ns.clone()),
1399 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400 ..Default::default()
1401 },
1402 spec: Some(JobSpec {
1403 backoff_limit: Some(0),
1404 template: PodTemplateSpec {
1405 metadata: Some(ObjectMeta::default()),
1406 spec: Some(PodSpec {
1407 automount_service_account_token: Some(false),
1408 restart_policy: Some("Never".to_string()),
1409 security_context: Some(source_job_pod_security_context(target)),
1410 containers: vec![Container {
1411 name: if verify { "verify" } else { "restore" }.to_string(),
1412 image: Some(restore.spec.restore_image.clone()),
1413 command: Some(command),
1414 env: Some(build_kanidm_db_env_vars(target)),
1415 security_context: Some(kanidm_job_security_context()),
1416 resources: Some(job_resource_requirements()),
1417 volume_mounts: Some(vec![
1418 VolumeMount {
1419 name: DATA_VOLUME.to_string(),
1420 mount_path: DATA_PATH.to_string(),
1421 ..Default::default()
1422 },
1423 VolumeMount {
1424 name: TLS_VOLUME.to_string(),
1425 mount_path: TLS_PATH.to_string(),
1426 read_only: Some(true),
1427 ..Default::default()
1428 },
1429 VolumeMount {
1430 name: "verification-output".to_string(),
1431 mount_path: "/verify".to_string(),
1432 ..Default::default()
1433 },
1434 ]),
1435 ..Default::default()
1436 }],
1437 volumes: Some(vec![
1438 Volume {
1439 name: DATA_VOLUME.to_string(),
1440 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441 claim_name: primary_pvc_name(target)?,
1442 read_only: Some(false),
1443 }),
1444 ..Default::default()
1445 },
1446 Volume {
1447 name: TLS_VOLUME.to_string(),
1448 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449 secret_name: Some(target.effective_tls_secret_name()),
1450 ..Default::default()
1451 }),
1452 ..Default::default()
1453 },
1454 Volume {
1455 name: "verification-output".to_string(),
1456 empty_dir: Some(Default::default()),
1457 ..Default::default()
1458 },
1459 ]),
1460 ..Default::default()
1461 }),
1462 },
1463 ..Default::default()
1464 }),
1465 ..Default::default()
1466 };
1467 api.create(&PostParams::default(), &job)
1468 .await
1469 .map(|_| ())
1470 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474 Running,
1475 Complete,
1476 Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480 let ns = restore.namespace().unwrap();
1481 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482 .get(name)
1483 .await
1484 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486 return Ok(JobState::Failed);
1487 }
1488 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489 return Ok(JobState::Complete);
1490 }
1491 Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495 restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499 let safety = restore.spec.safety_backup.as_ref();
1500 let skip = safety.is_some_and(|s| s.skip);
1501 !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505 let has_local = restore.spec.source.local.is_some();
1506 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507 if has_local && has_backup_ref {
1508 return Err(Error::MissingData(
1509 "source.local and source.backupRef are mutually exclusive".to_string(),
1510 ));
1511 }
1512 if !has_local && !has_backup_ref {
1513 return Err(Error::MissingData(
1514 "source must specify either local or backupRef".to_string(),
1515 ));
1516 }
1517 if let Some(local) = &restore.spec.source.local {
1518 if !safe_basename(&local.file_name) {
1519 return Err(Error::MissingData(
1520 "restore source fileName must be a safe basename".to_string(),
1521 ));
1522 }
1523 }
1524 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525 if backup_ref.name.is_empty() {
1526 return Err(Error::MissingData(
1527 "source.backupRef.name must not be empty".to_string(),
1528 ));
1529 }
1530 }
1531 Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535 restore
1536 .metadata
1537 .annotations
1538 .as_ref()
1539 .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540 .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544 let annotations = restore.metadata.annotations.as_ref();
1545 let reason = annotations
1546 .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547 .map(String::as_str)
1548 .filter(|value| !value.trim().is_empty());
1549 let approved_by = annotations
1550 .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551 .map(String::as_str)
1552 .filter(|value| !value.trim().is_empty());
1553
1554 if reason.is_none() || approved_by.is_none() {
1555 return Err(Error::MissingData(format!(
1556 "disaster-recovery restore requires non-empty {} and {} annotations",
1557 BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558 )));
1559 }
1560 Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564 restore: &KanidmRestore,
1565 target: &Kanidm,
1566 ctx: &RestoreContext,
1567 source: &DisasterRecoverySource,
1568) {
1569 ctx.metrics.disaster_recovery_total.add(1, &[]);
1570 let reason = restore
1571 .metadata
1572 .annotations
1573 .as_ref()
1574 .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575 .cloned()
1576 .unwrap_or_default();
1577 let approved_by = restore
1578 .metadata
1579 .annotations
1580 .as_ref()
1581 .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582 .cloned()
1583 .unwrap_or_default();
1584 let target_uid = target.uid().unwrap_or_default();
1585 let note = format!(
1586 "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587 source.name,
1588 source.uid,
1589 target.name_any(),
1590 target_uid,
1591 reason,
1592 approved_by
1593 );
1594 if let Err(error) = ctx
1595 .recorder
1596 .publish(
1597 &Event {
1598 type_: EventType::Warning,
1599 reason: "DisasterRecoveryOverride".to_string(),
1600 note: Some(note.clone()),
1601 action: "Restore".to_string(),
1602 secondary: None,
1603 },
1604 &restore.object_ref(&()),
1605 )
1606 .await
1607 {
1608 warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609 }
1610 warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614 let safety = restore.spec.safety_backup.as_ref();
1615 let skip = safety.is_some_and(|s| s.skip);
1616 if skip {
1617 let annotations = restore.metadata.annotations.as_ref();
1618 let reason = annotations
1619 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620 .map(|s| s.trim())
1621 .unwrap_or("");
1622 let approver = annotations
1623 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624 .map(|s| s.trim())
1625 .unwrap_or("");
1626 if reason.is_empty() {
1627 return Err(Error::MissingData(format!(
1628 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629 )));
1630 }
1631 if approver.is_empty() {
1632 return Err(Error::MissingData(format!(
1633 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634 )));
1635 }
1636 }
1637 if !skip {
1638 let has_repo = safety
1639 .and_then(|s| s.repository_ref.as_ref())
1640 .is_some_and(|r| !r.name.is_empty());
1641 if !has_repo {
1642 return Err(Error::MissingData(
1643 "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644 .to_string(),
1645 ));
1646 }
1647 }
1648 Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652 target
1653 .spec
1654 .replica_groups
1655 .iter()
1656 .map(|rg| ReplicaCountEntry {
1657 group: rg.name.clone(),
1658 replicas: rg.replicas,
1659 })
1660 .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664 if let Ok(target) = get_target(restore, ctx).await {
1665 let owns_maintenance =
1666 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667 if owns_maintenance {
1668 scale_desired(&target, ctx).await?;
1669 clear_restoring(restore, &target, ctx).await?;
1670 }
1671 }
1672 Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676 ctx.metrics.break_glass_total.add(1, &[]);
1677 let reason = restore
1678 .metadata
1679 .annotations
1680 .as_ref()
1681 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682 .cloned()
1683 .unwrap_or_default();
1684 let approver = restore
1685 .metadata
1686 .annotations
1687 .as_ref()
1688 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689 .cloned()
1690 .unwrap_or_default();
1691 let mut status = restore.status.clone().unwrap_or_default();
1692 let break_glass_condition = restore_condition(
1693 &status.conditions,
1694 CONDITION_BREAK_GLASS,
1695 CONDITION_TRUE,
1696 "BreakGlassOverride",
1697 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698 restore.metadata.generation,
1699 );
1700 status
1701 .conditions
1702 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703 status.conditions.push(break_glass_condition);
1704 patch_status(restore, ctx, status).await?;
1705 if let Err(error) = ctx
1706 .recorder
1707 .publish(
1708 &Event {
1709 type_: EventType::Warning,
1710 reason: "SafetyBackupSkipped".to_string(),
1711 note: Some(format!(
1712 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713 )),
1714 action: "Restore".to_string(),
1715 secondary: None,
1716 },
1717 &restore.object_ref(&()),
1718 )
1719 .await
1720 {
1721 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722 }
1723 warn!(
1724 restore = %restore.name_any(),
1725 reason = %reason,
1726 approver = %approver,
1727 "break-glass safety backup override activated"
1728 );
1729 Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733 let status = restore.status.as_ref().cloned().unwrap_or_default();
1734 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735 if let Ok(start_ts) = start.parse::<Timestamp>() {
1736 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737 ctx.metrics
1738 .safety_backup_duration_seconds
1739 .record(elapsed, &[]);
1740 }
1741 }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745 format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749 format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753 format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757 restore: &KanidmRestore,
1758 target: &Kanidm,
1759 ctx: &RestoreContext,
1760 name: &str,
1761) -> Result<()> {
1762 let ns = restore.namespace().unwrap();
1763 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764 if api
1765 .get_opt(name)
1766 .await
1767 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768 .is_some()
1769 {
1770 return Ok(());
1771 }
1772 let file_name = restore
1773 .spec
1774 .source
1775 .local
1776 .as_ref()
1777 .map(|l| l.file_name.clone())
1778 .unwrap_or_default();
1779 let operation_doc = serde_json::json!({
1780 "apiVersion": "backup.kaniop.rs/v1alpha1",
1781 "kind": "OperationDocument",
1782 "operation": "check",
1783 "path": format!("{BACKUP_PATH}/{file_name}"),
1784 "resultPath": "/run/kaniop-result/result.json",
1785 "format": "kanidmJsonGzip",
1786 })
1787 .to_string();
1788 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790 let data_mover_image = data_mover_image();
1791 let job = Job {
1792 metadata: ObjectMeta {
1793 name: Some(name.to_string()),
1794 namespace: Some(ns.clone()),
1795 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796 ..Default::default()
1797 },
1798 spec: Some(JobSpec {
1799 backoff_limit: Some(0),
1800 template: PodTemplateSpec {
1801 metadata: Some(ObjectMeta::default()),
1802 spec: Some(PodSpec {
1803 automount_service_account_token: Some(false),
1804 restart_policy: Some("Never".to_string()),
1805 security_context: Some(source_job_pod_security_context(target)),
1806 containers: vec![Container {
1807 name: "source-check".to_string(),
1808 image: Some(data_mover_image),
1809 command: Some(vec![
1810 "/bin/kaniop-data-mover".to_string(),
1811 "check".to_string(),
1812 "--operation-doc".to_string(),
1813 "/run/kaniop/operation.json".to_string(),
1814 ]),
1815 security_context: Some(hardened_security_context()),
1816 resources: Some(job_resource_requirements()),
1817 env: Some(vec![EnvVar {
1818 name: "RUST_LOG".to_string(),
1819 value: Some("info".to_string()),
1820 ..Default::default()
1821 }]),
1822 volume_mounts: Some(vec![
1823 VolumeMount {
1824 name: DATA_VOLUME.to_string(),
1825 mount_path: DATA_PATH.to_string(),
1826 read_only: Some(true),
1827 ..Default::default()
1828 },
1829 VolumeMount {
1830 name: "operation".to_string(),
1831 mount_path: "/run/kaniop".to_string(),
1832 read_only: Some(true),
1833 ..Default::default()
1834 },
1835 VolumeMount {
1836 name: "result".to_string(),
1837 mount_path: "/run/kaniop-result".to_string(),
1838 ..Default::default()
1839 },
1840 ]),
1841 ..Default::default()
1842 }],
1843 volumes: Some(vec![
1844 Volume {
1845 name: DATA_VOLUME.to_string(),
1846 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847 claim_name: primary_pvc_name(target)?,
1848 read_only: Some(true),
1849 }),
1850 ..Default::default()
1851 },
1852 Volume {
1853 name: "operation".to_string(),
1854 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855 name: operation_cm_name,
1856 ..Default::default()
1857 }),
1858 ..Default::default()
1859 },
1860 Volume {
1861 name: "result".to_string(),
1862 empty_dir: Some(Default::default()),
1863 ..Default::default()
1864 },
1865 ]),
1866 ..Default::default()
1867 }),
1868 },
1869 ..Default::default()
1870 }),
1871 ..Default::default()
1872 };
1873 api.create(&PostParams::default(), &job)
1874 .await
1875 .map(|_| ())
1876 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882 let ns = restore.namespace().unwrap_or_default();
1883 let uid = restore.uid().ok_or_else(|| {
1884 Error::MissingData(
1885 "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886 .to_string(),
1887 )
1888 })?;
1889 let namespace_uuid =
1890 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892 Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1903 let ns = restore.namespace().unwrap_or_default();
1904 let kanidm_uid = &restore.spec.target_ref.uid;
1905 let backup_id = compute_safety_backup_id(restore)?;
1906 Ok(format!(
1907 "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1908 ))
1909}
1910
1911struct VerifiedSafetyBackup {
1912 manifest_key: String,
1913 payload_sha256: String,
1914}
1915
1916async fn read_safety_backup_result(
1917 restore: &KanidmRestore,
1918 ctx: &RestoreContext,
1919 job_name: &str,
1920 expected_backup_id: &str,
1921) -> Result<VerifiedSafetyBackup> {
1922 let ns = restore.namespace().unwrap();
1923 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924 let pod_list = pods
1925 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926 .await
1927 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928 let pod = pod_list.items.first().ok_or_else(|| {
1929 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1930 })?;
1931 let pod_name = pod.name_any();
1932 let container_status = pod
1933 .status
1934 .as_ref()
1935 .and_then(|s| s.container_statuses.as_ref())
1936 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1937 .ok_or_else(|| {
1938 Error::MissingData(format!(
1939 "safety-upload container status not found in pod {pod_name}"
1940 ))
1941 })?;
1942 let termination_message = container_status
1943 .state
1944 .as_ref()
1945 .and_then(|state| state.terminated.as_ref())
1946 .and_then(|t| t.message.as_ref())
1947 .ok_or_else(|| {
1948 Error::MissingData(format!(
1949 "termination message not found in pod {pod_name}; result transport failed"
1950 ))
1951 })?;
1952 if termination_message.is_empty() {
1953 return Err(Error::MissingData(format!(
1954 "termination message is empty in pod {pod_name}; result document was not written"
1955 )));
1956 }
1957 let result_doc = parse_result_document(termination_message).map_err(|e| {
1958 Error::ParseError(format!(
1959 "failed to parse result document from pod {pod_name}: {e}"
1960 ))
1961 })?;
1962 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963 return Err(Error::ParseError(format!(
1964 "unsupported result document version: {}",
1965 result_doc.api_version
1966 )));
1967 }
1968 if result_doc.kind != "ResultDocument" {
1969 return Err(Error::ParseError(format!(
1970 "invalid result document kind: {}",
1971 result_doc.kind
1972 )));
1973 }
1974 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1975 return Err(Error::ParseError(format!(
1976 "result document operation mismatch: expected '{}', got '{}'",
1977 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1978 )));
1979 }
1980 if !result_doc.success {
1981 return Err(Error::ParseError(format!(
1982 "safety backup reported failure with exit code {:?}",
1983 result_doc.exit_code
1984 )));
1985 }
1986 if result_doc.exit_code != ExitCode::Success {
1987 return Err(Error::ParseError(format!(
1988 "safety backup exit code is not Success: {:?}",
1989 result_doc.exit_code
1990 )));
1991 }
1992 let doc_backup_id = result_doc
1993 .backup_id
1994 .as_deref()
1995 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996 if doc_backup_id != expected_backup_id {
1997 return Err(Error::ParseError(format!(
1998 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999 )));
2000 }
2001 let manifest_key = result_doc
2002 .manifest_key
2003 .clone()
2004 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005 if manifest_key.is_empty() {
2006 return Err(Error::ParseError(
2007 "result document manifestKey is empty".to_string(),
2008 ));
2009 }
2010 let payload_sha256 = result_doc
2011 .payload_sha256
2012 .clone()
2013 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014 if payload_sha256.is_empty() {
2015 return Err(Error::ParseError(
2016 "result document payloadSha256 is empty".to_string(),
2017 ));
2018 }
2019 info!(
2020 restore = %restore.name_any(),
2021 backup_id = %expected_backup_id,
2022 manifest_key = %manifest_key,
2023 "safety backup result verified"
2024 );
2025 Ok(VerifiedSafetyBackup {
2026 manifest_key,
2027 payload_sha256,
2028 })
2029}
2030
2031struct VerifiedSourcePrepResult {
2032 #[allow(dead_code)]
2033 manifest_key: String,
2034 payload_sha256: String,
2035}
2036
2037async fn read_source_prep_result(
2038 restore: &KanidmRestore,
2039 ctx: &RestoreContext,
2040 job_name: &str,
2041 expected_backup_id: &str,
2042) -> Result<VerifiedSourcePrepResult> {
2043 let ns = restore.namespace().unwrap();
2044 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2045 let pod_list = pods
2046 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2047 .await
2048 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2049 let pod = pod_list.items.first().ok_or_else(|| {
2050 Error::MissingData(format!("no pod found for source prep job {job_name}"))
2051 })?;
2052 let pod_name = pod.name_any();
2053 let container_status = pod
2054 .status
2055 .as_ref()
2056 .and_then(|s| s.container_statuses.as_ref())
2057 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2058 .ok_or_else(|| {
2059 Error::MissingData(format!(
2060 "source-prep container status not found in pod {pod_name}"
2061 ))
2062 })?;
2063 let termination_message = container_status
2064 .state
2065 .as_ref()
2066 .and_then(|state| state.terminated.as_ref())
2067 .and_then(|t| t.message.as_ref())
2068 .ok_or_else(|| {
2069 Error::MissingData(format!(
2070 "termination message not found in pod {pod_name}; result transport failed"
2071 ))
2072 })?;
2073 if termination_message.is_empty() {
2074 return Err(Error::MissingData(format!(
2075 "termination message is empty in pod {pod_name}; result document was not written"
2076 )));
2077 }
2078 let result_doc = parse_result_document(termination_message).map_err(|e| {
2079 Error::ParseError(format!(
2080 "failed to parse result document from pod {pod_name}: {e}"
2081 ))
2082 })?;
2083 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2084 return Err(Error::ParseError(format!(
2085 "unsupported result document version: {}",
2086 result_doc.api_version
2087 )));
2088 }
2089 if result_doc.kind != "ResultDocument" {
2090 return Err(Error::ParseError(format!(
2091 "invalid result document kind: {}",
2092 result_doc.kind
2093 )));
2094 }
2095 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2096 return Err(Error::ParseError(format!(
2097 "result document operation mismatch: expected '{}', got '{}'",
2098 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2099 )));
2100 }
2101 if !result_doc.success {
2102 return Err(Error::ParseError(format!(
2103 "source prep reported failure with exit code {:?}",
2104 result_doc.exit_code
2105 )));
2106 }
2107 if result_doc.exit_code != ExitCode::Success {
2108 return Err(Error::ParseError(format!(
2109 "source prep exit code is not Success: {:?}",
2110 result_doc.exit_code
2111 )));
2112 }
2113 let doc_backup_id = result_doc
2114 .backup_id
2115 .as_deref()
2116 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2117 if doc_backup_id != expected_backup_id {
2118 return Err(Error::ParseError(format!(
2119 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2120 )));
2121 }
2122 let manifest_key = result_doc
2123 .manifest_key
2124 .clone()
2125 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2126 if manifest_key.is_empty() {
2127 return Err(Error::ParseError(
2128 "result document manifestKey is empty".to_string(),
2129 ));
2130 }
2131 let payload_sha256 = result_doc
2132 .payload_sha256
2133 .clone()
2134 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2135 if payload_sha256.is_empty() {
2136 return Err(Error::ParseError(
2137 "result document payloadSha256 is empty".to_string(),
2138 ));
2139 }
2140 info!(
2141 restore = %restore.name_any(),
2142 backup_id = %expected_backup_id,
2143 manifest_key = %manifest_key,
2144 payload_sha256 = %payload_sha256,
2145 "source prep result verified"
2146 );
2147 Ok(VerifiedSourcePrepResult {
2148 manifest_key,
2149 payload_sha256,
2150 })
2151}
2152
2153fn source_job_pod_security_context(
2154 target: &Kanidm,
2155) -> k8s_openapi::api::core::v1::PodSecurityContext {
2156 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2157 security_context.seccomp_profile = Some(SeccompProfile {
2158 type_: "RuntimeDefault".to_string(),
2159 ..Default::default()
2160 });
2161 security_context
2162}
2163
2164fn hardened_security_context() -> SecurityContext {
2165 SecurityContext {
2166 allow_privilege_escalation: Some(false),
2167 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2168 drop: Some(vec!["ALL".to_string()]),
2169 ..Default::default()
2170 }),
2171 read_only_root_filesystem: Some(true),
2172 run_as_non_root: Some(true),
2173 seccomp_profile: Some(SeccompProfile {
2174 type_: "RuntimeDefault".to_string(),
2175 ..Default::default()
2176 }),
2177 ..Default::default()
2178 }
2179}
2180
2181fn kanidm_job_security_context() -> SecurityContext {
2182 SecurityContext {
2183 allow_privilege_escalation: Some(false),
2184 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2185 drop: Some(vec!["ALL".to_string()]),
2186 ..Default::default()
2187 }),
2188 seccomp_profile: Some(SeccompProfile {
2189 type_: "RuntimeDefault".to_string(),
2190 ..Default::default()
2191 }),
2192 ..Default::default()
2193 }
2194}
2195
2196fn job_resource_requirements() -> ResourceRequirements {
2197 ResourceRequirements {
2198 requests: Some(std::collections::BTreeMap::from([
2199 ("cpu".to_string(), Quantity("100m".to_string())),
2200 ("memory".to_string(), Quantity("128Mi".to_string())),
2201 ])),
2202 limits: Some(std::collections::BTreeMap::from([
2203 ("cpu".to_string(), Quantity("2".to_string())),
2204 ("memory".to_string(), Quantity("2Gi".to_string())),
2205 ])),
2206 ..Default::default()
2207 }
2208}
2209
2210async fn ensure_safety_backup_job(
2211 restore: &KanidmRestore,
2212 target: &Kanidm,
2213 ctx: &RestoreContext,
2214 name: &str,
2215 backup_id: &str,
2216 manifest_key: &str,
2217) -> Result<()> {
2218 let ns = restore.namespace().unwrap();
2219 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2220 if api
2221 .get_opt(name)
2222 .await
2223 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2224 .is_some()
2225 {
2226 return Ok(());
2227 }
2228 let safety = restore
2229 .spec
2230 .safety_backup
2231 .as_ref()
2232 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2233 let repo_ref = safety
2234 .repository_ref
2235 .as_ref()
2236 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2237 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2238 .get(&repo_ref.name)
2239 .await
2240 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2241 let _kanidm_uid = &restore.spec.target_ref.uid;
2242 let endpoint = &repo.spec.s3.endpoint;
2243 let region = &repo.spec.s3.region;
2244 let operation_doc = build_safety_upload_operation_doc(
2245 restore,
2246 target,
2247 backup_id,
2248 manifest_key,
2249 &repo.spec.s3.bucket,
2250 &repo.spec.s3.prefix,
2251 endpoint,
2252 region,
2253 repo.spec.s3.force_path_style,
2254 repo.spec.s3.insecure,
2255 repo.spec
2256 .s3
2257 .ca_bundle_ref
2258 .as_ref()
2259 .map(|_| ca_bundle_path())
2260 .as_deref(),
2261 repo.spec.encryption.as_ref(),
2262 )?;
2263 let operation_cm_name = format!("{name}-op");
2264 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2265 let auth_method = &repo.spec.authentication.writer;
2266 let data_mover_image = data_mover_image();
2267 let init_command = vec![
2268 "kanidmd".to_string(),
2269 "database".to_string(),
2270 "backup".to_string(),
2271 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2272 ];
2273 let upload_script = format!(
2274 r#"set -eu
2275/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2276RESULT_FILE="/run/kaniop-result/result.json"
2277TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2278if [ ! -f "$RESULT_FILE" ]; then
2279 echo "ERROR: result document not found at $RESULT_FILE"
2280 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2281 exit 1
2282fi
2283if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2284 echo "ERROR: safety backup upload reported failure"
2285 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2286 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2287 exit 1
2288fi
2289dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2290echo "safety backup upload completed successfully"
2291"#
2292 );
2293 let job = Job {
2294 metadata: ObjectMeta {
2295 name: Some(name.to_string()),
2296 namespace: Some(ns.clone()),
2297 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2298 ..Default::default()
2299 },
2300 spec: Some(JobSpec {
2301 backoff_limit: Some(0),
2302 template: PodTemplateSpec {
2303 metadata: Some(ObjectMeta::default()),
2304 spec: Some(PodSpec {
2305 automount_service_account_token: Some(false),
2306 restart_policy: Some("Never".to_string()),
2307 security_context: Some(source_job_pod_security_context(target)),
2308 init_containers: Some(vec![Container {
2309 name: "safety-backup".to_string(),
2310 image: Some(restore.spec.restore_image.clone()),
2311 command: Some(init_command),
2312 security_context: Some(SecurityContext {
2313 allow_privilege_escalation: Some(false),
2314 capabilities: Some(Capabilities {
2315 drop: Some(vec!["ALL".to_string()]),
2316 ..Default::default()
2317 }),
2318 read_only_root_filesystem: Some(true),
2319 seccomp_profile: Some(SeccompProfile {
2320 type_: "RuntimeDefault".to_string(),
2321 ..Default::default()
2322 }),
2323 ..Default::default()
2324 }),
2325 resources: Some(job_resource_requirements()),
2326 env: Some(build_kanidm_db_env_vars(target)),
2327 volume_mounts: Some(vec![
2328 VolumeMount {
2329 name: DATA_VOLUME.to_string(),
2330 mount_path: DATA_PATH.to_string(),
2331 read_only: Some(false),
2332 ..Default::default()
2333 },
2334 VolumeMount {
2335 name: TLS_VOLUME.to_string(),
2336 mount_path: TLS_PATH.to_string(),
2337 read_only: Some(true),
2338 ..Default::default()
2339 },
2340 VolumeMount {
2341 name: SHARED_VOLUME.to_string(),
2342 mount_path: SHARED_VOL_PATH.to_string(),
2343 ..Default::default()
2344 },
2345 ]),
2346 ..Default::default()
2347 }]),
2348 containers: vec![Container {
2349 name: "safety-upload".to_string(),
2350 image: Some(data_mover_image),
2351 command: Some(vec!["/bin/sh".to_string()]),
2352 args: Some(vec!["-c".to_string(), upload_script]),
2353 security_context: Some(hardened_security_context()),
2354 resources: Some(job_resource_requirements()),
2355 env: {
2356 let mut env_vars =
2357 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2358 env_vars.push(EnvVar {
2359 name: "RUST_LOG".to_string(),
2360 value: Some("info".to_string()),
2361 ..Default::default()
2362 });
2363 if repo.spec.s3.ca_bundle_ref.is_some() {
2364 env_vars.push(ca_bundle_env_var());
2365 }
2366 env_vars.extend(build_encryption_env_vars(
2367 repo.spec
2368 .encryption
2369 .as_ref()
2370 .and_then(|e| e.key_ref.as_ref()),
2371 ));
2372 Some(env_vars)
2373 },
2374 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2375 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2376 volume_mounts: {
2377 let mut mounts = vec![
2378 VolumeMount {
2379 name: SHARED_VOLUME.to_string(),
2380 mount_path: SHARED_VOL_PATH.to_string(),
2381 read_only: Some(true),
2382 ..Default::default()
2383 },
2384 VolumeMount {
2385 name: "operation".to_string(),
2386 mount_path: "/run/kaniop".to_string(),
2387 read_only: Some(true),
2388 ..Default::default()
2389 },
2390 VolumeMount {
2391 name: "result".to_string(),
2392 mount_path: "/run/kaniop-result".to_string(),
2393 ..Default::default()
2394 },
2395 ];
2396 mounts.extend(build_auth_volume_mounts(auth_method));
2397 if repo.spec.s3.ca_bundle_ref.is_some() {
2398 mounts.push(build_ca_bundle_volume_mount());
2399 }
2400 Some(mounts)
2401 },
2402 ..Default::default()
2403 }],
2404 volumes: {
2405 let mut vols = vec![
2406 Volume {
2407 name: DATA_VOLUME.to_string(),
2408 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2409 claim_name: primary_pvc_name(target)?,
2410 read_only: Some(false),
2411 }),
2412 ..Default::default()
2413 },
2414 Volume {
2415 name: TLS_VOLUME.to_string(),
2416 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2417 secret_name: Some(target.effective_tls_secret_name()),
2418 ..Default::default()
2419 }),
2420 ..Default::default()
2421 },
2422 Volume {
2423 name: SHARED_VOLUME.to_string(),
2424 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2425 size_limit: Some(crate::controller::backup_job_volume_size()),
2426 ..Default::default()
2427 }),
2428 ..Default::default()
2429 },
2430 Volume {
2431 name: "operation".to_string(),
2432 config_map: Some(ConfigMapVolumeSource {
2433 name: operation_cm_name,
2434 ..Default::default()
2435 }),
2436 ..Default::default()
2437 },
2438 Volume {
2439 name: "result".to_string(),
2440 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2441 size_limit: Some(Quantity("16Mi".to_string())),
2442 ..Default::default()
2443 }),
2444 ..Default::default()
2445 },
2446 ];
2447 vols.extend(build_auth_volumes(auth_method));
2448 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2449 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2450 }
2451 Some(vols)
2452 },
2453 ..Default::default()
2454 }),
2455 },
2456 ..Default::default()
2457 }),
2458 ..Default::default()
2459 };
2460 api.create(&PostParams::default(), &job)
2461 .await
2462 .map(|_| ())
2463 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2464}
2465
2466async fn ensure_source_prep_job(
2467 restore: &KanidmRestore,
2468 target: &Kanidm,
2469 ctx: &RestoreContext,
2470 name: &str,
2471) -> Result<()> {
2472 let ns = restore.namespace().unwrap();
2473 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2474 if api
2475 .get_opt(name)
2476 .await
2477 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2478 .is_some()
2479 {
2480 return Ok(());
2481 }
2482 let backup_name = restore
2483 .spec
2484 .source
2485 .backup_ref
2486 .as_ref()
2487 .map(|b| b.name.clone())
2488 .unwrap_or_default();
2489 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2490 .get(&backup_name)
2491 .await
2492 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2493 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2494 .get(&backup.spec.repository_ref.name)
2495 .await
2496 .map_err(|e| {
2497 Error::kube_error(
2498 "get",
2499 "KanidmBackupRepository",
2500 &ns,
2501 &backup.spec.repository_ref.name,
2502 e,
2503 )
2504 })?;
2505 let endpoint = &repo.spec.s3.endpoint;
2506 let region = &repo.spec.s3.region;
2507 let operation_doc = build_download_operation_doc(
2508 target,
2509 &backup.spec.manifest_key,
2510 &backup.spec.backup_id,
2511 &backup.spec.kanidm_ref.uid,
2512 &repo.spec.s3.bucket,
2513 &repo.spec.s3.prefix,
2514 endpoint,
2515 region,
2516 repo.spec.s3.force_path_style,
2517 repo.spec.s3.insecure,
2518 repo.spec
2519 .s3
2520 .ca_bundle_ref
2521 .as_ref()
2522 .map(|_| ca_bundle_path())
2523 .as_deref(),
2524 repo.spec.encryption.as_ref(),
2525 );
2526 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2527 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2528 let auth_method = &repo.spec.authentication.reader;
2529 let data_mover_image = data_mover_image();
2530 let download_script = format!(
2531 r#"set -eu
2532/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2533RESULT_FILE="/run/kaniop-result/result.json"
2534TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2535if [ ! -f "$RESULT_FILE" ]; then
2536 echo "ERROR: result document not found at $RESULT_FILE"
2537 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2538 exit 1
2539fi
2540if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2541 echo "ERROR: source preparation download reported failure"
2542 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2543 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2544 exit 1
2545fi
2546dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2547echo "source preparation download completed successfully"
2548"#
2549 );
2550 let job = Job {
2551 metadata: ObjectMeta {
2552 name: Some(name.to_string()),
2553 namespace: Some(ns.clone()),
2554 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2555 ..Default::default()
2556 },
2557 spec: Some(JobSpec {
2558 backoff_limit: Some(0),
2559 template: PodTemplateSpec {
2560 metadata: Some(ObjectMeta::default()),
2561 spec: Some(PodSpec {
2562 automount_service_account_token: Some(false),
2563 restart_policy: Some("Never".to_string()),
2564 security_context: Some(source_job_pod_security_context(target)),
2565 containers: vec![Container {
2566 name: "source-prep".to_string(),
2567 image: Some(data_mover_image),
2568 command: Some(vec!["/bin/sh".to_string()]),
2569 args: Some(vec!["-c".to_string(), download_script]),
2570 security_context: Some(hardened_security_context()),
2571 resources: Some(job_resource_requirements()),
2572 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2573 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2574 env: {
2575 let mut env_vars = build_auth_env_vars(
2576 auth_method,
2577 &backup.spec.repository_ref.name,
2578 AuthRole::Reader,
2579 );
2580 env_vars.push(EnvVar {
2581 name: "RUST_LOG".to_string(),
2582 value: Some("info".to_string()),
2583 ..Default::default()
2584 });
2585 if repo.spec.s3.ca_bundle_ref.is_some() {
2586 env_vars.push(ca_bundle_env_var());
2587 }
2588 env_vars.extend(build_encryption_env_vars(
2589 repo.spec
2590 .encryption
2591 .as_ref()
2592 .and_then(|e| e.key_ref.as_ref()),
2593 ));
2594 Some(env_vars)
2595 },
2596 volume_mounts: {
2597 let mut mounts = vec![
2598 VolumeMount {
2599 name: DATA_VOLUME.to_string(),
2600 mount_path: DATA_PATH.to_string(),
2601 ..Default::default()
2602 },
2603 VolumeMount {
2604 name: "operation".to_string(),
2605 mount_path: "/run/kaniop".to_string(),
2606 read_only: Some(true),
2607 ..Default::default()
2608 },
2609 VolumeMount {
2610 name: "result".to_string(),
2611 mount_path: "/run/kaniop-result".to_string(),
2612 ..Default::default()
2613 },
2614 ];
2615 mounts.extend(build_auth_volume_mounts(auth_method));
2616 if repo.spec.s3.ca_bundle_ref.is_some() {
2617 mounts.push(build_ca_bundle_volume_mount());
2618 }
2619 Some(mounts)
2620 },
2621 ..Default::default()
2622 }],
2623 volumes: {
2624 let mut vols = vec![
2625 Volume {
2626 name: DATA_VOLUME.to_string(),
2627 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2628 claim_name: primary_pvc_name(target)?,
2629 read_only: Some(false),
2630 }),
2631 ..Default::default()
2632 },
2633 Volume {
2634 name: "operation".to_string(),
2635 config_map: Some(ConfigMapVolumeSource {
2636 name: operation_cm_name,
2637 ..Default::default()
2638 }),
2639 ..Default::default()
2640 },
2641 Volume {
2642 name: "result".to_string(),
2643 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2644 size_limit: Some(Quantity("16Mi".to_string())),
2645 ..Default::default()
2646 }),
2647 ..Default::default()
2648 },
2649 ];
2650 vols.extend(build_auth_volumes(auth_method));
2651 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2652 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2653 }
2654 Some(vols)
2655 },
2656 ..Default::default()
2657 }),
2658 },
2659 ..Default::default()
2660 }),
2661 ..Default::default()
2662 };
2663 api.create(&PostParams::default(), &job)
2664 .await
2665 .map(|_| ())
2666 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2667}
2668
2669#[allow(clippy::too_many_arguments)]
2670fn build_safety_upload_operation_doc(
2671 restore: &KanidmRestore,
2672 target: &Kanidm,
2673 backup_id: &str,
2674 manifest_key: &str,
2675 bucket: &str,
2676 prefix: &str,
2677 endpoint: &str,
2678 region: &str,
2679 force_path_style: bool,
2680 insecure: bool,
2681 ca_bundle_path: Option<&str>,
2682 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2683) -> Result<String> {
2684 let enc_mode =
2685 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2686 let enc_key_id = encryption
2687 .and_then(|e| e.key_id.as_ref())
2688 .map(|s| serde_json::Value::String(s.clone()));
2689 let op = UploadOperation {
2690 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2691 bucket: bucket.to_string(),
2692 prefix: prefix.to_string(),
2693 endpoint: endpoint.to_string(),
2694 region: region.to_string(),
2695 force_path_style,
2696 insecure,
2697 ca_bundle_path: ca_bundle_path.map(str::to_string),
2698 backup_id: backup_id.to_string(),
2699 namespace_uid: restore.namespace().unwrap_or_default(),
2703 kanidm_uid: restore.spec.target_ref.uid.clone(),
2704 kanidm_name: restore.spec.target_ref.name.clone(),
2705 domain: target.spec.domain.clone(),
2706 kanidm_version: target
2707 .status
2708 .as_ref()
2709 .and_then(|s| s.version.as_ref())
2710 .map(|v| v.image_tag.clone())
2711 .unwrap_or_default(),
2712 image_digest: None,
2713 consistency: "kanidm-offline".to_string(),
2714 reason: "restore-safety".to_string(),
2715 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2716 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2717 result_path: "/run/kaniop-result/result.json".to_string(),
2718 max_concurrent_parts: 4,
2719 max_retries: 3,
2720 };
2721 let _ = manifest_key;
2722 let doc = OperationDocument {
2723 api_version: OPERATION_DOC_VERSION.to_string(),
2724 kind: "OperationDocument".to_string(),
2725 spec: OperationSpec::Upload(op),
2726 };
2727 serde_json::to_string(&doc)
2728 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2729}
2730
2731async fn validate_backup_ref(
2732 restore: &KanidmRestore,
2733 target: &Kanidm,
2734 ctx: &RestoreContext,
2735) -> Result<Option<DisasterRecoverySource>> {
2736 let backup_ref = restore
2737 .spec
2738 .source
2739 .backup_ref
2740 .as_ref()
2741 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2742 let ns = restore.namespace().unwrap();
2743 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2744 .get(&backup_ref.name)
2745 .await
2746 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2747 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2748 return Err(Error::MissingData(format!(
2749 "KanidmBackup '{}' is not Ready (phase: {:?})",
2750 backup_ref.name,
2751 backup.status.as_ref().map(|s| &s.phase)
2752 )));
2753 }
2754 let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2755 || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2756 let disaster_recovery_source = if source_identity_mismatch {
2757 if !disaster_recovery_requested(restore) {
2758 return Err(Error::MissingData(format!(
2759 "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2760 backup.spec.kanidm_ref.name,
2761 backup.spec.kanidm_ref.uid,
2762 restore.spec.target_ref.name,
2763 restore.spec.target_ref.uid,
2764 DISASTER_RECOVERY_ANNOTATION,
2765 BREAK_GLASS_REASON_ANNOTATION,
2766 BREAK_GLASS_APPROVED_BY_ANNOTATION,
2767 )));
2768 }
2769 validate_disaster_recovery_approval(restore)?;
2770 Some(DisasterRecoverySource {
2771 name: backup.spec.kanidm_ref.name.clone(),
2772 uid: backup.spec.kanidm_ref.uid.clone(),
2773 })
2774 } else {
2775 None
2776 };
2777 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2778 .get(&backup.spec.repository_ref.name)
2779 .await
2780 .map_err(|e| {
2781 Error::kube_error(
2782 "get",
2783 "KanidmBackupRepository",
2784 &ns,
2785 &backup.spec.repository_ref.name,
2786 e,
2787 )
2788 })?;
2789 if !has_accepted_condition(
2790 &repo
2791 .status
2792 .as_ref()
2793 .map(|s| &s.conditions)
2794 .cloned()
2795 .unwrap_or_default(),
2796 ) {
2797 return Err(Error::MissingData(format!(
2798 "KanidmBackupRepository '{}' configuration has not been accepted",
2799 backup.spec.repository_ref.name
2800 )));
2801 }
2802 validate_backup_compatibility(restore, target, &backup)?;
2803 Ok(disaster_recovery_source)
2804}
2805
2806fn has_accepted_condition(conditions: &[Condition]) -> bool {
2807 conditions
2808 .iter()
2809 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2810}
2811
2812fn validate_backup_compatibility(
2813 restore: &KanidmRestore,
2814 target: &Kanidm,
2815 backup: &KanidmBackup,
2816) -> Result<()> {
2817 let status = backup.status.as_ref();
2818 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2819 let target_version = target
2820 .status
2821 .as_ref()
2822 .and_then(|s| s.version.as_ref())
2823 .map(|v| v.image_tag.as_str());
2824 if let Some(tv) = target_version {
2825 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2826 return Err(Error::MissingData(format!(
2827 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2828 )));
2829 }
2830 }
2831 }
2832 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2833 if !backup_digest.is_empty()
2834 && restore.spec.restore_image.contains('@')
2835 && !restore.spec.restore_image.contains(backup_digest)
2836 {
2837 return Err(Error::MissingData(format!(
2838 "restore image digest does not match backup image digest '{backup_digest}'"
2839 )));
2840 }
2841 }
2842 Ok(())
2843}
2844
2845#[allow(clippy::too_many_arguments)]
2846fn build_download_operation_doc(
2847 target: &Kanidm,
2848 manifest_key: &str,
2849 expected_backup_id: &str,
2850 expected_source_uid: &str,
2851 bucket: &str,
2852 prefix: &str,
2853 endpoint: &str,
2854 region: &str,
2855 force_path_style: bool,
2856 insecure: bool,
2857 ca_bundle_path: Option<&str>,
2858 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2859) -> String {
2860 let enc_mode =
2861 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2862 let enc_key_id = encryption
2863 .and_then(|e| e.key_id.as_ref())
2864 .map(|s| serde_json::Value::String(s.clone()));
2865 serde_json::json!({
2866 "apiVersion": "backup.kaniop.rs/v1alpha1",
2867 "kind": "OperationDocument",
2868 "operation": "download",
2869 "manifestKey": manifest_key,
2870 "bucket": bucket,
2871 "prefix": prefix,
2872 "endpoint": endpoint,
2873 "region": region,
2874 "forcePathStyle": force_path_style,
2875 "insecure": insecure,
2876 "caBundlePath": ca_bundle_path,
2877 "expectedBackupId": expected_backup_id,
2878 "expectedKanidmUid": expected_source_uid,
2879 "expectedDomain": target.spec.domain,
2880 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2881 "resultPath": "/run/kaniop-result/result.json",
2882 "maxRetries": 3,
2883 "encryptionMode": enc_mode,
2884 "encryptionKeyId": enc_key_id,
2885 })
2886 .to_string()
2887}
2888
2889async fn ensure_operation_configmap(
2890 restore: &KanidmRestore,
2891 name: &str,
2892 data: &str,
2893 ns: &str,
2894 ctx: &RestoreContext,
2895) -> Result<()> {
2896 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2897 if api
2898 .get_opt(name)
2899 .await
2900 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2901 .is_some()
2902 {
2903 return Ok(());
2904 }
2905 let cm = ConfigMap {
2906 metadata: ObjectMeta {
2907 name: Some(name.to_string()),
2908 namespace: Some(ns.to_string()),
2909 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2910 ..Default::default()
2911 },
2912 data: Some(std::collections::BTreeMap::from([(
2913 "operation.json".to_string(),
2914 data.to_string(),
2915 )])),
2916 ..Default::default()
2917 };
2918 api.create(&PostParams::default(), &cm)
2919 .await
2920 .map(|_| ())
2921 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926 use super::{
2927 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2928 CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2929 DISASTER_RECOVERY_ANNOTATION,
2930 KanidmRestore, KanidmRestorePhase,
2931 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2932 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2933 disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2934 is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2935 restore_job_name, safe_basename, validate_disaster_recovery_approval,
2936 validate_safety_backup_config, validate_source, verify_job_name,
2937 };
2938 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2939 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2940 use k8s_openapi::jiff::Timestamp;
2941 use kube::api::ObjectMeta as ApiObjectMeta;
2942 use std::collections::BTreeMap;
2943
2944 fn make_restore(
2945 source: KanidmRestoreSource,
2946 safety: Option<SafetyBackupConfig>,
2947 ) -> KanidmRestore {
2948 KanidmRestore {
2949 metadata: ApiObjectMeta {
2950 name: Some("test-restore".to_string()),
2951 namespace: Some("default".to_string()),
2952 uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2953 ..Default::default()
2954 },
2955 spec: KanidmRestoreSpec {
2956 target_ref: KanidmRestoreTargetRef {
2957 name: "corp-idm".to_string(),
2958 uid: "test-uid-123".to_string(),
2959 },
2960 source,
2961 restore_image: "kanidm/server@sha256:abc".to_string(),
2962 safety_backup: safety,
2963 },
2964 status: None,
2965 }
2966 }
2967
2968 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2969 super::super::crd::Kanidm {
2970 status: Some(super::super::crd::KanidmStatus {
2971 available_replicas: 1,
2972 replicas: 1,
2973 unavailable_replicas: 0,
2974 updated_replicas: 1,
2975 replica_statuses: vec![],
2976 replica_column: "1/1".to_string(),
2977 secret_name: None,
2978 version: Some(super::super::crd::KanidmVersionStatus {
2979 image_tag: image_tag.to_string(),
2980 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2981 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2982 }),
2983 domain_appearance_image: None,
2984 mail_sender: None,
2985 conditions: None,
2986 }),
2987 ..Default::default()
2988 }
2989 }
2990
2991 #[test]
2992 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2993 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2994 }
2995
2996 #[test]
2997 fn completed_restore_sets_ready_condition() {
2998 let mut status = KanidmRestoreStatus {
2999 phase: KanidmRestorePhase::Completed,
3000 ..Default::default()
3001 };
3002 super::update_restore_conditions(&mut status, Some(7));
3003 assert!(status.conditions.iter().any(|condition| {
3004 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3005 }));
3006 assert!(!status.conditions.iter().any(|condition| {
3007 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3008 }));
3009 }
3010
3011 #[test]
3012 fn disaster_recovery_condition_survives_phase_updates() {
3013 let dr_condition = Condition {
3014 type_: CONDITION_DISASTER_RECOVERY.to_string(),
3015 status: CONDITION_TRUE.to_string(),
3016 reason: "ApprovedSourceIdentityMismatch".to_string(),
3017 message: "approved DR".to_string(),
3018 last_transition_time: Time(Timestamp::now()),
3019 observed_generation: Some(7),
3020 };
3021 let mut status = KanidmRestoreStatus {
3022 phase: KanidmRestorePhase::Quiescing,
3023 conditions: vec![dr_condition.clone()],
3024 ..Default::default()
3025 };
3026
3027 super::update_restore_conditions(&mut status, Some(8));
3028
3029 assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3030 }
3031
3032 #[test]
3033 fn failed_restore_sets_failed_condition() {
3034 let mut status = KanidmRestoreStatus {
3035 phase: KanidmRestorePhase::Failed,
3036 message: Some("verification failed".to_string()),
3037 ..Default::default()
3038 };
3039 super::update_restore_conditions(&mut status, Some(8));
3040 assert!(status.conditions.iter().any(|condition| {
3041 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3042 }));
3043 }
3044
3045 #[test]
3046 fn restore_filename_is_confined_to_backup_directory() {
3047 assert!(safe_basename("backup.json.gz"));
3048 assert!(!safe_basename("../backup.json.gz"));
3049 assert!(!safe_basename("a/b"));
3050 assert!(!safe_basename(""));
3051 }
3052
3053 #[test]
3054 fn restore_image_rejects_latest_and_unpinned_names() {
3055 assert!(mutable_image("kanidm/server:latest"));
3056 assert!(mutable_image("kanidm/server"));
3057 assert!(!mutable_image("kanidm/server:1.10.0"));
3058 assert!(!mutable_image("kanidm/server@sha256:abc"));
3059 }
3060
3061 #[test]
3062 fn source_validation_rejects_both_local_and_backup_ref() {
3063 let source = KanidmRestoreSource {
3064 local: Some(super::KanidmRestoreLocalSource {
3065 file_name: "backup.json".to_string(),
3066 }),
3067 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3068 name: "some-backup".to_string(),
3069 }),
3070 };
3071 let restore = make_restore(source, None);
3072 assert!(validate_source(&restore).is_err());
3073 }
3074
3075 #[test]
3076 fn source_validation_rejects_empty_source() {
3077 let source = KanidmRestoreSource {
3078 local: None,
3079 backup_ref: None,
3080 };
3081 let restore = make_restore(source, None);
3082 assert!(validate_source(&restore).is_err());
3083 }
3084
3085 #[test]
3086 fn source_validation_accepts_local_source() {
3087 let source = KanidmRestoreSource {
3088 local: Some(super::KanidmRestoreLocalSource {
3089 file_name: "backup.json".to_string(),
3090 }),
3091 backup_ref: None,
3092 };
3093 let restore = make_restore(source, None);
3094 assert!(validate_source(&restore).is_ok());
3095 }
3096
3097 #[test]
3098 fn source_validation_accepts_backup_ref_source() {
3099 let source = KanidmRestoreSource {
3100 local: None,
3101 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3102 name: "corp-idm-019c7c76".to_string(),
3103 }),
3104 };
3105 let restore = make_restore(source, None);
3106 assert!(validate_source(&restore).is_ok());
3107 }
3108
3109 #[test]
3110 fn source_validation_rejects_empty_backup_ref_name() {
3111 let source = KanidmRestoreSource {
3112 local: None,
3113 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3114 name: "".to_string(),
3115 }),
3116 };
3117 let restore = make_restore(source, None);
3118 assert!(validate_source(&restore).is_err());
3119 }
3120
3121 #[test]
3122 fn is_remote_source_true_for_backup_ref() {
3123 let source = KanidmRestoreSource {
3124 local: None,
3125 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3126 name: "backup-1".to_string(),
3127 }),
3128 };
3129 let restore = make_restore(source, None);
3130 assert!(is_remote_source(&restore));
3131 }
3132
3133 #[test]
3134 fn is_remote_source_false_for_local() {
3135 let source = KanidmRestoreSource {
3136 local: Some(super::KanidmRestoreLocalSource {
3137 file_name: "backup.json".to_string(),
3138 }),
3139 backup_ref: None,
3140 };
3141 let restore = make_restore(source, None);
3142 assert!(!is_remote_source(&restore));
3143 }
3144
3145 #[test]
3146 fn requires_safety_backup_default_true() {
3147 let source = KanidmRestoreSource {
3148 local: None,
3149 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3150 name: "backup-1".to_string(),
3151 }),
3152 };
3153 let restore = make_restore(source, None);
3154 assert!(requires_safety_backup(&restore));
3155 }
3156
3157 #[test]
3158 fn requires_safety_backup_false_when_skip() {
3159 let source = KanidmRestoreSource {
3160 local: None,
3161 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3162 name: "backup-1".to_string(),
3163 }),
3164 };
3165 let safety = SafetyBackupConfig {
3166 repository_ref: Some(SafetyBackupRepositoryRef {
3167 name: "offsite".to_string(),
3168 }),
3169 skip: true,
3170 };
3171 let restore = make_restore(source, Some(safety));
3172 assert!(!requires_safety_backup(&restore));
3173 }
3174
3175 #[test]
3176 fn break_glass_validation_rejects_skip_without_reason() {
3177 let source = KanidmRestoreSource {
3178 local: None,
3179 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3180 name: "backup-1".to_string(),
3181 }),
3182 };
3183 let safety = SafetyBackupConfig {
3184 repository_ref: Some(SafetyBackupRepositoryRef {
3185 name: "offsite".to_string(),
3186 }),
3187 skip: true,
3188 };
3189 let mut restore = make_restore(source, Some(safety));
3190 restore.metadata.annotations = Some(BTreeMap::from([(
3191 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3192 "commander@example.com".to_string(),
3193 )]));
3194 assert!(validate_safety_backup_config(&restore).is_err());
3195 }
3196
3197 #[test]
3198 fn break_glass_validation_rejects_skip_without_approver() {
3199 let source = KanidmRestoreSource {
3200 local: None,
3201 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3202 name: "backup-1".to_string(),
3203 }),
3204 };
3205 let safety = SafetyBackupConfig {
3206 repository_ref: Some(SafetyBackupRepositoryRef {
3207 name: "offsite".to_string(),
3208 }),
3209 skip: true,
3210 };
3211 let mut restore = make_restore(source, Some(safety));
3212 restore.metadata.annotations = Some(BTreeMap::from([(
3213 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3214 "PVC is unreadable".to_string(),
3215 )]));
3216 assert!(validate_safety_backup_config(&restore).is_err());
3217 }
3218
3219 #[test]
3220 fn break_glass_validation_accepts_valid_annotations() {
3221 let source = KanidmRestoreSource {
3222 local: None,
3223 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3224 name: "backup-1".to_string(),
3225 }),
3226 };
3227 let safety = SafetyBackupConfig {
3228 repository_ref: Some(SafetyBackupRepositoryRef {
3229 name: "offsite".to_string(),
3230 }),
3231 skip: true,
3232 };
3233 let mut restore = make_restore(source, Some(safety));
3234 restore.metadata.annotations = Some(BTreeMap::from([
3235 (
3236 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3237 "PVC is unreadable".to_string(),
3238 ),
3239 (
3240 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3241 "commander@example.com".to_string(),
3242 ),
3243 ]));
3244 assert!(validate_safety_backup_config(&restore).is_ok());
3245 }
3246
3247 #[test]
3248 fn disaster_recovery_requires_explicit_approval_annotations() {
3249 let source = KanidmRestoreSource {
3250 local: None,
3251 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3252 name: "backup-1".to_string(),
3253 }),
3254 };
3255 let mut restore = make_restore(source, None);
3256 restore.metadata.annotations = Some(BTreeMap::from([(
3257 DISASTER_RECOVERY_ANNOTATION.to_string(),
3258 "true".to_string(),
3259 )]));
3260 assert!(disaster_recovery_requested(&restore));
3261 assert!(validate_disaster_recovery_approval(&restore).is_err());
3262 }
3263
3264 #[test]
3265 fn disaster_recovery_accepts_reason_and_approver() {
3266 let source = KanidmRestoreSource {
3267 local: None,
3268 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3269 name: "backup-1".to_string(),
3270 }),
3271 };
3272 let mut restore = make_restore(source, None);
3273 restore.metadata.annotations = Some(BTreeMap::from([
3274 (
3275 DISASTER_RECOVERY_ANNOTATION.to_string(),
3276 "true".to_string(),
3277 ),
3278 (
3279 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3280 "original cluster lost".to_string(),
3281 ),
3282 (
3283 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3284 "incident-commander".to_string(),
3285 ),
3286 ]));
3287 assert!(disaster_recovery_requested(&restore));
3288 assert!(validate_disaster_recovery_approval(&restore).is_ok());
3289 }
3290
3291 #[test]
3292 fn remote_source_requires_repository_ref_when_not_skipping() {
3293 let source = KanidmRestoreSource {
3294 local: None,
3295 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3296 name: "backup-1".to_string(),
3297 }),
3298 };
3299 let safety = SafetyBackupConfig {
3300 repository_ref: None,
3301 skip: false,
3302 };
3303 let restore = make_restore(source, Some(safety));
3304 assert!(validate_safety_backup_config(&restore).is_err());
3305 }
3306
3307 #[test]
3308 fn remote_source_accepts_repository_ref() {
3309 let source = KanidmRestoreSource {
3310 local: None,
3311 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3312 name: "backup-1".to_string(),
3313 }),
3314 };
3315 let safety = SafetyBackupConfig {
3316 repository_ref: Some(SafetyBackupRepositoryRef {
3317 name: "offsite".to_string(),
3318 }),
3319 skip: false,
3320 };
3321 let restore = make_restore(source, Some(safety));
3322 assert!(validate_safety_backup_config(&restore).is_ok());
3323 }
3324
3325 #[test]
3326 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3327 let phases = vec![
3328 KanidmRestorePhase::Pending,
3329 KanidmRestorePhase::Validating,
3330 KanidmRestorePhase::Quiescing,
3331 KanidmRestorePhase::SafetyBackup,
3332 KanidmRestorePhase::PreparingSource,
3333 KanidmRestorePhase::RestoringPrimary,
3334 KanidmRestorePhase::Verifying,
3335 KanidmRestorePhase::RebuildingReplicas,
3336 KanidmRestorePhase::Resuming,
3337 KanidmRestorePhase::Completed,
3338 ];
3339 assert_eq!(phases.len(), 10);
3340 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3341 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3342 }
3343
3344 #[test]
3345 fn status_default_has_empty_phase_timestamps() {
3346 let status = KanidmRestoreStatus::default();
3347 assert!(status.phase_timestamps.is_empty());
3348 }
3349
3350 #[test]
3351 fn status_default_has_empty_original_replicas() {
3352 let status = KanidmRestoreStatus::default();
3353 assert!(status.original_replicas.is_empty());
3354 }
3355
3356 #[test]
3357 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3358 let mut target = super::super::crd::Kanidm::default();
3359 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3360 run_as_non_root: Some(false),
3361 run_as_user: Some(389),
3362 run_as_group: Some(389),
3363 fs_group: Some(389),
3364 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3365 supplemental_groups: Some(vec![390, 391]),
3366 ..Default::default()
3367 });
3368
3369 let ctx = super::source_job_pod_security_context(&target);
3370
3371 assert_eq!(ctx.run_as_non_root, Some(false));
3372 assert_eq!(ctx.run_as_user, Some(389));
3373 assert_eq!(ctx.run_as_group, Some(389));
3374 assert_eq!(ctx.fs_group, Some(389));
3375 assert_eq!(
3376 ctx.fs_group_change_policy.as_deref(),
3377 Some("OnRootMismatch")
3378 );
3379 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3380 assert_eq!(
3381 ctx.seccomp_profile
3382 .as_ref()
3383 .map(|profile| profile.type_.as_str()),
3384 Some("RuntimeDefault")
3385 );
3386 }
3387
3388 #[test]
3389 fn hardened_security_context_drops_all_capabilities() {
3390 let ctx = hardened_security_context();
3391 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3392 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3393 assert_eq!(ctx.run_as_non_root, Some(true));
3394 let caps = ctx.capabilities.unwrap();
3395 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3396 assert_eq!(
3397 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3398 Some("RuntimeDefault")
3399 );
3400 }
3401
3402 #[test]
3403 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3404 let ctx = kanidm_job_security_context();
3405 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3406 assert!(ctx.run_as_non_root.is_none());
3407 assert!(ctx.read_only_root_filesystem.is_none());
3408 let caps = ctx.capabilities.unwrap();
3409 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3410 assert_eq!(
3411 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3412 Some("RuntimeDefault")
3413 );
3414 }
3415
3416 #[test]
3417 fn safety_backup_job_name_is_deterministic() {
3418 let source = KanidmRestoreSource {
3419 local: Some(super::KanidmRestoreLocalSource {
3420 file_name: "backup.json".to_string(),
3421 }),
3422 backup_ref: None,
3423 };
3424 let restore = make_restore(source, None);
3425 let name1 = super::safety_backup_job_name(&restore);
3426 let name2 = super::safety_backup_job_name(&restore);
3427 assert_eq!(name1, name2);
3428 assert_eq!(name1, "test-restore-safety-backup");
3429 }
3430
3431 #[test]
3432 fn source_prep_job_name_is_deterministic() {
3433 let source = KanidmRestoreSource {
3434 local: None,
3435 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3436 name: "backup-1".to_string(),
3437 }),
3438 };
3439 let restore = make_restore(source, None);
3440 let name = super::source_prep_job_name(&restore);
3441 assert_eq!(name, "test-restore-source-prep");
3442 }
3443
3444 #[test]
3445 fn break_glass_condition_is_recorded_in_status() {
3446 let mut status = KanidmRestoreStatus::default();
3447 let condition = super::restore_condition(
3448 &status.conditions,
3449 CONDITION_BREAK_GLASS,
3450 CONDITION_TRUE,
3451 "BreakGlassOverride",
3452 "Safety backup skipped",
3453 Some(1),
3454 );
3455 status.conditions.push(condition);
3456 assert!(status.conditions.iter().any(|c| {
3457 c.type_ == CONDITION_BREAK_GLASS
3458 && c.status == CONDITION_TRUE
3459 && c.reason == "BreakGlassOverride"
3460 }));
3461 }
3462
3463 #[test]
3464 fn replica_count_entry_serialization() {
3465 let entry = ReplicaCountEntry {
3466 group: "primary".to_string(),
3467 replicas: 3,
3468 };
3469 let json = serde_json::to_string(&entry).unwrap();
3470 assert!(json.contains("\"group\":\"primary\""));
3471 assert!(json.contains("\"replicas\":3"));
3472 }
3473
3474 #[test]
3475 fn local_source_requires_repository_ref_when_not_skipping() {
3476 let source = KanidmRestoreSource {
3477 local: Some(super::KanidmRestoreLocalSource {
3478 file_name: "backup.json".to_string(),
3479 }),
3480 backup_ref: None,
3481 };
3482 let safety = SafetyBackupConfig {
3483 repository_ref: None,
3484 skip: false,
3485 };
3486 let restore = make_restore(source, Some(safety));
3487 assert!(validate_safety_backup_config(&restore).is_err());
3488 }
3489
3490 #[test]
3491 fn local_source_accepts_repository_ref_when_not_skipping() {
3492 let source = KanidmRestoreSource {
3493 local: Some(super::KanidmRestoreLocalSource {
3494 file_name: "backup.json".to_string(),
3495 }),
3496 backup_ref: None,
3497 };
3498 let safety = SafetyBackupConfig {
3499 repository_ref: Some(SafetyBackupRepositoryRef {
3500 name: "offsite".to_string(),
3501 }),
3502 skip: false,
3503 };
3504 let restore = make_restore(source, Some(safety));
3505 assert!(validate_safety_backup_config(&restore).is_ok());
3506 }
3507
3508 #[test]
3509 fn skip_without_break_glass_annotations_is_rejected() {
3510 let source = KanidmRestoreSource {
3511 local: Some(super::KanidmRestoreLocalSource {
3512 file_name: "backup.json".to_string(),
3513 }),
3514 backup_ref: None,
3515 };
3516 let safety = SafetyBackupConfig {
3517 repository_ref: None,
3518 skip: true,
3519 };
3520 let restore = make_restore(source, Some(safety));
3521 assert!(validate_safety_backup_config(&restore).is_err());
3522 }
3523
3524 #[test]
3525 fn has_accepted_condition_true() {
3526 let conditions = vec![Condition {
3527 type_: "Ready".to_string(),
3528 status: "True".to_string(),
3529 reason: "Accepted".to_string(),
3530 message: String::new(),
3531 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3532 k8s_openapi::jiff::Timestamp::now(),
3533 ),
3534 observed_generation: None,
3535 }];
3536 assert!(has_accepted_condition(&conditions));
3537 }
3538
3539 #[test]
3540 fn has_accepted_condition_false_when_not_accepted_reason() {
3541 let conditions = vec![Condition {
3542 type_: "Ready".to_string(),
3543 status: "True".to_string(),
3544 reason: "SomeOtherReason".to_string(),
3545 message: String::new(),
3546 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3547 k8s_openapi::jiff::Timestamp::now(),
3548 ),
3549 observed_generation: None,
3550 }];
3551 assert!(!has_accepted_condition(&conditions));
3552 }
3553
3554 #[test]
3555 fn has_accepted_condition_false_when_status_false() {
3556 let conditions = vec![Condition {
3557 type_: "Ready".to_string(),
3558 status: "False".to_string(),
3559 reason: "Accepted".to_string(),
3560 message: String::new(),
3561 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3562 k8s_openapi::jiff::Timestamp::now(),
3563 ),
3564 observed_generation: None,
3565 }];
3566 assert!(!has_accepted_condition(&conditions));
3567 }
3568
3569 #[test]
3570 fn has_accepted_condition_empty() {
3571 assert!(!has_accepted_condition(&[]));
3572 }
3573
3574 #[test]
3575 fn safety_backup_upload_operation_doc_is_valid() {
3576 let source = KanidmRestoreSource {
3577 local: None,
3578 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3579 name: "backup-1".to_string(),
3580 }),
3581 };
3582 let mut restore = make_restore(source, None);
3583 restore.metadata.namespace = Some("test-ns".to_string());
3584 let target = super::super::crd::Kanidm::default();
3585 let doc_str = super::build_safety_upload_operation_doc(
3586 &restore,
3587 &target,
3588 "test-backup-id",
3589 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3590 "my-bucket",
3591 "prod",
3592 "https://s3.example.com",
3593 "us-east-1",
3594 false,
3595 false,
3596 None,
3597 None,
3598 )
3599 .unwrap();
3600 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3601 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3602 assert_eq!(parsed["kind"], "OperationDocument");
3603 assert_eq!(parsed["operation"], "upload");
3604 assert_eq!(parsed["bucket"], "my-bucket");
3605 assert_eq!(parsed["prefix"], "prod");
3606 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3607 assert_eq!(parsed["region"], "us-east-1");
3608 assert_eq!(parsed["backupId"], "test-backup-id");
3609 assert_eq!(parsed["consistency"], "kanidm-offline");
3610 assert_eq!(parsed["reason"], "restore-safety");
3611 assert_eq!(
3612 parsed["payloadPath"],
3613 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3614 );
3615 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3616 }
3617
3618 #[test]
3619 fn download_operation_doc_uses_real_repository_data() {
3620 let source = KanidmRestoreSource {
3621 local: None,
3622 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3623 name: "backup-1".to_string(),
3624 }),
3625 };
3626 let _restore = make_restore(source, None);
3627 let target = super::super::crd::Kanidm::default();
3628 let doc_str = super::build_download_operation_doc(
3629 &target,
3630 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3631 "019c7c76-f423-7a12-8f41-2bea7588a303",
3632 "source-uid-456",
3633 "real-bucket",
3634 "real-prefix",
3635 "https://real-endpoint.com",
3636 "eu-west-1",
3637 true,
3638 false,
3639 None,
3640 None,
3641 );
3642 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3643 assert_eq!(parsed["bucket"], "real-bucket");
3644 assert_eq!(parsed["prefix"], "real-prefix");
3645 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3646 assert_eq!(parsed["region"], "eu-west-1");
3647 assert_eq!(parsed["forcePathStyle"], true);
3648 assert_eq!(
3649 parsed["manifestKey"],
3650 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3651 );
3652 assert_eq!(
3653 parsed["expectedBackupId"],
3654 "019c7c76-f423-7a12-8f41-2bea7588a303"
3655 );
3656 assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3657 }
3658
3659 #[test]
3660 fn safety_backup_ref_is_set_after_verification() {
3661 let status = KanidmRestoreStatus {
3662 phase: KanidmRestorePhase::SafetyBackup,
3663 safety_backup_job_name: Some("test-safety-backup".to_string()),
3664 safety_backup_ref: Some("safety-test-restore".to_string()),
3665 safety_backup_expected_backup_id: Some(
3666 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3667 ),
3668 safety_backup_manifest_key: Some(
3669 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3670 ),
3671 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3672 ..Default::default()
3673 };
3674 assert!(status.safety_backup_ref.is_some());
3675 assert!(
3676 status
3677 .safety_backup_ref
3678 .as_ref()
3679 .unwrap()
3680 .starts_with("safety-")
3681 );
3682 assert!(status.safety_backup_expected_backup_id.is_some());
3683 assert!(status.safety_backup_manifest_key.is_some());
3684 assert!(status.safety_backup_payload_sha256.is_some());
3685 }
3686
3687 #[test]
3688 fn database_mutation_not_started_without_safety_backup_ref() {
3689 let status = KanidmRestoreStatus {
3690 phase: KanidmRestorePhase::PreparingSource,
3691 safety_backup_ref: None,
3692 database_mutation_started: false,
3693 ..Default::default()
3694 };
3695 assert!(!status.database_mutation_started);
3696 assert!(status.safety_backup_ref.is_none());
3697 }
3698
3699 #[test]
3700 fn pre_boundary_failure_does_not_set_mutation_started() {
3701 let status = KanidmRestoreStatus {
3702 phase: KanidmRestorePhase::Failed,
3703 message: Some(
3704 "safety backup job failed; target restored to original state".to_string(),
3705 ),
3706 database_mutation_started: false,
3707 ..Default::default()
3708 };
3709 assert!(!status.database_mutation_started);
3710 }
3711
3712 #[test]
3713 fn shared_volume_constants_are_distinct() {
3714 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3715 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3716 }
3717
3718 #[test]
3719 fn validate_backup_compatibility_matching_versions_passes() {
3720 let source = KanidmRestoreSource {
3721 local: None,
3722 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3723 name: "backup-1".to_string(),
3724 }),
3725 };
3726 let restore = make_restore(source, None);
3727 let target = kanidm_with_version("1.10.4");
3728 let backup = kaniop_backup_core::crd::KanidmBackup {
3729 metadata: kube::api::ObjectMeta::default(),
3730 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3731 backup_id: "id".to_string(),
3732 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3733 name: "k".to_string(),
3734 uid: "uid".to_string(),
3735 },
3736 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3737 name: "repo".to_string(),
3738 },
3739 manifest_key: "key".to_string(),
3740 },
3741 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3742 kanidm_version: Some("1.10.4".to_string()),
3743 ..Default::default()
3744 }),
3745 };
3746 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3747 }
3748
3749 #[test]
3750 fn validate_backup_compatibility_version_mismatch_fails() {
3751 let source = KanidmRestoreSource {
3752 local: None,
3753 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3754 name: "backup-1".to_string(),
3755 }),
3756 };
3757 let restore = make_restore(source, None);
3758 let target = kanidm_with_version("1.11.0");
3759 let backup = kaniop_backup_core::crd::KanidmBackup {
3760 metadata: kube::api::ObjectMeta::default(),
3761 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3762 backup_id: "id".to_string(),
3763 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3764 name: "k".to_string(),
3765 uid: "uid".to_string(),
3766 },
3767 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3768 name: "repo".to_string(),
3769 },
3770 manifest_key: "key".to_string(),
3771 },
3772 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3773 kanidm_version: Some("1.10.4".to_string()),
3774 ..Default::default()
3775 }),
3776 };
3777 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3778 }
3779
3780 #[test]
3781 fn validate_backup_compatibility_no_backup_version_passes() {
3782 let source = KanidmRestoreSource {
3783 local: None,
3784 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3785 name: "backup-1".to_string(),
3786 }),
3787 };
3788 let restore = make_restore(source, None);
3789 let target = super::super::crd::Kanidm::default();
3790 let backup = kaniop_backup_core::crd::KanidmBackup {
3791 metadata: kube::api::ObjectMeta::default(),
3792 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3793 backup_id: "id".to_string(),
3794 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3795 name: "k".to_string(),
3796 uid: "uid".to_string(),
3797 },
3798 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3799 name: "repo".to_string(),
3800 },
3801 manifest_key: "key".to_string(),
3802 },
3803 status: None,
3804 };
3805 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3806 }
3807
3808 #[test]
3809 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3810 let source = KanidmRestoreSource {
3811 local: None,
3812 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3813 name: "backup-1".to_string(),
3814 }),
3815 };
3816 let mut restore = make_restore(source, None);
3817 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3818 let target = super::super::crd::Kanidm::default();
3819 let backup = kaniop_backup_core::crd::KanidmBackup {
3820 metadata: kube::api::ObjectMeta::default(),
3821 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3822 backup_id: "id".to_string(),
3823 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3824 name: "k".to_string(),
3825 uid: "uid".to_string(),
3826 },
3827 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3828 name: "repo".to_string(),
3829 },
3830 manifest_key: "key".to_string(),
3831 },
3832 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3833 image_digest: Some("sha256:originaldigest".to_string()),
3834 ..Default::default()
3835 }),
3836 };
3837 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3838 }
3839
3840 #[test]
3841 fn validate_backup_compatibility_image_digest_match_passes() {
3842 let source = KanidmRestoreSource {
3843 local: None,
3844 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3845 name: "backup-1".to_string(),
3846 }),
3847 };
3848 let mut restore = make_restore(source, None);
3849 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3850 let target = super::super::crd::Kanidm::default();
3851 let backup = kaniop_backup_core::crd::KanidmBackup {
3852 metadata: kube::api::ObjectMeta::default(),
3853 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3854 backup_id: "id".to_string(),
3855 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3856 name: "k".to_string(),
3857 uid: "uid".to_string(),
3858 },
3859 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3860 name: "repo".to_string(),
3861 },
3862 manifest_key: "key".to_string(),
3863 },
3864 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3865 image_digest: Some("sha256:abc123".to_string()),
3866 ..Default::default()
3867 }),
3868 };
3869 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3870 }
3871
3872 #[test]
3873 fn safety_backup_id_is_deterministic() {
3874 let source = KanidmRestoreSource {
3875 local: None,
3876 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3877 name: "backup-1".to_string(),
3878 }),
3879 };
3880 let restore = make_restore(source, None);
3881 let id1 = super::compute_safety_backup_id(&restore).unwrap();
3882 let id2 = super::compute_safety_backup_id(&restore).unwrap();
3883 assert_eq!(id1, id2);
3884 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3885 }
3886
3887 #[test]
3888 fn safety_manifest_key_is_deterministic() {
3889 let source = KanidmRestoreSource {
3890 local: None,
3891 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3892 name: "backup-1".to_string(),
3893 }),
3894 };
3895 let restore = make_restore(source, None);
3896 let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3897 let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3898 assert_eq!(key1, key2);
3899 assert!(key1.contains("manifest.json"));
3900 assert!(key1.starts_with("v1/tenants/"));
3901 assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3904 assert!(
3905 !key1.contains("a1b2c3d4"),
3906 "safety manifest key must not key the tenant segment by object UID: {key1}"
3907 );
3908 }
3909
3910 #[test]
3918 fn safety_manifest_key_matches_deployed_keyspace_layout() {
3919 let source = KanidmRestoreSource {
3920 local: None,
3921 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3922 name: "backup-1".to_string(),
3923 }),
3924 };
3925 let restore = make_restore(source, None);
3926 let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3927 assert_eq!(
3928 super::compute_safety_backup_id(&restore).unwrap(),
3929 expected_backup_id
3930 );
3931 assert_eq!(
3932 super::compute_safety_manifest_key(&restore).unwrap(),
3933 format!(
3934 "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
3935 )
3936 );
3937 }
3938
3939 #[test]
3940 fn safety_backup_id_survives_restart() {
3941 let source = KanidmRestoreSource {
3942 local: None,
3943 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3944 name: "backup-1".to_string(),
3945 }),
3946 };
3947 let mut restore = make_restore(source, None);
3948 let original_id = super::compute_safety_backup_id(&restore).unwrap();
3949 assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
3953 restore.status = Some(KanidmRestoreStatus {
3954 phase: KanidmRestorePhase::SafetyBackup,
3955 safety_backup_expected_backup_id: Some(original_id.clone()),
3956 ..Default::default()
3957 });
3958 let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3959 assert_eq!(original_id, recomputed_id);
3960 }
3961
3962 #[test]
3963 fn safety_backup_id_derived_from_uid_not_name() {
3964 let source = KanidmRestoreSource {
3965 local: None,
3966 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3967 name: "backup-1".to_string(),
3968 }),
3969 };
3970 let restore1 = make_restore(source.clone(), None);
3971 let mut restore2 = make_restore(source, None);
3972 restore2.metadata.name = Some("different-name-same-uid".to_string());
3973 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3974 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3975 assert_eq!(id1, id2);
3976 }
3977
3978 #[test]
3979 fn safety_backup_id_differs_for_different_uids() {
3980 let source = KanidmRestoreSource {
3981 local: None,
3982 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3983 name: "backup-1".to_string(),
3984 }),
3985 };
3986 let restore1 = make_restore(source.clone(), None);
3987 let mut restore2 = make_restore(source, None);
3988 restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3989 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3990 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3991 assert_ne!(id1, id2);
3992 }
3993
3994 #[test]
3995 fn safety_backup_id_errors_without_uid() {
3996 let source = KanidmRestoreSource {
3997 local: None,
3998 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3999 name: "backup-1".to_string(),
4000 }),
4001 };
4002 let mut restore = make_restore(source, None);
4003 restore.metadata.uid = None;
4004 assert!(super::compute_safety_backup_id(&restore).is_err());
4005 }
4006
4007 #[test]
4008 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4009 let source = KanidmRestoreSource {
4010 local: None,
4011 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4012 name: "backup-1".to_string(),
4013 }),
4014 };
4015 let restore = make_restore(source, None);
4016 assert!(super::requires_safety_backup(&restore));
4017 let status_without_ref = KanidmRestoreStatus {
4018 phase: KanidmRestorePhase::RestoringPrimary,
4019 safety_backup_ref: None,
4020 database_mutation_started: false,
4021 ..Default::default()
4022 };
4023 assert!(status_without_ref.safety_backup_ref.is_none());
4024 assert!(!status_without_ref.database_mutation_started);
4025 }
4026
4027 #[test]
4028 fn mutation_boundary_allows_progression_when_safety_ref_set() {
4029 let status_with_ref = KanidmRestoreStatus {
4030 phase: KanidmRestorePhase::RestoringPrimary,
4031 safety_backup_ref: Some("safety-test-restore".to_string()),
4032 safety_backup_manifest_key: Some(
4033 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4034 ),
4035 safety_backup_payload_sha256: Some("abc123".to_string()),
4036 database_mutation_started: false,
4037 ..Default::default()
4038 };
4039 assert!(status_with_ref.safety_backup_ref.is_some());
4040 assert!(!status_with_ref.database_mutation_started);
4041 }
4042
4043 #[test]
4044 fn result_document_verification_rejects_missing_backup_id() {
4045 let result_json = r#"{
4046 "apiVersion": "backup.kaniop.rs/v1alpha1",
4047 "kind": "ResultDocument",
4048 "operation": "upload",
4049 "success": true,
4050 "exitCode": "success",
4051 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4052 "payloadSha256": "abc123"
4053 }"#;
4054 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4055 assert!(result.backup_id.is_none());
4056 }
4057
4058 #[test]
4059 fn result_document_verification_rejects_missing_manifest_key() {
4060 let result_json = r#"{
4061 "apiVersion": "backup.kaniop.rs/v1alpha1",
4062 "kind": "ResultDocument",
4063 "operation": "upload",
4064 "success": true,
4065 "exitCode": "success",
4066 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4067 "payloadSha256": "abc123"
4068 }"#;
4069 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4070 assert!(result.manifest_key.is_none());
4071 }
4072
4073 #[test]
4074 fn result_document_verification_rejects_missing_payload_sha256() {
4075 let result_json = r#"{
4076 "apiVersion": "backup.kaniop.rs/v1alpha1",
4077 "kind": "ResultDocument",
4078 "operation": "upload",
4079 "success": true,
4080 "exitCode": "success",
4081 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4082 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4083 }"#;
4084 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4085 assert!(result.payload_sha256.is_none());
4086 }
4087
4088 #[test]
4089 fn result_document_verification_rejects_failure_exit_code() {
4090 let result_json = r#"{
4091 "apiVersion": "backup.kaniop.rs/v1alpha1",
4092 "kind": "ResultDocument",
4093 "operation": "upload",
4094 "success": false,
4095 "exitCode": "integrity",
4096 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4097 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4098 "payloadSha256": "abc123"
4099 }"#;
4100 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4101 assert!(!result.success);
4102 assert_eq!(
4103 result.exit_code,
4104 kaniop_backup_core::result::ExitCode::Integrity
4105 );
4106 }
4107
4108 #[test]
4109 fn result_document_verification_rejects_wrong_operation() {
4110 let result_json = r#"{
4111 "apiVersion": "backup.kaniop.rs/v1alpha1",
4112 "kind": "ResultDocument",
4113 "operation": "download",
4114 "success": true,
4115 "exitCode": "success",
4116 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4117 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4118 "payloadSha256": "abc123"
4119 }"#;
4120 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4121 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4122 }
4123
4124 #[test]
4125 fn result_document_verification_accepts_valid_document() {
4126 let result_json = r#"{
4127 "apiVersion": "backup.kaniop.rs/v1alpha1",
4128 "kind": "ResultDocument",
4129 "operation": "upload",
4130 "success": true,
4131 "exitCode": "success",
4132 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4133 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4134 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4135 }"#;
4136 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4137 assert!(result.success);
4138 assert_eq!(
4139 result.exit_code,
4140 kaniop_backup_core::result::ExitCode::Success
4141 );
4142 assert_eq!(result.operation, "upload");
4143 assert!(result.backup_id.is_some());
4144 assert!(result.manifest_key.is_some());
4145 assert!(result.payload_sha256.is_some());
4146 }
4147
4148 #[test]
4149 fn termination_message_path_constant_is_valid() {
4150 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4151 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4152 }
4153
4154 #[test]
4155 fn status_tracks_safety_backup_evidence() {
4156 let status = KanidmRestoreStatus {
4157 phase: KanidmRestorePhase::PreparingSource,
4158 safety_backup_ref: Some("safety-test-restore".to_string()),
4159 safety_backup_expected_backup_id: Some(
4160 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4161 ),
4162 safety_backup_manifest_key: Some(
4163 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4164 ),
4165 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4166 ..Default::default()
4167 };
4168 assert!(status.safety_backup_ref.is_some());
4169 assert!(status.safety_backup_expected_backup_id.is_some());
4170 assert!(status.safety_backup_manifest_key.is_some());
4171 assert!(status.safety_backup_payload_sha256.is_some());
4172 }
4173
4174 #[test]
4175 fn fail_closed_semantics_preserved_on_result_parse_failure() {
4176 let invalid_json = "not valid json";
4177 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4178 assert!(result.is_err());
4179 }
4180
4181 #[test]
4182 fn fail_closed_semantics_preserved_on_oversized_result() {
4183 let oversized_json = format!(
4184 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4185 kaniop_backup_core::result::RESULT_DOC_VERSION,
4186 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4187 );
4188 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4189 assert!(result.is_err());
4190 }
4191
4192 #[test]
4193 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4194 let source = KanidmRestoreSource {
4195 local: None,
4196 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4197 name: "backup-1".to_string(),
4198 }),
4199 };
4200 let mut restore = make_restore(source, None);
4201 restore.metadata.namespace = Some("test-ns".to_string());
4202 let target = super::super::crd::Kanidm::default();
4203 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4204 let doc_str = super::build_safety_upload_operation_doc(
4205 &restore,
4206 &target,
4207 "test-backup-id",
4208 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4209 "my-bucket",
4210 "prod",
4211 "https://s3.example.com",
4212 "us-east-1",
4213 false,
4214 false,
4215 Some(&ca_path),
4216 None,
4217 )
4218 .unwrap();
4219 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4220 assert_eq!(parsed["caBundlePath"], ca_path);
4221 }
4222
4223 #[test]
4224 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4225 let source = KanidmRestoreSource {
4226 local: None,
4227 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4228 name: "backup-1".to_string(),
4229 }),
4230 };
4231 let mut restore = make_restore(source, None);
4232 restore.metadata.namespace = Some("test-ns".to_string());
4233 let target = super::super::crd::Kanidm::default();
4234 let doc_str = super::build_safety_upload_operation_doc(
4235 &restore,
4236 &target,
4237 "test-backup-id",
4238 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4239 "my-bucket",
4240 "prod",
4241 "https://s3.example.com",
4242 "us-east-1",
4243 false,
4244 false,
4245 None,
4246 None,
4247 )
4248 .unwrap();
4249 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4250 assert!(parsed["caBundlePath"].is_null());
4251 }
4252
4253 #[test]
4254 fn download_operation_doc_includes_ca_bundle_path_when_set() {
4255 let source = KanidmRestoreSource {
4256 local: None,
4257 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4258 name: "backup-1".to_string(),
4259 }),
4260 };
4261 let _restore = make_restore(source, None);
4262 let target = super::super::crd::Kanidm::default();
4263 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4264 let doc_str = super::build_download_operation_doc(
4265 &target,
4266 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4267 "019c7c76-f423-7a12-8f41-2bea7588a303",
4268 "source-uid-456",
4269 "real-bucket",
4270 "real-prefix",
4271 "https://real-endpoint.com",
4272 "eu-west-1",
4273 true,
4274 false,
4275 Some(&ca_path),
4276 None,
4277 );
4278 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4279 assert_eq!(parsed["caBundlePath"], ca_path);
4280 }
4281
4282 #[test]
4283 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4284 let source = KanidmRestoreSource {
4285 local: None,
4286 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4287 name: "backup-1".to_string(),
4288 }),
4289 };
4290 let _restore = make_restore(source, None);
4291 let target = super::super::crd::Kanidm::default();
4292 let doc_str = super::build_download_operation_doc(
4293 &target,
4294 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4295 "019c7c76-f423-7a12-8f41-2bea7588a303",
4296 "source-uid-456",
4297 "real-bucket",
4298 "real-prefix",
4299 "https://real-endpoint.com",
4300 "eu-west-1",
4301 true,
4302 false,
4303 None,
4304 None,
4305 );
4306 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4307 assert!(parsed["caBundlePath"].is_null());
4308 }
4309
4310 #[test]
4311 fn source_prep_result_operation_is_download() {
4312 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4313 assert_ne!(
4314 super::SOURCE_PREP_RESULT_OPERATION,
4315 super::SAFETY_BACKUP_RESULT_OPERATION
4316 );
4317 }
4318
4319 #[test]
4320 fn download_result_document_verification_accepts_valid_document() {
4321 let result_json = r#"{
4322 "apiVersion": "backup.kaniop.rs/v1alpha1",
4323 "kind": "ResultDocument",
4324 "operation": "download",
4325 "success": true,
4326 "exitCode": "success",
4327 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4328 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4329 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4330 }"#;
4331 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4332 assert!(result.success);
4333 assert_eq!(
4334 result.exit_code,
4335 kaniop_backup_core::result::ExitCode::Success
4336 );
4337 assert_eq!(result.operation, "download");
4338 assert_eq!(
4339 result.backup_id.as_deref(),
4340 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4341 );
4342 assert!(result.manifest_key.is_some());
4343 assert!(result.payload_sha256.is_some());
4344 }
4345
4346 #[test]
4347 fn download_result_document_rejects_wrong_operation() {
4348 let result_json = r#"{
4349 "apiVersion": "backup.kaniop.rs/v1alpha1",
4350 "kind": "ResultDocument",
4351 "operation": "upload",
4352 "success": true,
4353 "exitCode": "success",
4354 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4355 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4356 "payloadSha256": "abc123"
4357 }"#;
4358 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4359 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4360 }
4361
4362 #[test]
4363 fn download_result_document_rejects_failure_success_field() {
4364 let result_json = r#"{
4365 "apiVersion": "backup.kaniop.rs/v1alpha1",
4366 "kind": "ResultDocument",
4367 "operation": "download",
4368 "success": false,
4369 "exitCode": "retryable",
4370 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4371 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4372 "payloadSha256": "abc123"
4373 }"#;
4374 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4375 assert!(!result.success);
4376 assert_ne!(
4377 result.exit_code,
4378 kaniop_backup_core::result::ExitCode::Success
4379 );
4380 }
4381
4382 #[test]
4383 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4384 let status = KanidmRestoreStatus {
4385 phase: KanidmRestorePhase::PreparingSource,
4386 database_mutation_started: false,
4387 ..Default::default()
4388 };
4389 assert!(!status.database_mutation_started);
4390 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4391 }
4392
4393 #[test]
4394 fn backup_job_volume_size_default_is_10gi() {
4395 let qty = crate::controller::backup_job_volume_size();
4396 assert_eq!(qty, Quantity("10Gi".to_string()));
4397 }
4398
4399 #[test]
4400 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4401 use kaniop_backup_core::auth::build_encryption_env_vars;
4402 use kaniop_backup_core::crd::SecretRef;
4403
4404 let key_ref = SecretRef {
4405 name: "kek-secret".to_string(),
4406 };
4407 let env_vars = build_encryption_env_vars(Some(&key_ref));
4408 assert_eq!(env_vars.len(), 1);
4409 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4410 let secret_ref = env_vars[0]
4411 .value_from
4412 .as_ref()
4413 .unwrap()
4414 .secret_key_ref
4415 .as_ref()
4416 .unwrap();
4417 assert_eq!(secret_ref.name, "kek-secret");
4418 assert_eq!(secret_ref.key, "encryption-key");
4419 }
4420
4421 #[test]
4422 fn encryption_env_vars_absent_when_no_key_ref() {
4423 use kaniop_backup_core::auth::build_encryption_env_vars;
4424
4425 let env_vars = build_encryption_env_vars(None);
4426 assert!(
4427 env_vars.is_empty(),
4428 "encryption env vars must be empty when key_ref is None"
4429 );
4430 }
4431
4432 #[test]
4433 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4434 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4435
4436 let source = KanidmRestoreSource {
4437 local: None,
4438 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4439 name: "backup-1".to_string(),
4440 }),
4441 };
4442 let mut restore = make_restore(source, None);
4443 restore.metadata.namespace = Some("test-ns".to_string());
4444 let target = super::super::crd::Kanidm::default();
4445 let encryption = RepositoryEncryption {
4446 mode: EncryptionMode::ClientSide,
4447 key_id: None,
4448 key_ref: Some(SecretRef {
4449 name: "kek-secret".to_string(),
4450 }),
4451 };
4452 let doc_str = super::build_safety_upload_operation_doc(
4453 &restore,
4454 &target,
4455 "test-backup-id",
4456 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4457 "my-bucket",
4458 "prod",
4459 "https://s3.example.com",
4460 "us-east-1",
4461 false,
4462 false,
4463 None,
4464 Some(&encryption),
4465 )
4466 .unwrap();
4467 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4468 assert_eq!(parsed["encryptionMode"], "clientSide");
4469 assert!(parsed["encryptionKeyId"].is_null());
4470 }
4471
4472 #[test]
4473 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4474 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4475
4476 let source = KanidmRestoreSource {
4477 local: None,
4478 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4479 name: "backup-1".to_string(),
4480 }),
4481 };
4482 let _restore = make_restore(source, None);
4483 let target = super::super::crd::Kanidm::default();
4484 let encryption = RepositoryEncryption {
4485 mode: EncryptionMode::ProviderKms,
4486 key_id: Some("alias/kaniop-backups".to_string()),
4487 key_ref: None,
4488 };
4489 let doc_str = super::build_download_operation_doc(
4490 &target,
4491 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4492 "019c7c76-f423-7a12-8f41-2bea7588a303",
4493 "source-uid-456",
4494 "real-bucket",
4495 "real-prefix",
4496 "https://real-endpoint.com",
4497 "eu-west-1",
4498 true,
4499 false,
4500 None,
4501 Some(&encryption),
4502 );
4503 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4504 assert_eq!(parsed["encryptionMode"], "providerKms");
4505 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4506 }
4507
4508 #[test]
4509 fn post_mutation_failure_records_job_names() {
4510 let existing = KanidmRestoreStatus {
4511 phase: KanidmRestorePhase::RestoringPrimary,
4512 database_mutation_started: true,
4513 ..Default::default()
4514 };
4515
4516 let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4517 assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4518 assert_eq!(
4519 restore_failed.restore_job_name.as_deref(),
4520 Some("my-restore-job")
4521 );
4522 assert!(restore_failed.verify_job_name.is_none());
4523 assert!(restore_failed.database_mutation_started);
4524 assert_eq!(
4525 restore_failed.message.as_deref(),
4526 Some("database restore job failed")
4527 );
4528
4529 let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4530 assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4531 assert_eq!(
4532 verify_failed.verify_job_name.as_deref(),
4533 Some("my-verify-job")
4534 );
4535 assert!(verify_failed.restore_job_name.is_none());
4536 assert!(verify_failed.database_mutation_started);
4537 assert_eq!(
4538 verify_failed.message.as_deref(),
4539 Some("database verification failed")
4540 );
4541 }
4542
4543 #[test]
4544 fn restore_and_verify_job_names_are_deterministic() {
4545 let source = KanidmRestoreSource {
4546 local: Some(super::KanidmRestoreLocalSource {
4547 file_name: "backup.json".to_string(),
4548 }),
4549 backup_ref: None,
4550 };
4551 let restore = make_restore(source, None);
4552 assert_eq!(restore_job_name(&restore), "test-restore-restore");
4553 assert_eq!(verify_job_name(&restore), "test-restore-verify");
4554 }
4555}