1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Gauge, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78 feature = "schemars",
79 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82 category = "kaniop",
83 group = "kaniop.rs",
84 version = "v1beta1",
85 kind = "KanidmRestore",
86 plural = "kanidmrestores",
87 singular = "kanidmrestore",
88 shortname = "idmrestore",
89 namespaced,
90 status = "KanidmRestoreStatus",
91 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97 pub target_ref: KanidmRestoreTargetRef,
98 pub source: KanidmRestoreSource,
99 pub restore_image: String,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108 pub name: String,
109 pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115 feature = "schemars",
116 schemars(extend("x-kubernetes-validations" = [
117 {
118 "message": "exactly one of local or backupRef must be set",
119 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120 }
121 ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125 #[serde(skip_serializing_if = "Option::is_none")]
127 pub local: Option<KanidmRestoreLocalSource>,
128 #[serde(skip_serializing_if = "Option::is_none")]
130 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137 pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145 pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153 #[serde(skip_serializing_if = "Option::is_none")]
155 pub repository_ref: Option<SafetyBackupRepositoryRef>,
156 #[serde(default)]
158 pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165 pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171 #[default]
172 Pending,
173 Validating,
174 Quiescing,
175 SafetyBackup,
176 PreparingSource,
177 RestoringPrimary,
178 Verifying,
179 RebuildingReplicas,
180 Resuming,
181 Completed,
182 Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_generation: Option<i64>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_target_uid: Option<String>,
193 #[serde(default)]
194 pub phase: KanidmRestorePhase,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub restore_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub verify_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub safety_backup_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub source_prep_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_ref: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_expected_backup_id: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_manifest_key: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_payload_sha256: Option<String>,
211 #[serde(default)]
212 pub replicas_cleared: bool,
213 #[serde(default)]
214 pub certificates_cleared: bool,
215 #[serde(default)]
216 pub database_mutation_started: bool,
217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
218 pub original_replicas: Vec<ReplicaCountEntry>,
219 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220 pub phase_timestamps: std::collections::BTreeMap<String, String>,
221 #[serde(default, skip_serializing_if = "Vec::is_empty")]
222 #[cfg_attr(
223 feature = "schemars",
224 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225 )]
226 pub conditions: Vec<Condition>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235 pub group: String,
236 pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241 attempts: Counter<u64>,
242 outcomes: Counter<u64>,
243 duration_seconds: Histogram<f64>,
244 start_timestamp_seconds: Gauge<i64>,
245 break_glass_total: Counter<u64>,
246 disaster_recovery_total: Counter<u64>,
247 safety_backup_duration_seconds: Histogram<f64>,
248}
249
250impl RestoreMetrics {
251 fn new() -> Self {
252 let meter = global::meter("kaniop");
253 Self {
254 attempts: meter
255 .u64_counter("restore_attempts")
256 .with_description("Number of Kanidm restore attempts started")
257 .build(),
258 outcomes: meter
259 .u64_counter("restore_outcomes")
260 .with_description("Number of terminal Kanidm restore outcomes")
261 .build(),
262 duration_seconds: meter
263 .f64_histogram("restore_duration_seconds")
264 .with_description("Kanidm restore duration from object creation to terminal phase")
265 .with_unit("s")
266 .build(),
267 start_timestamp_seconds: meter
268 .i64_gauge("restore_start_timestamp_seconds")
269 .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
270 .with_unit("s")
271 .build(),
272 break_glass_total: meter
273 .u64_counter("restore_break_glass")
274 .with_description("Number of break-glass safety backup overrides")
275 .build(),
276 disaster_recovery_total: meter
277 .u64_counter("restore_disaster_recovery")
278 .with_description("Number of approved disaster-recovery source identity overrides")
279 .build(),
280 safety_backup_duration_seconds: meter
281 .f64_histogram("restore_safety_backup_duration_seconds")
282 .with_description("Duration of pre-restore safety backup creation")
283 .with_unit("s")
284 .build(),
285 }
286 }
287}
288
289fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
290 let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
291 let active = restore.metadata.deletion_timestamp.is_none()
292 && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
293 let timestamp = if active {
294 restore
295 .metadata
296 .creation_timestamp
297 .as_ref()
298 .map(|created| created.0.as_second())
299 .unwrap_or_else(|| Timestamp::now().as_second())
300 } else {
301 0
302 };
303 let attributes = [
304 KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
305 KeyValue::new("restore", restore.name_any()),
306 ];
307 metrics.start_timestamp_seconds.record(timestamp, &attributes);
308}
309
310#[derive(Clone)]
311struct RestoreContext {
312 client: Client,
313 recorder: Recorder,
314 metrics: RestoreMetrics,
315}
316
317async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
318 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
319 match phase {
320 KanidmRestorePhase::Pending => {
321 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
322 Ok(Action::requeue(REQUEUE))
323 }
324 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
325 Ok((target, disaster_recovery_source)) => {
326 mark_restoring(&restore, &target, &ctx).await?;
327 if let Some(source) = disaster_recovery_source.as_ref() {
328 record_disaster_recovery_override(&restore, &target, &ctx, source).await;
329 }
330 let mut status = restore.status.clone().unwrap_or_default();
331 if let Some(source) = disaster_recovery_source.as_ref() {
332 let target_uid = target.uid().unwrap_or_default();
333 let condition = restore_condition(
334 &status.conditions,
335 CONDITION_DISASTER_RECOVERY,
336 CONDITION_TRUE,
337 "ApprovedSourceIdentityMismatch",
338 &format!(
339 "Approved disaster recovery from source {}/{} into target {}/{}",
340 source.name,
341 source.uid,
342 target.name_any(),
343 target_uid
344 ),
345 restore.metadata.generation,
346 );
347 status
348 .conditions
349 .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
350 status.conditions.push(condition);
351 }
352 status.observed_target_uid = target.uid();
353 status.original_replicas = capture_original_replicas(&target);
354 status.phase = KanidmRestorePhase::Quiescing;
355 status.message = None;
356 patch_status(&restore, &ctx, status).await?;
357 Ok(Action::requeue(REQUEUE))
358 }
359 Err(error) => {
360 set_phase(
361 &restore,
362 &ctx,
363 KanidmRestorePhase::Failed,
364 Some(error.to_string()),
365 )
366 .await?;
367 Ok(Action::requeue(Duration::from_secs(300)))
368 }
369 },
370 KanidmRestorePhase::Quiescing => {
371 let target = get_target(&restore, &ctx).await?;
372 scale_all(&target, &ctx, 0).await?;
373 if target_pods_stopped(&target, &ctx).await?
374 && target_volumes_detached(&target, &ctx).await?
375 {
376 let next_phase = if requires_safety_backup(&restore) {
377 KanidmRestorePhase::SafetyBackup
378 } else {
379 record_break_glass(&restore, &ctx).await?;
380 KanidmRestorePhase::PreparingSource
381 };
382 if next_phase == KanidmRestorePhase::PreparingSource {
383 let refreshed = Api::<KanidmRestore>::namespaced(
384 ctx.client.clone(),
385 restore.namespace().as_deref().unwrap_or_default(),
386 )
387 .get(&restore.name_any())
388 .await
389 .map_err(|error| {
390 Error::kube_error(
391 "get",
392 "KanidmRestore",
393 restore.namespace().as_deref().unwrap_or_default(),
394 restore.name_any(),
395 error,
396 )
397 })?;
398 set_phase(&refreshed, &ctx, next_phase, None).await?;
399 } else {
400 set_phase(&restore, &ctx, next_phase, None).await?;
401 }
402 }
403 Ok(Action::requeue(REQUEUE))
404 }
405 KanidmRestorePhase::SafetyBackup => {
406 let target = get_target(&restore, &ctx).await?;
407 let name = safety_backup_job_name(&restore);
408 let expected_backup_id = compute_safety_backup_id(&restore)?;
409 let expected_manifest_key = compute_safety_manifest_key(&restore)?;
410 ensure_safety_backup_job(
411 &restore,
412 &target,
413 &ctx,
414 &name,
415 &expected_backup_id,
416 &expected_manifest_key,
417 )
418 .await?;
419 let mut status = restore.status.clone().unwrap_or_default();
420 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421 match job_state(&restore, &ctx, &name).await? {
422 JobState::Complete => {
423 let result =
424 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425 match result {
426 Ok(verified) => {
427 status.safety_backup_job_name = Some(name);
428 status.safety_backup_ref =
429 Some(format!("safety-{}", restore.name_any()));
430 status.safety_backup_manifest_key = Some(verified.manifest_key);
431 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432 record_safety_backup_duration(&restore, &ctx).await;
433 status.phase = KanidmRestorePhase::PreparingSource;
434 status.message = None;
435 patch_status(&restore, &ctx, status).await?;
436 }
437 Err(error) => {
438 resume_before_mutation(&restore, &ctx).await?;
439 set_phase(
440 &restore,
441 &ctx,
442 KanidmRestorePhase::Failed,
443 Some(format!("safety backup result verification failed: {error}")),
444 )
445 .await?;
446 }
447 }
448 }
449 JobState::Failed => {
450 resume_before_mutation(&restore, &ctx).await?;
451 set_phase(
452 &restore,
453 &ctx,
454 KanidmRestorePhase::Failed,
455 Some(
456 "safety backup job failed; target restored to original state"
457 .to_string(),
458 ),
459 )
460 .await?;
461 }
462 JobState::Running => {
463 patch_status(&restore, &ctx, status).await?;
464 }
465 }
466 Ok(Action::requeue(REQUEUE))
467 }
468 KanidmRestorePhase::PreparingSource => {
469 if is_remote_source(&restore) {
470 let target = get_target(&restore, &ctx).await?;
471 let name = source_prep_job_name(&restore);
472 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473 match job_state(&restore, &ctx, &name).await? {
474 JobState::Complete => {
475 let backup_id = restore
476 .spec
477 .source
478 .backup_ref
479 .as_ref()
480 .map(|b| b.name.clone())
481 .unwrap_or_default();
482 let backup = {
483 let ns = restore.namespace().unwrap();
484 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485 .get(&backup_id)
486 .await
487 .map_err(|e| {
488 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489 })?
490 };
491 let expected_backup_id = &backup.spec.backup_id;
492 let result =
493 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494 .await;
495 match result {
496 Ok(verified) => {
497 if let Some(expected_sha256) = backup
498 .status
499 .as_ref()
500 .and_then(|s| s.payload_sha256.as_deref())
501 {
502 if verified.payload_sha256 != expected_sha256 {
503 resume_before_mutation(&restore, &ctx).await?;
504 set_phase(
505 &restore,
506 &ctx,
507 KanidmRestorePhase::Failed,
508 Some(format!(
509 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
510 verified.payload_sha256
511 )),
512 )
513 .await?;
514 return Ok(Action::requeue(REQUEUE));
515 }
516 }
517 let mut status = restore.status.clone().unwrap_or_default();
518 status.source_prep_job_name = Some(name);
519 status.phase = KanidmRestorePhase::RestoringPrimary;
520 status.message = None;
521 patch_status(&restore, &ctx, status).await?;
522 }
523 Err(error) => {
524 resume_before_mutation(&restore, &ctx).await?;
525 set_phase(
526 &restore,
527 &ctx,
528 KanidmRestorePhase::Failed,
529 Some(format!(
530 "source preparation result verification failed: {error}"
531 )),
532 )
533 .await?;
534 }
535 }
536 }
537 JobState::Failed => {
538 resume_before_mutation(&restore, &ctx).await?;
539 set_phase(
540 &restore,
541 &ctx,
542 KanidmRestorePhase::Failed,
543 Some(
544 "source preparation job failed; target restored to original state"
545 .to_string(),
546 ),
547 )
548 .await?;
549 }
550 JobState::Running => {}
551 }
552 } else {
553 let target = get_target(&restore, &ctx).await?;
554 let name = source_check_job_name(&restore);
555 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
556 match job_state(&restore, &ctx, &name).await? {
557 JobState::Complete => {
558 let mut status = restore.status.clone().unwrap_or_default();
559 status.source_prep_job_name = Some(name);
560 status.phase = KanidmRestorePhase::RestoringPrimary;
561 status.message = None;
562 patch_status(&restore, &ctx, status).await?;
563 }
564 JobState::Failed => {
565 resume_before_mutation(&restore, &ctx).await?;
566 set_phase(
567 &restore,
568 &ctx,
569 KanidmRestorePhase::Failed,
570 Some(
571 "local backup file check failed; target restored to original state"
572 .to_string(),
573 ),
574 )
575 .await?;
576 }
577 JobState::Running => {}
578 }
579 }
580 Ok(Action::requeue(REQUEUE))
581 }
582 KanidmRestorePhase::RestoringPrimary => {
583 let target = get_target(&restore, &ctx).await?;
584 let status = restore.status.as_ref().cloned().unwrap_or_default();
585 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
586 return Err(Error::MissingData(
587 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
588 ));
589 }
590 if !status.database_mutation_started {
591 let mut status = status;
592 status.database_mutation_started = true;
593 status.message = Some(
594 "database mutation boundary persisted; starting primary restore".to_string(),
595 );
596 patch_status(&restore, &ctx, status).await?;
597 return Ok(Action::requeue(REQUEUE));
598 }
599 let name = restore_job_name(&restore);
600 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
601 match job_state(&restore, &ctx, &name).await? {
602 JobState::Complete => {
603 let mut status = restore.status.clone().unwrap_or_default();
604 status.restore_job_name = Some(name);
605 status.phase = KanidmRestorePhase::Verifying;
606 status.message = None;
607 patch_status(&restore, &ctx, status).await?;
608 }
609 JobState::Failed => {
610 let status = restore.status.clone().unwrap_or_default();
611 let failed_status = restore_primary_failed_status(&status, &name);
612 patch_status(&restore, &ctx, failed_status).await?;
613 }
614 JobState::Running => {}
615 }
616 Ok(Action::requeue(REQUEUE))
617 }
618 KanidmRestorePhase::Verifying => {
619 let target = get_target(&restore, &ctx).await?;
620 let name = verify_job_name(&restore);
621 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
622 match job_state(&restore, &ctx, &name).await? {
623 JobState::Complete => {
624 let mut status = restore.status.clone().unwrap_or_default();
625 status.verify_job_name = Some(name);
626 status.phase = KanidmRestorePhase::RebuildingReplicas;
627 status.message = None;
628 patch_status(&restore, &ctx, status).await?;
629 }
630 JobState::Failed => {
631 let status = restore.status.clone().unwrap_or_default();
632 let failed_status = verifying_failed_status(&status, &name);
633 patch_status(&restore, &ctx, failed_status).await?;
634 }
635 JobState::Running => {}
636 }
637 Ok(Action::requeue(REQUEUE))
638 }
639 KanidmRestorePhase::RebuildingReplicas => {
640 let target = get_target(&restore, &ctx).await?;
641 let mut status = restore.status.clone().unwrap_or_default();
642 if !status.replicas_cleared {
643 if delete_secondary_pvcs(&target, &ctx).await? {
644 status.replicas_cleared = true;
645 status.message = Some("secondary database state cleared".to_string());
646 patch_status(&restore, &ctx, status).await?;
647 }
648 return Ok(Action::requeue(REQUEUE));
649 }
650
651 scale_primary(&target, &ctx, 1).await?;
652 if !primary_ready(&target, &ctx).await? {
653 return Ok(Action::requeue(REQUEUE));
654 }
655 if !status.certificates_cleared {
656 delete_replica_cert_secrets(&target, &ctx).await?;
657 delete_admin_secret(&target, &ctx).await?;
658 status.certificates_cleared = true;
659 status.message = Some(
660 "replica certificates and admin secret cleared for regeneration".to_string(),
661 );
662 patch_status(&restore, &ctx, status).await?;
663 return Ok(Action::requeue(REQUEUE));
664 }
665 status.phase = KanidmRestorePhase::Resuming;
666 status.message = None;
667 patch_status(&restore, &ctx, status).await?;
668 Ok(Action::requeue(REQUEUE))
669 }
670 KanidmRestorePhase::Resuming => {
671 let target = get_target(&restore, &ctx).await?;
672 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
673 clear_restoring(&restore, &target, &ctx).await?;
674 Ok(Action::requeue(Duration::from_secs(3600)))
675 }
676 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
677 Ok(Action::requeue(Duration::from_secs(3600)))
678 }
679 }
680}
681
682#[derive(Clone, Debug, PartialEq, Eq)]
683struct DisasterRecoverySource {
684 name: String,
685 uid: String,
686}
687
688async fn validate(
689 restore: &KanidmRestore,
690 ctx: &RestoreContext,
691) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
692 let target = get_target(restore, ctx).await?;
693 let actual_uid = target
694 .uid()
695 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
696 if actual_uid != restore.spec.target_ref.uid {
697 return Err(Error::MissingData(format!(
698 "target UID mismatch: expected {}, got {}",
699 restore.spec.target_ref.uid, actual_uid
700 )));
701 }
702 validate_source(restore)?;
703 let disaster_recovery_source = if is_remote_source(restore) {
704 validate_backup_ref(restore, &target, ctx).await?
705 } else {
706 None
707 };
708 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
709 {
710 return Err(Error::MissingData(format!(
711 "restoreImage must be the target's pinned Kanidm image (target image is {})",
712 target.spec.image
713 )));
714 }
715 let storage = target
716 .spec
717 .storage
718 .as_ref()
719 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
720 if storage.empty_dir.is_some()
721 || storage.ephemeral.is_some()
722 || storage.volume_claim_template.is_none()
723 {
724 return Err(Error::MissingData(
725 "restore requires PVC-backed Kanidm storage".to_string(),
726 ));
727 }
728 let primaries = target
729 .spec
730 .replica_groups
731 .iter()
732 .filter(|rg| rg.primary_node)
733 .count();
734 if primaries != 1 {
735 return Err(Error::MissingData(
736 "backup/restore requires exactly one primary replica group".to_string(),
737 ));
738 }
739 let ns = restore.namespace().unwrap();
740 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
741 .list(&ListParams::default())
742 .await
743 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
744 if restores.items.iter().any(|other| {
745 other.name_any() != restore.name_any()
746 && other.spec.target_ref.name == restore.spec.target_ref.name
747 && !matches!(
748 other.status.as_ref().map(|s| s.phase),
749 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
750 )
751 }) {
752 return Err(Error::MissingData(
753 "another active restore targets this Kanidm".to_string(),
754 ));
755 }
756 validate_safety_backup_config(restore)?;
757 Ok((target, disaster_recovery_source))
758}
759
760fn safe_basename(name: &str) -> bool {
761 !name.is_empty()
762 && name != "."
763 && name != ".."
764 && !name.contains('/')
765 && !name.contains('\\')
766 && !name.contains('\0')
767}
768
769fn mutable_image(image: &str) -> bool {
770 image == "kanidm/server:latest"
771 || image.ends_with(":latest")
772 || (!image.contains('@')
773 && !image
774 .rsplit('/')
775 .next()
776 .is_some_and(|part| part.contains(':')))
777}
778
779async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
780 let ns = restore
781 .namespace()
782 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
783 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784 .get(&restore.spec.target_ref.name)
785 .await
786 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
787}
788
789async fn mark_restoring(
790 restore: &KanidmRestore,
791 target: &Kanidm,
792 ctx: &RestoreContext,
793) -> Result<()> {
794 let ns = target.namespace().unwrap();
795 let uid = restore
796 .uid()
797 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
798 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
799 .patch(
800 &target.name_any(),
801 &PatchParams::default(),
802 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
803 )
804 .await
805 .map(|_| ())
806 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
807}
808
809async fn clear_restoring(
810 restore: &KanidmRestore,
811 target: &Kanidm,
812 ctx: &RestoreContext,
813) -> Result<()> {
814 let ns = target.namespace().unwrap();
815 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
816 return Ok(());
817 }
818 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
819 .patch(
820 &target.name_any(),
821 &PatchParams::default(),
822 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
823 )
824 .await
825 .map(|_| ())
826 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
827}
828
829async fn patch_status(
830 restore: &KanidmRestore,
831 ctx: &RestoreContext,
832 mut status: KanidmRestoreStatus,
833) -> Result<()> {
834 let ns = restore.namespace().unwrap();
835 let previous_phase = restore.status.as_ref().map(|current| current.phase);
836 let next_phase = status.phase;
837 status.observed_generation = restore.metadata.generation;
838 update_restore_conditions(&mut status, restore.metadata.generation);
839
840 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
841 .patch_status(
842 &restore.name_any(),
843 &PatchParams::default(),
844 &Patch::Merge(json!({"status": &status})),
845 )
846 .await
847 .map_err(|e| {
848 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
849 })?;
850
851 if previous_phase != Some(next_phase) {
852 record_restore_transition(
853 restore,
854 ctx,
855 previous_phase,
856 next_phase,
857 status.message.as_deref(),
858 )
859 .await;
860 }
861 Ok(())
862}
863
864fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
865 let previous = status.conditions.clone();
866 let phase = status.phase;
867 let phase_reason = format!("{phase:?}");
868 let message = status
869 .message
870 .clone()
871 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
872 let terminal_success = phase == KanidmRestorePhase::Completed;
873 let terminal_failure = phase == KanidmRestorePhase::Failed;
874 let progressing = !terminal_success && !terminal_failure;
875
876 let sticky_conditions: Vec<Condition> = previous
877 .iter()
878 .filter(|condition| {
879 condition.type_ == CONDITION_BREAK_GLASS
880 || condition.type_ == CONDITION_DISASTER_RECOVERY
881 })
882 .cloned()
883 .collect();
884 status.conditions = vec![
885 restore_condition(
886 &previous,
887 CONDITION_PROGRESSING,
888 if progressing {
889 CONDITION_TRUE
890 } else {
891 CONDITION_FALSE
892 },
893 &phase_reason,
894 &message,
895 generation,
896 ),
897 restore_condition(
898 &previous,
899 CONDITION_READY,
900 if terminal_success {
901 CONDITION_TRUE
902 } else {
903 CONDITION_FALSE
904 },
905 if terminal_success {
906 "RestoreCompleted"
907 } else {
908 "RestoreNotCompleted"
909 },
910 if terminal_success {
911 "Kanidm restore completed successfully."
912 } else {
913 "Kanidm restore has not completed successfully."
914 },
915 generation,
916 ),
917 restore_condition(
918 &previous,
919 CONDITION_FAILED,
920 if terminal_failure {
921 CONDITION_TRUE
922 } else {
923 CONDITION_FALSE
924 },
925 if terminal_failure {
926 "RestoreFailed"
927 } else {
928 "NoRestoreFailure"
929 },
930 if terminal_failure {
931 status
932 .message
933 .as_deref()
934 .unwrap_or("Kanidm restore failed.")
935 } else {
936 "No terminal restore failure has been recorded."
937 },
938 generation,
939 ),
940 ];
941 status.conditions.extend(sticky_conditions);
942}
943
944fn restore_condition(
945 previous: &[Condition],
946 condition_type: &str,
947 condition_status: &str,
948 reason: &str,
949 message: &str,
950 generation: Option<i64>,
951) -> Condition {
952 let last_transition_time = previous
953 .iter()
954 .find(|condition| {
955 condition.type_ == condition_type
956 && condition.status == condition_status
957 && condition.reason == reason
958 })
959 .map(|condition| condition.last_transition_time.clone())
960 .unwrap_or_else(|| Time(Timestamp::now()));
961 Condition {
962 type_: condition_type.to_string(),
963 status: condition_status.to_string(),
964 reason: reason.to_string(),
965 message: message.to_string(),
966 last_transition_time,
967 observed_generation: generation,
968 }
969}
970
971async fn record_restore_transition(
972 restore: &KanidmRestore,
973 ctx: &RestoreContext,
974 previous_phase: Option<KanidmRestorePhase>,
975 phase: KanidmRestorePhase,
976 message: Option<&str>,
977) {
978 if phase == KanidmRestorePhase::Validating {
979 ctx.metrics.attempts.add(1, &[]);
980 }
981
982 let result = match phase {
983 KanidmRestorePhase::Completed => Some("success"),
984 KanidmRestorePhase::Failed => Some("failure"),
985 _ => None,
986 };
987 if let Some(result) = result {
988 let attributes = [
989 KeyValue::new("result", result),
990 KeyValue::new("phase", format!("{phase:?}")),
991 ];
992 ctx.metrics.outcomes.add(1, &attributes);
993 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
994 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
995 ctx.metrics.duration_seconds.record(elapsed, &attributes);
996 }
997 }
998
999 let reason = if phase == KanidmRestorePhase::Failed {
1000 "RestoreFailed"
1001 } else {
1002 "RestorePhaseChanged"
1003 };
1004 let note = message.map(str::to_string).or_else(|| {
1005 Some(format!(
1006 "Kanidm restore phase changed from {} to {phase:?}.",
1007 previous_phase
1008 .map(|previous| format!("{previous:?}"))
1009 .unwrap_or_else(|| "None".to_string())
1010 ))
1011 });
1012 if let Err(error) = ctx
1013 .recorder
1014 .publish(
1015 &Event {
1016 type_: if phase == KanidmRestorePhase::Failed {
1017 EventType::Warning
1018 } else {
1019 EventType::Normal
1020 },
1021 reason: reason.to_string(),
1022 note,
1023 action: "Restore".to_string(),
1024 secondary: None,
1025 },
1026 &restore.object_ref(&()),
1027 )
1028 .await
1029 {
1030 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1031 }
1032}
1033
1034async fn set_phase(
1035 restore: &KanidmRestore,
1036 ctx: &RestoreContext,
1037 phase: KanidmRestorePhase,
1038 message: Option<String>,
1039) -> Result<()> {
1040 let mut status = restore.status.clone().unwrap_or_default();
1041 status.phase = phase;
1042 status.message = message;
1043 patch_status(restore, ctx, status).await
1044}
1045
1046async fn fail_after_mutation(
1047 restore: &KanidmRestore,
1048 ctx: &RestoreContext,
1049 message: &str,
1050) -> Result<()> {
1051 set_phase(
1052 restore,
1053 ctx,
1054 KanidmRestorePhase::Failed,
1055 Some(message.to_string()),
1056 )
1057 .await
1058}
1059
1060fn restore_primary_failed_status(
1061 existing: &KanidmRestoreStatus,
1062 job_name: &str,
1063) -> KanidmRestoreStatus {
1064 let mut status = existing.clone();
1065 status.restore_job_name = Some(job_name.to_string());
1066 status.phase = KanidmRestorePhase::Failed;
1067 status.message = Some("database restore job failed".to_string());
1068 status
1069}
1070
1071fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1072 let mut status = existing.clone();
1073 status.verify_job_name = Some(job_name.to_string());
1074 status.phase = KanidmRestorePhase::Failed;
1075 status.message = Some("database verification failed".to_string());
1076 status
1077}
1078
1079fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1080 target
1081 .spec
1082 .replica_groups
1083 .iter()
1084 .find(|rg| rg.primary_node)
1085 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1086}
1087
1088async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1089 let ns = target.namespace().unwrap();
1090 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1091 for rg in &target.spec.replica_groups {
1092 let name = target.statefulset_name(&rg.name);
1093 api.patch(
1094 &name,
1095 &PatchParams::default(),
1096 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1097 )
1098 .await
1099 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1100 }
1101 Ok(())
1102}
1103
1104async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1105 let ns = target.namespace().unwrap();
1106 let rg = primary_group(target)?;
1107 let name = target.statefulset_name(&rg.name);
1108 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1109 .patch(
1110 &name,
1111 &PatchParams::default(),
1112 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1113 )
1114 .await
1115 .map(|_| ())
1116 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1117}
1118
1119async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1120 let ns = target.namespace().unwrap();
1121 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1122 for rg in &target.spec.replica_groups {
1123 let name = target.statefulset_name(&rg.name);
1124 api.patch(
1125 &name,
1126 &PatchParams::default(),
1127 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1128 )
1129 .await
1130 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1131 }
1132 Ok(())
1133}
1134
1135async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1136 let ns = target.namespace().unwrap();
1137 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1138 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1139 .await
1140 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1141 Ok(pods.items.is_empty())
1142}
1143
1144async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1145 let ns = target.namespace().unwrap();
1146 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1147 let mut pv_names = BTreeSet::new();
1148 let primary_name = primary_pvc_name(target)?;
1149 let primary_pvc = pvc_api
1150 .get(&primary_name)
1151 .await
1152 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1153 let primary_pv = primary_pvc
1154 .spec
1155 .as_ref()
1156 .and_then(|spec| spec.volume_name.clone())
1157 .ok_or_else(|| {
1158 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1159 })?;
1160 pv_names.insert(primary_pv);
1161
1162 for rg in &target.spec.replica_groups {
1163 let sts = target.statefulset_name(&rg.name);
1164 for ordinal in 0..rg.replicas {
1165 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1166 if name == primary_name {
1167 continue;
1168 }
1169 if let Some(pvc) = pvc_api
1170 .get_opt(&name)
1171 .await
1172 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1173 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1174 {
1175 pv_names.insert(pv_name);
1176 }
1177 }
1178 }
1179
1180 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1181 .list(&ListParams::default())
1182 .await
1183 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1184 Ok(!attachments.items.iter().any(|attachment| {
1185 attachment
1186 .spec
1187 .source
1188 .persistent_volume_name
1189 .as_ref()
1190 .is_some_and(|pv_name| pv_names.contains(pv_name))
1191 }))
1192}
1193
1194async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1195 let ns = target.namespace().unwrap();
1196 let name = target.statefulset_name(&primary_group(target)?.name);
1197 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1198 .get(&name)
1199 .await
1200 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1201 Ok(sts
1202 .status
1203 .as_ref()
1204 .and_then(|s| s.ready_replicas)
1205 .unwrap_or(0)
1206 >= 1)
1207}
1208
1209fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1210 let rg = primary_group(target)?;
1211 Ok(format!(
1212 "{DATA_VOLUME}-{}-0",
1213 target.statefulset_name(&rg.name)
1214 ))
1215}
1216
1217async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1218 let ns = target.namespace().unwrap();
1219 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1220 let mut all_absent = true;
1221 for rg in &target.spec.replica_groups {
1222 let sts = target.statefulset_name(&rg.name);
1223 for ordinal in 0..rg.replicas {
1224 if rg.primary_node && ordinal == 0 {
1225 continue;
1226 }
1227 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1228 match api
1229 .get_opt(&name)
1230 .await
1231 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1232 {
1233 None => {}
1234 Some(pvc) => {
1235 all_absent = false;
1236 if pvc.metadata.deletion_timestamp.is_none() {
1237 match api.delete(&name, &DeleteParams::default()).await {
1238 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1239 Err(kube::Error::Api(status)) if status.code == 404 => {}
1240 Err(error) => {
1241 return Err(Error::kube_error(
1242 "delete",
1243 "PersistentVolumeClaim",
1244 &ns,
1245 &name,
1246 error,
1247 ));
1248 }
1249 }
1250 }
1251 }
1252 }
1253 }
1254 }
1255 Ok(all_absent)
1256}
1257
1258async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1259 let ns = target.namespace().unwrap();
1260 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1261 let label_selector = format!(
1262 "{}={},{}={}",
1263 CLUSTER_LABEL,
1264 target.name_any(),
1265 SECRET_TYPE_LABEL,
1266 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1267 );
1268 let secrets = api
1269 .list(&ListParams::default().labels(&label_selector))
1270 .await
1271 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1272 let mut all_absent = secrets.items.is_empty();
1273 for secret in secrets.items {
1274 let name = secret.name_any();
1275 if secret.metadata.deletion_timestamp.is_none() {
1276 match api.delete(&name, &DeleteParams::default()).await {
1277 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1278 Err(kube::Error::Api(status)) if status.code == 404 => {}
1279 Err(error) => {
1280 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1281 }
1282 }
1283 all_absent = false;
1284 }
1285 }
1286 Ok(all_absent)
1287}
1288
1289async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1290 let ns = target.namespace().unwrap();
1291 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1292 let name = format!("{}-admin-passwords", target.name_any());
1293 match api
1294 .get_opt(&name)
1295 .await
1296 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1297 {
1298 None => Ok(true),
1299 Some(secret) => {
1300 if secret.metadata.deletion_timestamp.is_some() {
1301 return Ok(false);
1302 }
1303 match api.delete(&name, &DeleteParams::default()).await {
1304 Ok(_) => {
1305 debug!(secret = %name, "deleting stale admin secret");
1306 Ok(false)
1307 }
1308 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1309 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1310 }
1311 }
1312 }
1313}
1314
1315fn restore_job_name(restore: &KanidmRestore) -> String {
1316 format!("{}-restore", restore.name_any())
1317}
1318fn verify_job_name(restore: &KanidmRestore) -> String {
1319 format!("{}-verify", restore.name_any())
1320}
1321
1322fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1323 let origin = target
1324 .spec
1325 .origin
1326 .clone()
1327 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1328 vec![
1329 EnvVar {
1330 name: "KANIDM_DB_PATH".to_string(),
1331 value: Some("/data/kanidm.db".to_string()),
1332 ..Default::default()
1333 },
1334 EnvVar {
1335 name: "KANIDM_DOMAIN".to_string(),
1336 value: Some(target.spec.domain.clone()),
1337 ..Default::default()
1338 },
1339 EnvVar {
1340 name: "KANIDM_ORIGIN".to_string(),
1341 value: Some(origin),
1342 ..Default::default()
1343 },
1344 EnvVar {
1345 name: "KANIDM_TLS_CHAIN".to_string(),
1346 value: Some(format!("{TLS_PATH}/tls.crt")),
1347 ..Default::default()
1348 },
1349 EnvVar {
1350 name: "KANIDM_TLS_KEY".to_string(),
1351 value: Some(format!("{TLS_PATH}/tls.key")),
1352 ..Default::default()
1353 },
1354 ]
1355}
1356
1357async fn ensure_database_job(
1358 restore: &KanidmRestore,
1359 target: &Kanidm,
1360 ctx: &RestoreContext,
1361 name: &str,
1362 verify: bool,
1363) -> Result<()> {
1364 let ns = restore.namespace().unwrap();
1365 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1366 if api
1367 .get_opt(name)
1368 .await
1369 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1370 .is_some()
1371 {
1372 return Ok(());
1373 }
1374 let mut command = vec![
1375 "kanidmd".to_string(),
1376 "database".to_string(),
1377 if verify { "backup" } else { "restore" }.to_string(),
1378 ];
1379 if verify {
1380 command.push("/verify/verification.json.gz".to_string());
1381 } else {
1382 if is_remote_source(restore) {
1383 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1384 } else {
1385 let file_name = restore
1386 .spec
1387 .source
1388 .local
1389 .as_ref()
1390 .map(|l| l.file_name.clone())
1391 .unwrap_or_default();
1392 command.push(format!("{BACKUP_PATH}/{file_name}"));
1393 }
1394 }
1395 let job = Job {
1396 metadata: ObjectMeta {
1397 name: Some(name.to_string()),
1398 namespace: Some(ns.clone()),
1399 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1400 ..Default::default()
1401 },
1402 spec: Some(JobSpec {
1403 backoff_limit: Some(0),
1404 template: PodTemplateSpec {
1405 metadata: Some(ObjectMeta::default()),
1406 spec: Some(PodSpec {
1407 automount_service_account_token: Some(false),
1408 restart_policy: Some("Never".to_string()),
1409 security_context: Some(source_job_pod_security_context(target)),
1410 containers: vec![Container {
1411 name: if verify { "verify" } else { "restore" }.to_string(),
1412 image: Some(restore.spec.restore_image.clone()),
1413 command: Some(command),
1414 env: Some(build_kanidm_db_env_vars(target)),
1415 security_context: Some(kanidm_job_security_context()),
1416 resources: Some(job_resource_requirements()),
1417 volume_mounts: Some(vec![
1418 VolumeMount {
1419 name: DATA_VOLUME.to_string(),
1420 mount_path: DATA_PATH.to_string(),
1421 ..Default::default()
1422 },
1423 VolumeMount {
1424 name: TLS_VOLUME.to_string(),
1425 mount_path: TLS_PATH.to_string(),
1426 read_only: Some(true),
1427 ..Default::default()
1428 },
1429 VolumeMount {
1430 name: "verification-output".to_string(),
1431 mount_path: "/verify".to_string(),
1432 ..Default::default()
1433 },
1434 ]),
1435 ..Default::default()
1436 }],
1437 volumes: Some(vec![
1438 Volume {
1439 name: DATA_VOLUME.to_string(),
1440 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1441 claim_name: primary_pvc_name(target)?,
1442 read_only: Some(false),
1443 }),
1444 ..Default::default()
1445 },
1446 Volume {
1447 name: TLS_VOLUME.to_string(),
1448 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1449 secret_name: Some(target.effective_tls_secret_name()),
1450 ..Default::default()
1451 }),
1452 ..Default::default()
1453 },
1454 Volume {
1455 name: "verification-output".to_string(),
1456 empty_dir: Some(Default::default()),
1457 ..Default::default()
1458 },
1459 ]),
1460 ..Default::default()
1461 }),
1462 },
1463 ..Default::default()
1464 }),
1465 ..Default::default()
1466 };
1467 api.create(&PostParams::default(), &job)
1468 .await
1469 .map(|_| ())
1470 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1471}
1472
1473enum JobState {
1474 Running,
1475 Complete,
1476 Failed,
1477}
1478
1479async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1480 let ns = restore.namespace().unwrap();
1481 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1482 .get(name)
1483 .await
1484 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1485 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1486 return Ok(JobState::Failed);
1487 }
1488 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1489 return Ok(JobState::Complete);
1490 }
1491 Ok(JobState::Running)
1492}
1493
1494fn is_remote_source(restore: &KanidmRestore) -> bool {
1495 restore.spec.source.backup_ref.is_some()
1496}
1497
1498fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1499 let safety = restore.spec.safety_backup.as_ref();
1500 let skip = safety.is_some_and(|s| s.skip);
1501 !skip
1502}
1503
1504fn validate_source(restore: &KanidmRestore) -> Result<()> {
1505 let has_local = restore.spec.source.local.is_some();
1506 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1507 if has_local && has_backup_ref {
1508 return Err(Error::MissingData(
1509 "source.local and source.backupRef are mutually exclusive".to_string(),
1510 ));
1511 }
1512 if !has_local && !has_backup_ref {
1513 return Err(Error::MissingData(
1514 "source must specify either local or backupRef".to_string(),
1515 ));
1516 }
1517 if let Some(local) = &restore.spec.source.local {
1518 if !safe_basename(&local.file_name) {
1519 return Err(Error::MissingData(
1520 "restore source fileName must be a safe basename".to_string(),
1521 ));
1522 }
1523 }
1524 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1525 if backup_ref.name.is_empty() {
1526 return Err(Error::MissingData(
1527 "source.backupRef.name must not be empty".to_string(),
1528 ));
1529 }
1530 }
1531 Ok(())
1532}
1533
1534fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1535 restore
1536 .metadata
1537 .annotations
1538 .as_ref()
1539 .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1540 .is_some_and(|value| value == "true")
1541}
1542
1543fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1544 let annotations = restore.metadata.annotations.as_ref();
1545 let reason = annotations
1546 .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1547 .map(String::as_str)
1548 .filter(|value| !value.trim().is_empty());
1549 let approved_by = annotations
1550 .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1551 .map(String::as_str)
1552 .filter(|value| !value.trim().is_empty());
1553
1554 if reason.is_none() || approved_by.is_none() {
1555 return Err(Error::MissingData(format!(
1556 "disaster-recovery restore requires non-empty {} and {} annotations",
1557 BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1558 )));
1559 }
1560 Ok(())
1561}
1562
1563async fn record_disaster_recovery_override(
1564 restore: &KanidmRestore,
1565 target: &Kanidm,
1566 ctx: &RestoreContext,
1567 source: &DisasterRecoverySource,
1568) {
1569 ctx.metrics.disaster_recovery_total.add(1, &[]);
1570 let reason = restore
1571 .metadata
1572 .annotations
1573 .as_ref()
1574 .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1575 .cloned()
1576 .unwrap_or_default();
1577 let approved_by = restore
1578 .metadata
1579 .annotations
1580 .as_ref()
1581 .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1582 .cloned()
1583 .unwrap_or_default();
1584 let target_uid = target.uid().unwrap_or_default();
1585 let note = format!(
1586 "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1587 source.name,
1588 source.uid,
1589 target.name_any(),
1590 target_uid,
1591 reason,
1592 approved_by
1593 );
1594 if let Err(error) = ctx
1595 .recorder
1596 .publish(
1597 &Event {
1598 type_: EventType::Warning,
1599 reason: "DisasterRecoveryOverride".to_string(),
1600 note: Some(note.clone()),
1601 action: "Restore".to_string(),
1602 secondary: None,
1603 },
1604 &restore.object_ref(&()),
1605 )
1606 .await
1607 {
1608 warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1609 }
1610 warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1611}
1612
1613fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1614 let safety = restore.spec.safety_backup.as_ref();
1615 let skip = safety.is_some_and(|s| s.skip);
1616 if skip {
1617 let annotations = restore.metadata.annotations.as_ref();
1618 let reason = annotations
1619 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1620 .map(|s| s.trim())
1621 .unwrap_or("");
1622 let approver = annotations
1623 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1624 .map(|s| s.trim())
1625 .unwrap_or("");
1626 if reason.is_empty() {
1627 return Err(Error::MissingData(format!(
1628 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1629 )));
1630 }
1631 if approver.is_empty() {
1632 return Err(Error::MissingData(format!(
1633 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1634 )));
1635 }
1636 }
1637 if !skip {
1638 let has_repo = safety
1639 .and_then(|s| s.repository_ref.as_ref())
1640 .is_some_and(|r| !r.name.is_empty());
1641 if !has_repo {
1642 return Err(Error::MissingData(
1643 "safety backup requires safetyBackup.repositoryRef when skip is not set"
1644 .to_string(),
1645 ));
1646 }
1647 }
1648 Ok(())
1649}
1650
1651fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1652 target
1653 .spec
1654 .replica_groups
1655 .iter()
1656 .map(|rg| ReplicaCountEntry {
1657 group: rg.name.clone(),
1658 replicas: rg.replicas,
1659 })
1660 .collect()
1661}
1662
1663async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1664 if let Ok(target) = get_target(restore, ctx).await {
1665 let owns_maintenance =
1666 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1667 if owns_maintenance {
1668 scale_desired(&target, ctx).await?;
1669 clear_restoring(restore, &target, ctx).await?;
1670 }
1671 }
1672 Ok(())
1673}
1674
1675async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1676 ctx.metrics.break_glass_total.add(1, &[]);
1677 let reason = restore
1678 .metadata
1679 .annotations
1680 .as_ref()
1681 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1682 .cloned()
1683 .unwrap_or_default();
1684 let approver = restore
1685 .metadata
1686 .annotations
1687 .as_ref()
1688 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1689 .cloned()
1690 .unwrap_or_default();
1691 let mut status = restore.status.clone().unwrap_or_default();
1692 let break_glass_condition = restore_condition(
1693 &status.conditions,
1694 CONDITION_BREAK_GLASS,
1695 CONDITION_TRUE,
1696 "BreakGlassOverride",
1697 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1698 restore.metadata.generation,
1699 );
1700 status
1701 .conditions
1702 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1703 status.conditions.push(break_glass_condition);
1704 patch_status(restore, ctx, status).await?;
1705 if let Err(error) = ctx
1706 .recorder
1707 .publish(
1708 &Event {
1709 type_: EventType::Warning,
1710 reason: "SafetyBackupSkipped".to_string(),
1711 note: Some(format!(
1712 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1713 )),
1714 action: "Restore".to_string(),
1715 secondary: None,
1716 },
1717 &restore.object_ref(&()),
1718 )
1719 .await
1720 {
1721 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1722 }
1723 warn!(
1724 restore = %restore.name_any(),
1725 reason = %reason,
1726 approver = %approver,
1727 "break-glass safety backup override activated"
1728 );
1729 Ok(())
1730}
1731
1732async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1733 let status = restore.status.as_ref().cloned().unwrap_or_default();
1734 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1735 if let Ok(start_ts) = start.parse::<Timestamp>() {
1736 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1737 ctx.metrics
1738 .safety_backup_duration_seconds
1739 .record(elapsed, &[]);
1740 }
1741 }
1742}
1743
1744fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1745 format!("{}-safety-backup", restore.name_any())
1746}
1747
1748fn source_prep_job_name(restore: &KanidmRestore) -> String {
1749 format!("{}-source-prep", restore.name_any())
1750}
1751
1752fn source_check_job_name(restore: &KanidmRestore) -> String {
1753 format!("{}-source-check", restore.name_any())
1754}
1755
1756async fn ensure_source_check_job(
1757 restore: &KanidmRestore,
1758 target: &Kanidm,
1759 ctx: &RestoreContext,
1760 name: &str,
1761) -> Result<()> {
1762 let ns = restore.namespace().unwrap();
1763 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1764 if api
1765 .get_opt(name)
1766 .await
1767 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1768 .is_some()
1769 {
1770 return Ok(());
1771 }
1772 let file_name = restore
1773 .spec
1774 .source
1775 .local
1776 .as_ref()
1777 .map(|l| l.file_name.clone())
1778 .unwrap_or_default();
1779 let operation_doc = serde_json::json!({
1780 "apiVersion": "backup.kaniop.rs/v1alpha1",
1781 "kind": "OperationDocument",
1782 "operation": "check",
1783 "path": format!("{BACKUP_PATH}/{file_name}"),
1784 "resultPath": "/run/kaniop-result/result.json",
1785 "format": "kanidmJsonGzip",
1786 })
1787 .to_string();
1788 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1789 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1790 let data_mover_image = data_mover_image();
1791 let job = Job {
1792 metadata: ObjectMeta {
1793 name: Some(name.to_string()),
1794 namespace: Some(ns.clone()),
1795 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1796 ..Default::default()
1797 },
1798 spec: Some(JobSpec {
1799 backoff_limit: Some(0),
1800 template: PodTemplateSpec {
1801 metadata: Some(ObjectMeta::default()),
1802 spec: Some(PodSpec {
1803 automount_service_account_token: Some(false),
1804 restart_policy: Some("Never".to_string()),
1805 security_context: Some(source_job_pod_security_context(target)),
1806 containers: vec![Container {
1807 name: "source-check".to_string(),
1808 image: Some(data_mover_image),
1809 command: Some(vec![
1810 "/bin/kaniop-data-mover".to_string(),
1811 "check".to_string(),
1812 "--operation-doc".to_string(),
1813 "/run/kaniop/operation.json".to_string(),
1814 ]),
1815 security_context: Some(hardened_security_context()),
1816 resources: Some(job_resource_requirements()),
1817 env: Some(vec![EnvVar {
1818 name: "RUST_LOG".to_string(),
1819 value: Some("info".to_string()),
1820 ..Default::default()
1821 }]),
1822 volume_mounts: Some(vec![
1823 VolumeMount {
1824 name: DATA_VOLUME.to_string(),
1825 mount_path: DATA_PATH.to_string(),
1826 read_only: Some(true),
1827 ..Default::default()
1828 },
1829 VolumeMount {
1830 name: "operation".to_string(),
1831 mount_path: "/run/kaniop".to_string(),
1832 read_only: Some(true),
1833 ..Default::default()
1834 },
1835 VolumeMount {
1836 name: "result".to_string(),
1837 mount_path: "/run/kaniop-result".to_string(),
1838 ..Default::default()
1839 },
1840 ]),
1841 ..Default::default()
1842 }],
1843 volumes: Some(vec![
1844 Volume {
1845 name: DATA_VOLUME.to_string(),
1846 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1847 claim_name: primary_pvc_name(target)?,
1848 read_only: Some(true),
1849 }),
1850 ..Default::default()
1851 },
1852 Volume {
1853 name: "operation".to_string(),
1854 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1855 name: operation_cm_name,
1856 ..Default::default()
1857 }),
1858 ..Default::default()
1859 },
1860 Volume {
1861 name: "result".to_string(),
1862 empty_dir: Some(Default::default()),
1863 ..Default::default()
1864 },
1865 ]),
1866 ..Default::default()
1867 }),
1868 },
1869 ..Default::default()
1870 }),
1871 ..Default::default()
1872 };
1873 api.create(&PostParams::default(), &job)
1874 .await
1875 .map(|_| ())
1876 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1877}
1878
1879const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1880
1881fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1882 let ns = restore.namespace().unwrap_or_default();
1883 let uid = restore.uid().ok_or_else(|| {
1884 Error::MissingData(
1885 "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1886 .to_string(),
1887 )
1888 })?;
1889 let namespace_uuid =
1890 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1891 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1892 Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1893}
1894
1895fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1903 let ns = restore.namespace().unwrap_or_default();
1904 let kanidm_uid = &restore.spec.target_ref.uid;
1905 let backup_id = compute_safety_backup_id(restore)?;
1906 Ok(format!(
1907 "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1908 ))
1909}
1910
1911struct VerifiedSafetyBackup {
1912 manifest_key: String,
1913 payload_sha256: String,
1914}
1915
1916async fn read_safety_backup_result(
1917 restore: &KanidmRestore,
1918 ctx: &RestoreContext,
1919 job_name: &str,
1920 expected_backup_id: &str,
1921) -> Result<VerifiedSafetyBackup> {
1922 let ns = restore.namespace().unwrap();
1923 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924 let pod_list = pods
1925 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926 .await
1927 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928 let pod = pod_list.items.first().ok_or_else(|| {
1929 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1930 })?;
1931 let pod_name = pod.name_any();
1932 let container_status = pod
1933 .status
1934 .as_ref()
1935 .and_then(|s| s.container_statuses.as_ref())
1936 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1937 .ok_or_else(|| {
1938 Error::MissingData(format!(
1939 "safety-upload container status not found in pod {pod_name}"
1940 ))
1941 })?;
1942 let termination_message = container_status
1943 .state
1944 .as_ref()
1945 .and_then(|state| state.terminated.as_ref())
1946 .and_then(|t| t.message.as_ref())
1947 .ok_or_else(|| {
1948 Error::MissingData(format!(
1949 "termination message not found in pod {pod_name}; result transport failed"
1950 ))
1951 })?;
1952 if termination_message.is_empty() {
1953 return Err(Error::MissingData(format!(
1954 "termination message is empty in pod {pod_name}; result document was not written"
1955 )));
1956 }
1957 let result_doc = parse_result_document(termination_message).map_err(|e| {
1958 Error::ParseError(format!(
1959 "failed to parse result document from pod {pod_name}: {e}"
1960 ))
1961 })?;
1962 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963 return Err(Error::ParseError(format!(
1964 "unsupported result document version: {}",
1965 result_doc.api_version
1966 )));
1967 }
1968 if result_doc.kind != "ResultDocument" {
1969 return Err(Error::ParseError(format!(
1970 "invalid result document kind: {}",
1971 result_doc.kind
1972 )));
1973 }
1974 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1975 return Err(Error::ParseError(format!(
1976 "result document operation mismatch: expected '{}', got '{}'",
1977 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1978 )));
1979 }
1980 if !result_doc.success {
1981 return Err(Error::ParseError(format!(
1982 "safety backup reported failure with exit code {:?}",
1983 result_doc.exit_code
1984 )));
1985 }
1986 if result_doc.exit_code != ExitCode::Success {
1987 return Err(Error::ParseError(format!(
1988 "safety backup exit code is not Success: {:?}",
1989 result_doc.exit_code
1990 )));
1991 }
1992 let doc_backup_id = result_doc
1993 .backup_id
1994 .as_deref()
1995 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996 if doc_backup_id != expected_backup_id {
1997 return Err(Error::ParseError(format!(
1998 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999 )));
2000 }
2001 let manifest_key = result_doc
2002 .manifest_key
2003 .clone()
2004 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005 if manifest_key.is_empty() {
2006 return Err(Error::ParseError(
2007 "result document manifestKey is empty".to_string(),
2008 ));
2009 }
2010 let payload_sha256 = result_doc
2011 .payload_sha256
2012 .clone()
2013 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014 if payload_sha256.is_empty() {
2015 return Err(Error::ParseError(
2016 "result document payloadSha256 is empty".to_string(),
2017 ));
2018 }
2019 info!(
2020 restore = %restore.name_any(),
2021 backup_id = %expected_backup_id,
2022 manifest_key = %manifest_key,
2023 "safety backup result verified"
2024 );
2025 Ok(VerifiedSafetyBackup {
2026 manifest_key,
2027 payload_sha256,
2028 })
2029}
2030
2031struct VerifiedSourcePrepResult {
2032 #[allow(dead_code)]
2033 manifest_key: String,
2034 payload_sha256: String,
2035}
2036
2037async fn read_source_prep_result(
2038 restore: &KanidmRestore,
2039 ctx: &RestoreContext,
2040 job_name: &str,
2041 expected_backup_id: &str,
2042) -> Result<VerifiedSourcePrepResult> {
2043 let ns = restore.namespace().unwrap();
2044 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2045 let pod_list = pods
2046 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2047 .await
2048 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2049 let pod = pod_list.items.first().ok_or_else(|| {
2050 Error::MissingData(format!("no pod found for source prep job {job_name}"))
2051 })?;
2052 let pod_name = pod.name_any();
2053 let container_status = pod
2054 .status
2055 .as_ref()
2056 .and_then(|s| s.container_statuses.as_ref())
2057 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2058 .ok_or_else(|| {
2059 Error::MissingData(format!(
2060 "source-prep container status not found in pod {pod_name}"
2061 ))
2062 })?;
2063 let termination_message = container_status
2064 .state
2065 .as_ref()
2066 .and_then(|state| state.terminated.as_ref())
2067 .and_then(|t| t.message.as_ref())
2068 .ok_or_else(|| {
2069 Error::MissingData(format!(
2070 "termination message not found in pod {pod_name}; result transport failed"
2071 ))
2072 })?;
2073 if termination_message.is_empty() {
2074 return Err(Error::MissingData(format!(
2075 "termination message is empty in pod {pod_name}; result document was not written"
2076 )));
2077 }
2078 let result_doc = parse_result_document(termination_message).map_err(|e| {
2079 Error::ParseError(format!(
2080 "failed to parse result document from pod {pod_name}: {e}"
2081 ))
2082 })?;
2083 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2084 return Err(Error::ParseError(format!(
2085 "unsupported result document version: {}",
2086 result_doc.api_version
2087 )));
2088 }
2089 if result_doc.kind != "ResultDocument" {
2090 return Err(Error::ParseError(format!(
2091 "invalid result document kind: {}",
2092 result_doc.kind
2093 )));
2094 }
2095 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2096 return Err(Error::ParseError(format!(
2097 "result document operation mismatch: expected '{}', got '{}'",
2098 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2099 )));
2100 }
2101 if !result_doc.success {
2102 return Err(Error::ParseError(format!(
2103 "source prep reported failure with exit code {:?}",
2104 result_doc.exit_code
2105 )));
2106 }
2107 if result_doc.exit_code != ExitCode::Success {
2108 return Err(Error::ParseError(format!(
2109 "source prep exit code is not Success: {:?}",
2110 result_doc.exit_code
2111 )));
2112 }
2113 let doc_backup_id = result_doc
2114 .backup_id
2115 .as_deref()
2116 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2117 if doc_backup_id != expected_backup_id {
2118 return Err(Error::ParseError(format!(
2119 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2120 )));
2121 }
2122 let manifest_key = result_doc
2123 .manifest_key
2124 .clone()
2125 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2126 if manifest_key.is_empty() {
2127 return Err(Error::ParseError(
2128 "result document manifestKey is empty".to_string(),
2129 ));
2130 }
2131 let payload_sha256 = result_doc
2132 .payload_sha256
2133 .clone()
2134 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2135 if payload_sha256.is_empty() {
2136 return Err(Error::ParseError(
2137 "result document payloadSha256 is empty".to_string(),
2138 ));
2139 }
2140 info!(
2141 restore = %restore.name_any(),
2142 backup_id = %expected_backup_id,
2143 manifest_key = %manifest_key,
2144 payload_sha256 = %payload_sha256,
2145 "source prep result verified"
2146 );
2147 Ok(VerifiedSourcePrepResult {
2148 manifest_key,
2149 payload_sha256,
2150 })
2151}
2152
2153fn source_job_pod_security_context(
2154 target: &Kanidm,
2155) -> k8s_openapi::api::core::v1::PodSecurityContext {
2156 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2157 security_context.seccomp_profile = Some(SeccompProfile {
2158 type_: "RuntimeDefault".to_string(),
2159 ..Default::default()
2160 });
2161 security_context
2162}
2163
2164fn hardened_security_context() -> SecurityContext {
2165 SecurityContext {
2166 allow_privilege_escalation: Some(false),
2167 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2168 drop: Some(vec!["ALL".to_string()]),
2169 ..Default::default()
2170 }),
2171 read_only_root_filesystem: Some(true),
2172 run_as_non_root: Some(true),
2173 seccomp_profile: Some(SeccompProfile {
2174 type_: "RuntimeDefault".to_string(),
2175 ..Default::default()
2176 }),
2177 ..Default::default()
2178 }
2179}
2180
2181fn kanidm_job_security_context() -> SecurityContext {
2182 SecurityContext {
2183 allow_privilege_escalation: Some(false),
2184 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2185 drop: Some(vec!["ALL".to_string()]),
2186 ..Default::default()
2187 }),
2188 seccomp_profile: Some(SeccompProfile {
2189 type_: "RuntimeDefault".to_string(),
2190 ..Default::default()
2191 }),
2192 ..Default::default()
2193 }
2194}
2195
2196fn job_resource_requirements() -> ResourceRequirements {
2197 ResourceRequirements {
2198 requests: Some(std::collections::BTreeMap::from([
2199 ("cpu".to_string(), Quantity("100m".to_string())),
2200 ("memory".to_string(), Quantity("128Mi".to_string())),
2201 ])),
2202 limits: Some(std::collections::BTreeMap::from([
2203 ("cpu".to_string(), Quantity("2".to_string())),
2204 ("memory".to_string(), Quantity("2Gi".to_string())),
2205 ])),
2206 ..Default::default()
2207 }
2208}
2209
2210async fn ensure_safety_backup_job(
2211 restore: &KanidmRestore,
2212 target: &Kanidm,
2213 ctx: &RestoreContext,
2214 name: &str,
2215 backup_id: &str,
2216 manifest_key: &str,
2217) -> Result<()> {
2218 let ns = restore.namespace().unwrap();
2219 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2220 if api
2221 .get_opt(name)
2222 .await
2223 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2224 .is_some()
2225 {
2226 return Ok(());
2227 }
2228 let safety = restore
2229 .spec
2230 .safety_backup
2231 .as_ref()
2232 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2233 let repo_ref = safety
2234 .repository_ref
2235 .as_ref()
2236 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2237 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2238 .get(&repo_ref.name)
2239 .await
2240 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2241 let _kanidm_uid = &restore.spec.target_ref.uid;
2242 let endpoint = &repo.spec.s3.endpoint;
2243 let region = &repo.spec.s3.region;
2244 let operation_doc = build_safety_upload_operation_doc(
2245 restore,
2246 target,
2247 backup_id,
2248 manifest_key,
2249 &repo.spec.s3.bucket,
2250 &repo.spec.s3.prefix,
2251 endpoint,
2252 region,
2253 repo.spec.s3.force_path_style,
2254 repo.spec.s3.insecure,
2255 repo.spec
2256 .s3
2257 .ca_bundle_ref
2258 .as_ref()
2259 .map(|_| ca_bundle_path())
2260 .as_deref(),
2261 repo.spec.encryption.as_ref(),
2262 )?;
2263 let operation_cm_name = format!("{name}-op");
2264 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2265 let auth_method = &repo.spec.authentication.writer;
2266 let data_mover_image = data_mover_image();
2267 let init_command = vec![
2268 "kanidmd".to_string(),
2269 "database".to_string(),
2270 "backup".to_string(),
2271 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2272 ];
2273 let upload_script = format!(
2274 r#"set -eu
2275/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2276RESULT_FILE="/run/kaniop-result/result.json"
2277TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2278if [ ! -f "$RESULT_FILE" ]; then
2279 echo "ERROR: result document not found at $RESULT_FILE"
2280 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2281 exit 1
2282fi
2283if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2284 echo "ERROR: safety backup upload reported failure"
2285 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2286 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2287 exit 1
2288fi
2289dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2290echo "safety backup upload completed successfully"
2291"#
2292 );
2293 let job = Job {
2294 metadata: ObjectMeta {
2295 name: Some(name.to_string()),
2296 namespace: Some(ns.clone()),
2297 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2298 ..Default::default()
2299 },
2300 spec: Some(JobSpec {
2301 backoff_limit: Some(0),
2302 template: PodTemplateSpec {
2303 metadata: Some(ObjectMeta::default()),
2304 spec: Some(PodSpec {
2305 automount_service_account_token: Some(false),
2306 restart_policy: Some("Never".to_string()),
2307 security_context: Some(source_job_pod_security_context(target)),
2308 init_containers: Some(vec![Container {
2309 name: "safety-backup".to_string(),
2310 image: Some(restore.spec.restore_image.clone()),
2311 command: Some(init_command),
2312 security_context: Some(SecurityContext {
2313 allow_privilege_escalation: Some(false),
2314 capabilities: Some(Capabilities {
2315 drop: Some(vec!["ALL".to_string()]),
2316 ..Default::default()
2317 }),
2318 read_only_root_filesystem: Some(true),
2319 seccomp_profile: Some(SeccompProfile {
2320 type_: "RuntimeDefault".to_string(),
2321 ..Default::default()
2322 }),
2323 ..Default::default()
2324 }),
2325 resources: Some(job_resource_requirements()),
2326 env: Some(build_kanidm_db_env_vars(target)),
2327 volume_mounts: Some(vec![
2328 VolumeMount {
2329 name: DATA_VOLUME.to_string(),
2330 mount_path: DATA_PATH.to_string(),
2331 read_only: Some(false),
2332 ..Default::default()
2333 },
2334 VolumeMount {
2335 name: TLS_VOLUME.to_string(),
2336 mount_path: TLS_PATH.to_string(),
2337 read_only: Some(true),
2338 ..Default::default()
2339 },
2340 VolumeMount {
2341 name: SHARED_VOLUME.to_string(),
2342 mount_path: SHARED_VOL_PATH.to_string(),
2343 ..Default::default()
2344 },
2345 ]),
2346 ..Default::default()
2347 }]),
2348 containers: vec![Container {
2349 name: "safety-upload".to_string(),
2350 image: Some(data_mover_image),
2351 command: Some(vec!["/bin/sh".to_string()]),
2352 args: Some(vec!["-c".to_string(), upload_script]),
2353 security_context: Some(hardened_security_context()),
2354 resources: Some(job_resource_requirements()),
2355 env: {
2356 let mut env_vars =
2357 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2358 env_vars.push(EnvVar {
2359 name: "RUST_LOG".to_string(),
2360 value: Some("info".to_string()),
2361 ..Default::default()
2362 });
2363 if repo.spec.s3.ca_bundle_ref.is_some() {
2364 env_vars.push(ca_bundle_env_var());
2365 }
2366 env_vars.extend(build_encryption_env_vars(
2367 repo.spec
2368 .encryption
2369 .as_ref()
2370 .and_then(|e| e.key_ref.as_ref()),
2371 ));
2372 Some(env_vars)
2373 },
2374 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2375 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2376 volume_mounts: {
2377 let mut mounts = vec![
2378 VolumeMount {
2379 name: SHARED_VOLUME.to_string(),
2380 mount_path: SHARED_VOL_PATH.to_string(),
2381 read_only: Some(true),
2382 ..Default::default()
2383 },
2384 VolumeMount {
2385 name: "operation".to_string(),
2386 mount_path: "/run/kaniop".to_string(),
2387 read_only: Some(true),
2388 ..Default::default()
2389 },
2390 VolumeMount {
2391 name: "result".to_string(),
2392 mount_path: "/run/kaniop-result".to_string(),
2393 ..Default::default()
2394 },
2395 ];
2396 mounts.extend(build_auth_volume_mounts(auth_method));
2397 if repo.spec.s3.ca_bundle_ref.is_some() {
2398 mounts.push(build_ca_bundle_volume_mount());
2399 }
2400 Some(mounts)
2401 },
2402 ..Default::default()
2403 }],
2404 volumes: {
2405 let mut vols = vec![
2406 Volume {
2407 name: DATA_VOLUME.to_string(),
2408 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2409 claim_name: primary_pvc_name(target)?,
2410 read_only: Some(false),
2411 }),
2412 ..Default::default()
2413 },
2414 Volume {
2415 name: TLS_VOLUME.to_string(),
2416 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2417 secret_name: Some(target.effective_tls_secret_name()),
2418 ..Default::default()
2419 }),
2420 ..Default::default()
2421 },
2422 Volume {
2423 name: SHARED_VOLUME.to_string(),
2424 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2425 size_limit: Some(crate::controller::backup_job_volume_size()),
2426 ..Default::default()
2427 }),
2428 ..Default::default()
2429 },
2430 Volume {
2431 name: "operation".to_string(),
2432 config_map: Some(ConfigMapVolumeSource {
2433 name: operation_cm_name,
2434 ..Default::default()
2435 }),
2436 ..Default::default()
2437 },
2438 Volume {
2439 name: "result".to_string(),
2440 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2441 size_limit: Some(Quantity("16Mi".to_string())),
2442 ..Default::default()
2443 }),
2444 ..Default::default()
2445 },
2446 ];
2447 vols.extend(build_auth_volumes(auth_method));
2448 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2449 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2450 }
2451 Some(vols)
2452 },
2453 ..Default::default()
2454 }),
2455 },
2456 ..Default::default()
2457 }),
2458 ..Default::default()
2459 };
2460 api.create(&PostParams::default(), &job)
2461 .await
2462 .map(|_| ())
2463 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2464}
2465
2466async fn ensure_source_prep_job(
2467 restore: &KanidmRestore,
2468 target: &Kanidm,
2469 ctx: &RestoreContext,
2470 name: &str,
2471) -> Result<()> {
2472 let ns = restore.namespace().unwrap();
2473 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2474 if api
2475 .get_opt(name)
2476 .await
2477 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2478 .is_some()
2479 {
2480 return Ok(());
2481 }
2482 let backup_name = restore
2483 .spec
2484 .source
2485 .backup_ref
2486 .as_ref()
2487 .map(|b| b.name.clone())
2488 .unwrap_or_default();
2489 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2490 .get(&backup_name)
2491 .await
2492 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2493 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2494 .get(&backup.spec.repository_ref.name)
2495 .await
2496 .map_err(|e| {
2497 Error::kube_error(
2498 "get",
2499 "KanidmBackupRepository",
2500 &ns,
2501 &backup.spec.repository_ref.name,
2502 e,
2503 )
2504 })?;
2505 let endpoint = &repo.spec.s3.endpoint;
2506 let region = &repo.spec.s3.region;
2507 let operation_doc = build_download_operation_doc(
2508 target,
2509 &backup.spec.manifest_key,
2510 &backup.spec.backup_id,
2511 &backup.spec.kanidm_ref.uid,
2512 &repo.spec.s3.bucket,
2513 &repo.spec.s3.prefix,
2514 endpoint,
2515 region,
2516 repo.spec.s3.force_path_style,
2517 repo.spec.s3.insecure,
2518 repo.spec
2519 .s3
2520 .ca_bundle_ref
2521 .as_ref()
2522 .map(|_| ca_bundle_path())
2523 .as_deref(),
2524 repo.spec.encryption.as_ref(),
2525 );
2526 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2527 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2528 let auth_method = &repo.spec.authentication.reader;
2529 let data_mover_image = data_mover_image();
2530 let download_script = format!(
2531 r#"set -eu
2532/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2533RESULT_FILE="/run/kaniop-result/result.json"
2534TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2535if [ ! -f "$RESULT_FILE" ]; then
2536 echo "ERROR: result document not found at $RESULT_FILE"
2537 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2538 exit 1
2539fi
2540if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2541 echo "ERROR: source preparation download reported failure"
2542 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2543 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2544 exit 1
2545fi
2546dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2547echo "source preparation download completed successfully"
2548"#
2549 );
2550 let job = Job {
2551 metadata: ObjectMeta {
2552 name: Some(name.to_string()),
2553 namespace: Some(ns.clone()),
2554 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2555 ..Default::default()
2556 },
2557 spec: Some(JobSpec {
2558 backoff_limit: Some(0),
2559 template: PodTemplateSpec {
2560 metadata: Some(ObjectMeta::default()),
2561 spec: Some(PodSpec {
2562 automount_service_account_token: Some(false),
2563 restart_policy: Some("Never".to_string()),
2564 security_context: Some(source_job_pod_security_context(target)),
2565 containers: vec![Container {
2566 name: "source-prep".to_string(),
2567 image: Some(data_mover_image),
2568 command: Some(vec!["/bin/sh".to_string()]),
2569 args: Some(vec!["-c".to_string(), download_script]),
2570 security_context: Some(hardened_security_context()),
2571 resources: Some(job_resource_requirements()),
2572 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2573 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2574 env: {
2575 let mut env_vars = build_auth_env_vars(
2576 auth_method,
2577 &backup.spec.repository_ref.name,
2578 AuthRole::Reader,
2579 );
2580 env_vars.push(EnvVar {
2581 name: "RUST_LOG".to_string(),
2582 value: Some("info".to_string()),
2583 ..Default::default()
2584 });
2585 if repo.spec.s3.ca_bundle_ref.is_some() {
2586 env_vars.push(ca_bundle_env_var());
2587 }
2588 env_vars.extend(build_encryption_env_vars(
2589 repo.spec
2590 .encryption
2591 .as_ref()
2592 .and_then(|e| e.key_ref.as_ref()),
2593 ));
2594 Some(env_vars)
2595 },
2596 volume_mounts: {
2597 let mut mounts = vec![
2598 VolumeMount {
2599 name: DATA_VOLUME.to_string(),
2600 mount_path: DATA_PATH.to_string(),
2601 ..Default::default()
2602 },
2603 VolumeMount {
2604 name: "operation".to_string(),
2605 mount_path: "/run/kaniop".to_string(),
2606 read_only: Some(true),
2607 ..Default::default()
2608 },
2609 VolumeMount {
2610 name: "result".to_string(),
2611 mount_path: "/run/kaniop-result".to_string(),
2612 ..Default::default()
2613 },
2614 ];
2615 mounts.extend(build_auth_volume_mounts(auth_method));
2616 if repo.spec.s3.ca_bundle_ref.is_some() {
2617 mounts.push(build_ca_bundle_volume_mount());
2618 }
2619 Some(mounts)
2620 },
2621 ..Default::default()
2622 }],
2623 volumes: {
2624 let mut vols = vec![
2625 Volume {
2626 name: DATA_VOLUME.to_string(),
2627 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2628 claim_name: primary_pvc_name(target)?,
2629 read_only: Some(false),
2630 }),
2631 ..Default::default()
2632 },
2633 Volume {
2634 name: "operation".to_string(),
2635 config_map: Some(ConfigMapVolumeSource {
2636 name: operation_cm_name,
2637 ..Default::default()
2638 }),
2639 ..Default::default()
2640 },
2641 Volume {
2642 name: "result".to_string(),
2643 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2644 size_limit: Some(Quantity("16Mi".to_string())),
2645 ..Default::default()
2646 }),
2647 ..Default::default()
2648 },
2649 ];
2650 vols.extend(build_auth_volumes(auth_method));
2651 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2652 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2653 }
2654 Some(vols)
2655 },
2656 ..Default::default()
2657 }),
2658 },
2659 ..Default::default()
2660 }),
2661 ..Default::default()
2662 };
2663 api.create(&PostParams::default(), &job)
2664 .await
2665 .map(|_| ())
2666 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2667}
2668
2669#[allow(clippy::too_many_arguments)]
2670fn build_safety_upload_operation_doc(
2671 restore: &KanidmRestore,
2672 target: &Kanidm,
2673 backup_id: &str,
2674 manifest_key: &str,
2675 bucket: &str,
2676 prefix: &str,
2677 endpoint: &str,
2678 region: &str,
2679 force_path_style: bool,
2680 insecure: bool,
2681 ca_bundle_path: Option<&str>,
2682 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2683) -> Result<String> {
2684 let enc_mode =
2685 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2686 let enc_key_id = encryption
2687 .and_then(|e| e.key_id.as_ref())
2688 .map(|s| serde_json::Value::String(s.clone()));
2689 let op = UploadOperation {
2690 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2691 bucket: bucket.to_string(),
2692 prefix: prefix.to_string(),
2693 endpoint: endpoint.to_string(),
2694 region: region.to_string(),
2695 force_path_style,
2696 insecure,
2697 ca_bundle_path: ca_bundle_path.map(str::to_string),
2698 backup_id: backup_id.to_string(),
2699 namespace_uid: restore.namespace().unwrap_or_default(),
2703 kanidm_uid: restore.spec.target_ref.uid.clone(),
2704 kanidm_name: restore.spec.target_ref.name.clone(),
2705 domain: target.spec.domain.clone(),
2706 kanidm_version: target
2707 .status
2708 .as_ref()
2709 .and_then(|s| s.version.as_ref())
2710 .map(|v| v.image_tag.clone())
2711 .unwrap_or_default(),
2712 image_digest: None,
2713 consistency: "kanidm-offline".to_string(),
2714 reason: "restore-safety".to_string(),
2715 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2716 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2717 result_path: "/run/kaniop-result/result.json".to_string(),
2718 max_concurrent_parts: 4,
2719 max_retries: 3,
2720 };
2721 let _ = manifest_key;
2722 let doc = OperationDocument {
2723 api_version: OPERATION_DOC_VERSION.to_string(),
2724 kind: "OperationDocument".to_string(),
2725 spec: OperationSpec::Upload(op),
2726 };
2727 serde_json::to_string(&doc)
2728 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2729}
2730
2731async fn validate_backup_ref(
2732 restore: &KanidmRestore,
2733 target: &Kanidm,
2734 ctx: &RestoreContext,
2735) -> Result<Option<DisasterRecoverySource>> {
2736 let backup_ref = restore
2737 .spec
2738 .source
2739 .backup_ref
2740 .as_ref()
2741 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2742 let ns = restore.namespace().unwrap();
2743 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2744 .get(&backup_ref.name)
2745 .await
2746 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2747 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2748 return Err(Error::MissingData(format!(
2749 "KanidmBackup '{}' is not Ready (phase: {:?})",
2750 backup_ref.name,
2751 backup.status.as_ref().map(|s| &s.phase)
2752 )));
2753 }
2754 let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2755 || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2756 let disaster_recovery_source = if source_identity_mismatch {
2757 if !disaster_recovery_requested(restore) {
2758 return Err(Error::MissingData(format!(
2759 "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2760 backup.spec.kanidm_ref.name,
2761 backup.spec.kanidm_ref.uid,
2762 restore.spec.target_ref.name,
2763 restore.spec.target_ref.uid,
2764 DISASTER_RECOVERY_ANNOTATION,
2765 BREAK_GLASS_REASON_ANNOTATION,
2766 BREAK_GLASS_APPROVED_BY_ANNOTATION,
2767 )));
2768 }
2769 validate_disaster_recovery_approval(restore)?;
2770 Some(DisasterRecoverySource {
2771 name: backup.spec.kanidm_ref.name.clone(),
2772 uid: backup.spec.kanidm_ref.uid.clone(),
2773 })
2774 } else {
2775 None
2776 };
2777 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2778 .get(&backup.spec.repository_ref.name)
2779 .await
2780 .map_err(|e| {
2781 Error::kube_error(
2782 "get",
2783 "KanidmBackupRepository",
2784 &ns,
2785 &backup.spec.repository_ref.name,
2786 e,
2787 )
2788 })?;
2789 if !has_accepted_condition(
2790 &repo
2791 .status
2792 .as_ref()
2793 .map(|s| &s.conditions)
2794 .cloned()
2795 .unwrap_or_default(),
2796 ) {
2797 return Err(Error::MissingData(format!(
2798 "KanidmBackupRepository '{}' configuration has not been accepted",
2799 backup.spec.repository_ref.name
2800 )));
2801 }
2802 validate_backup_compatibility(restore, target, &backup)?;
2803 Ok(disaster_recovery_source)
2804}
2805
2806fn has_accepted_condition(conditions: &[Condition]) -> bool {
2807 conditions
2808 .iter()
2809 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2810}
2811
2812fn validate_backup_compatibility(
2813 restore: &KanidmRestore,
2814 target: &Kanidm,
2815 backup: &KanidmBackup,
2816) -> Result<()> {
2817 let status = backup.status.as_ref();
2818 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2819 let target_version = target
2820 .status
2821 .as_ref()
2822 .and_then(|s| s.version.as_ref())
2823 .map(|v| v.image_tag.as_str());
2824 if let Some(tv) = target_version {
2825 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2826 return Err(Error::MissingData(format!(
2827 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2828 )));
2829 }
2830 }
2831 }
2832 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2833 if !backup_digest.is_empty()
2834 && restore.spec.restore_image.contains('@')
2835 && !restore.spec.restore_image.contains(backup_digest)
2836 {
2837 return Err(Error::MissingData(format!(
2838 "restore image digest does not match backup image digest '{backup_digest}'"
2839 )));
2840 }
2841 }
2842 Ok(())
2843}
2844
2845#[allow(clippy::too_many_arguments)]
2846fn build_download_operation_doc(
2847 target: &Kanidm,
2848 manifest_key: &str,
2849 expected_backup_id: &str,
2850 expected_source_uid: &str,
2851 bucket: &str,
2852 prefix: &str,
2853 endpoint: &str,
2854 region: &str,
2855 force_path_style: bool,
2856 insecure: bool,
2857 ca_bundle_path: Option<&str>,
2858 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2859) -> String {
2860 let enc_mode =
2861 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2862 let enc_key_id = encryption
2863 .and_then(|e| e.key_id.as_ref())
2864 .map(|s| serde_json::Value::String(s.clone()));
2865 serde_json::json!({
2866 "apiVersion": "backup.kaniop.rs/v1alpha1",
2867 "kind": "OperationDocument",
2868 "operation": "download",
2869 "manifestKey": manifest_key,
2870 "bucket": bucket,
2871 "prefix": prefix,
2872 "endpoint": endpoint,
2873 "region": region,
2874 "forcePathStyle": force_path_style,
2875 "insecure": insecure,
2876 "caBundlePath": ca_bundle_path,
2877 "expectedBackupId": expected_backup_id,
2878 "expectedKanidmUid": expected_source_uid,
2879 "expectedDomain": target.spec.domain,
2880 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2881 "resultPath": "/run/kaniop-result/result.json",
2882 "maxRetries": 3,
2883 "encryptionMode": enc_mode,
2884 "encryptionKeyId": enc_key_id,
2885 })
2886 .to_string()
2887}
2888
2889async fn ensure_operation_configmap(
2890 restore: &KanidmRestore,
2891 name: &str,
2892 data: &str,
2893 ns: &str,
2894 ctx: &RestoreContext,
2895) -> Result<()> {
2896 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2897 if api
2898 .get_opt(name)
2899 .await
2900 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2901 .is_some()
2902 {
2903 return Ok(());
2904 }
2905 let cm = ConfigMap {
2906 metadata: ObjectMeta {
2907 name: Some(name.to_string()),
2908 namespace: Some(ns.to_string()),
2909 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2910 ..Default::default()
2911 },
2912 data: Some(std::collections::BTreeMap::from([(
2913 "operation.json".to_string(),
2914 data.to_string(),
2915 )])),
2916 ..Default::default()
2917 };
2918 api.create(&PostParams::default(), &cm)
2919 .await
2920 .map(|_| ())
2921 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2922}
2923
2924#[cfg(test)]
2925mod tests {
2926 use super::{
2927 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2928 CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2929 DISASTER_RECOVERY_ANNOTATION,
2930 KanidmRestore, KanidmRestorePhase,
2931 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2932 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2933 disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2934 is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2935 restore_job_name, safe_basename, validate_disaster_recovery_approval,
2936 validate_safety_backup_config, validate_source, verify_job_name,
2937 };
2938 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2939 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2940 use k8s_openapi::jiff::Timestamp;
2941 use kube::api::ObjectMeta as ApiObjectMeta;
2942 use std::collections::BTreeMap;
2943
2944 fn make_restore(
2945 source: KanidmRestoreSource,
2946 safety: Option<SafetyBackupConfig>,
2947 ) -> KanidmRestore {
2948 KanidmRestore {
2949 metadata: ApiObjectMeta {
2950 name: Some("test-restore".to_string()),
2951 namespace: Some("default".to_string()),
2952 uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2953 ..Default::default()
2954 },
2955 spec: KanidmRestoreSpec {
2956 target_ref: KanidmRestoreTargetRef {
2957 name: "corp-idm".to_string(),
2958 uid: "test-uid-123".to_string(),
2959 },
2960 source,
2961 restore_image: "kanidm/server@sha256:abc".to_string(),
2962 safety_backup: safety,
2963 },
2964 status: None,
2965 }
2966 }
2967
2968 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2969 super::super::crd::Kanidm {
2970 status: Some(super::super::crd::KanidmStatus {
2971 available_replicas: 1,
2972 replicas: 1,
2973 unavailable_replicas: 0,
2974 updated_replicas: 1,
2975 replica_statuses: vec![],
2976 replica_column: "1/1".to_string(),
2977 secret_name: None,
2978 version: Some(super::super::crd::KanidmVersionStatus {
2979 image_tag: image_tag.to_string(),
2980 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2981 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2982 }),
2983 domain_appearance_image: None,
2984 mail_sender: None,
2985 conditions: None,
2986 }),
2987 ..Default::default()
2988 }
2989 }
2990
2991 #[test]
2992 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2993 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2994 }
2995
2996 #[test]
2997 fn completed_restore_sets_ready_condition() {
2998 let mut status = KanidmRestoreStatus {
2999 phase: KanidmRestorePhase::Completed,
3000 ..Default::default()
3001 };
3002 super::update_restore_conditions(&mut status, Some(7));
3003 assert!(status.conditions.iter().any(|condition| {
3004 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3005 }));
3006 assert!(!status.conditions.iter().any(|condition| {
3007 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3008 }));
3009 }
3010
3011 #[test]
3012 fn disaster_recovery_condition_survives_phase_updates() {
3013 let dr_condition = Condition {
3014 type_: CONDITION_DISASTER_RECOVERY.to_string(),
3015 status: CONDITION_TRUE.to_string(),
3016 reason: "ApprovedSourceIdentityMismatch".to_string(),
3017 message: "approved DR".to_string(),
3018 last_transition_time: Time(Timestamp::now()),
3019 observed_generation: Some(7),
3020 };
3021 let mut status = KanidmRestoreStatus {
3022 phase: KanidmRestorePhase::Quiescing,
3023 conditions: vec![dr_condition.clone()],
3024 ..Default::default()
3025 };
3026
3027 super::update_restore_conditions(&mut status, Some(8));
3028
3029 assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3030 }
3031
3032 #[test]
3033 fn failed_restore_sets_failed_condition() {
3034 let mut status = KanidmRestoreStatus {
3035 phase: KanidmRestorePhase::Failed,
3036 message: Some("verification failed".to_string()),
3037 ..Default::default()
3038 };
3039 super::update_restore_conditions(&mut status, Some(8));
3040 assert!(status.conditions.iter().any(|condition| {
3041 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3042 }));
3043 }
3044
3045 #[test]
3046 fn restore_filename_is_confined_to_backup_directory() {
3047 assert!(safe_basename("backup.json.gz"));
3048 assert!(!safe_basename("../backup.json.gz"));
3049 assert!(!safe_basename("a/b"));
3050 assert!(!safe_basename(""));
3051 }
3052
3053 #[test]
3054 fn restore_image_rejects_latest_and_unpinned_names() {
3055 assert!(mutable_image("kanidm/server:latest"));
3056 assert!(mutable_image("kanidm/server"));
3057 assert!(!mutable_image("kanidm/server:1.10.0"));
3058 assert!(!mutable_image("kanidm/server:1.11.2@sha256:serverdigest"));
3059 assert!(!mutable_image("kanidm/server@sha256:abc"));
3060 assert!(mutable_image("registry.example.com:5000/kanidm/server"));
3061 }
3062
3063 #[test]
3064 fn source_validation_rejects_both_local_and_backup_ref() {
3065 let source = KanidmRestoreSource {
3066 local: Some(super::KanidmRestoreLocalSource {
3067 file_name: "backup.json".to_string(),
3068 }),
3069 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3070 name: "some-backup".to_string(),
3071 }),
3072 };
3073 let restore = make_restore(source, None);
3074 assert!(validate_source(&restore).is_err());
3075 }
3076
3077 #[test]
3078 fn source_validation_rejects_empty_source() {
3079 let source = KanidmRestoreSource {
3080 local: None,
3081 backup_ref: None,
3082 };
3083 let restore = make_restore(source, None);
3084 assert!(validate_source(&restore).is_err());
3085 }
3086
3087 #[test]
3088 fn source_validation_accepts_local_source() {
3089 let source = KanidmRestoreSource {
3090 local: Some(super::KanidmRestoreLocalSource {
3091 file_name: "backup.json".to_string(),
3092 }),
3093 backup_ref: None,
3094 };
3095 let restore = make_restore(source, None);
3096 assert!(validate_source(&restore).is_ok());
3097 }
3098
3099 #[test]
3100 fn source_validation_accepts_backup_ref_source() {
3101 let source = KanidmRestoreSource {
3102 local: None,
3103 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3104 name: "corp-idm-019c7c76".to_string(),
3105 }),
3106 };
3107 let restore = make_restore(source, None);
3108 assert!(validate_source(&restore).is_ok());
3109 }
3110
3111 #[test]
3112 fn source_validation_rejects_empty_backup_ref_name() {
3113 let source = KanidmRestoreSource {
3114 local: None,
3115 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3116 name: "".to_string(),
3117 }),
3118 };
3119 let restore = make_restore(source, None);
3120 assert!(validate_source(&restore).is_err());
3121 }
3122
3123 #[test]
3124 fn is_remote_source_true_for_backup_ref() {
3125 let source = KanidmRestoreSource {
3126 local: None,
3127 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3128 name: "backup-1".to_string(),
3129 }),
3130 };
3131 let restore = make_restore(source, None);
3132 assert!(is_remote_source(&restore));
3133 }
3134
3135 #[test]
3136 fn is_remote_source_false_for_local() {
3137 let source = KanidmRestoreSource {
3138 local: Some(super::KanidmRestoreLocalSource {
3139 file_name: "backup.json".to_string(),
3140 }),
3141 backup_ref: None,
3142 };
3143 let restore = make_restore(source, None);
3144 assert!(!is_remote_source(&restore));
3145 }
3146
3147 #[test]
3148 fn requires_safety_backup_default_true() {
3149 let source = KanidmRestoreSource {
3150 local: None,
3151 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3152 name: "backup-1".to_string(),
3153 }),
3154 };
3155 let restore = make_restore(source, None);
3156 assert!(requires_safety_backup(&restore));
3157 }
3158
3159 #[test]
3160 fn requires_safety_backup_false_when_skip() {
3161 let source = KanidmRestoreSource {
3162 local: None,
3163 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3164 name: "backup-1".to_string(),
3165 }),
3166 };
3167 let safety = SafetyBackupConfig {
3168 repository_ref: Some(SafetyBackupRepositoryRef {
3169 name: "offsite".to_string(),
3170 }),
3171 skip: true,
3172 };
3173 let restore = make_restore(source, Some(safety));
3174 assert!(!requires_safety_backup(&restore));
3175 }
3176
3177 #[test]
3178 fn break_glass_validation_rejects_skip_without_reason() {
3179 let source = KanidmRestoreSource {
3180 local: None,
3181 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3182 name: "backup-1".to_string(),
3183 }),
3184 };
3185 let safety = SafetyBackupConfig {
3186 repository_ref: Some(SafetyBackupRepositoryRef {
3187 name: "offsite".to_string(),
3188 }),
3189 skip: true,
3190 };
3191 let mut restore = make_restore(source, Some(safety));
3192 restore.metadata.annotations = Some(BTreeMap::from([(
3193 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3194 "commander@example.com".to_string(),
3195 )]));
3196 assert!(validate_safety_backup_config(&restore).is_err());
3197 }
3198
3199 #[test]
3200 fn break_glass_validation_rejects_skip_without_approver() {
3201 let source = KanidmRestoreSource {
3202 local: None,
3203 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3204 name: "backup-1".to_string(),
3205 }),
3206 };
3207 let safety = SafetyBackupConfig {
3208 repository_ref: Some(SafetyBackupRepositoryRef {
3209 name: "offsite".to_string(),
3210 }),
3211 skip: true,
3212 };
3213 let mut restore = make_restore(source, Some(safety));
3214 restore.metadata.annotations = Some(BTreeMap::from([(
3215 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3216 "PVC is unreadable".to_string(),
3217 )]));
3218 assert!(validate_safety_backup_config(&restore).is_err());
3219 }
3220
3221 #[test]
3222 fn break_glass_validation_accepts_valid_annotations() {
3223 let source = KanidmRestoreSource {
3224 local: None,
3225 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3226 name: "backup-1".to_string(),
3227 }),
3228 };
3229 let safety = SafetyBackupConfig {
3230 repository_ref: Some(SafetyBackupRepositoryRef {
3231 name: "offsite".to_string(),
3232 }),
3233 skip: true,
3234 };
3235 let mut restore = make_restore(source, Some(safety));
3236 restore.metadata.annotations = Some(BTreeMap::from([
3237 (
3238 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3239 "PVC is unreadable".to_string(),
3240 ),
3241 (
3242 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3243 "commander@example.com".to_string(),
3244 ),
3245 ]));
3246 assert!(validate_safety_backup_config(&restore).is_ok());
3247 }
3248
3249 #[test]
3250 fn disaster_recovery_requires_explicit_approval_annotations() {
3251 let source = KanidmRestoreSource {
3252 local: None,
3253 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3254 name: "backup-1".to_string(),
3255 }),
3256 };
3257 let mut restore = make_restore(source, None);
3258 restore.metadata.annotations = Some(BTreeMap::from([(
3259 DISASTER_RECOVERY_ANNOTATION.to_string(),
3260 "true".to_string(),
3261 )]));
3262 assert!(disaster_recovery_requested(&restore));
3263 assert!(validate_disaster_recovery_approval(&restore).is_err());
3264 }
3265
3266 #[test]
3267 fn disaster_recovery_accepts_reason_and_approver() {
3268 let source = KanidmRestoreSource {
3269 local: None,
3270 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3271 name: "backup-1".to_string(),
3272 }),
3273 };
3274 let mut restore = make_restore(source, None);
3275 restore.metadata.annotations = Some(BTreeMap::from([
3276 (
3277 DISASTER_RECOVERY_ANNOTATION.to_string(),
3278 "true".to_string(),
3279 ),
3280 (
3281 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3282 "original cluster lost".to_string(),
3283 ),
3284 (
3285 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3286 "incident-commander".to_string(),
3287 ),
3288 ]));
3289 assert!(disaster_recovery_requested(&restore));
3290 assert!(validate_disaster_recovery_approval(&restore).is_ok());
3291 }
3292
3293 #[test]
3294 fn remote_source_requires_repository_ref_when_not_skipping() {
3295 let source = KanidmRestoreSource {
3296 local: None,
3297 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3298 name: "backup-1".to_string(),
3299 }),
3300 };
3301 let safety = SafetyBackupConfig {
3302 repository_ref: None,
3303 skip: false,
3304 };
3305 let restore = make_restore(source, Some(safety));
3306 assert!(validate_safety_backup_config(&restore).is_err());
3307 }
3308
3309 #[test]
3310 fn remote_source_accepts_repository_ref() {
3311 let source = KanidmRestoreSource {
3312 local: None,
3313 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3314 name: "backup-1".to_string(),
3315 }),
3316 };
3317 let safety = SafetyBackupConfig {
3318 repository_ref: Some(SafetyBackupRepositoryRef {
3319 name: "offsite".to_string(),
3320 }),
3321 skip: false,
3322 };
3323 let restore = make_restore(source, Some(safety));
3324 assert!(validate_safety_backup_config(&restore).is_ok());
3325 }
3326
3327 #[test]
3328 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3329 let phases = vec![
3330 KanidmRestorePhase::Pending,
3331 KanidmRestorePhase::Validating,
3332 KanidmRestorePhase::Quiescing,
3333 KanidmRestorePhase::SafetyBackup,
3334 KanidmRestorePhase::PreparingSource,
3335 KanidmRestorePhase::RestoringPrimary,
3336 KanidmRestorePhase::Verifying,
3337 KanidmRestorePhase::RebuildingReplicas,
3338 KanidmRestorePhase::Resuming,
3339 KanidmRestorePhase::Completed,
3340 ];
3341 assert_eq!(phases.len(), 10);
3342 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3343 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3344 }
3345
3346 #[test]
3347 fn status_default_has_empty_phase_timestamps() {
3348 let status = KanidmRestoreStatus::default();
3349 assert!(status.phase_timestamps.is_empty());
3350 }
3351
3352 #[test]
3353 fn status_default_has_empty_original_replicas() {
3354 let status = KanidmRestoreStatus::default();
3355 assert!(status.original_replicas.is_empty());
3356 }
3357
3358 #[test]
3359 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3360 let mut target = super::super::crd::Kanidm::default();
3361 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3362 run_as_non_root: Some(false),
3363 run_as_user: Some(389),
3364 run_as_group: Some(389),
3365 fs_group: Some(389),
3366 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3367 supplemental_groups: Some(vec![390, 391]),
3368 ..Default::default()
3369 });
3370
3371 let ctx = super::source_job_pod_security_context(&target);
3372
3373 assert_eq!(ctx.run_as_non_root, Some(false));
3374 assert_eq!(ctx.run_as_user, Some(389));
3375 assert_eq!(ctx.run_as_group, Some(389));
3376 assert_eq!(ctx.fs_group, Some(389));
3377 assert_eq!(
3378 ctx.fs_group_change_policy.as_deref(),
3379 Some("OnRootMismatch")
3380 );
3381 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3382 assert_eq!(
3383 ctx.seccomp_profile
3384 .as_ref()
3385 .map(|profile| profile.type_.as_str()),
3386 Some("RuntimeDefault")
3387 );
3388 }
3389
3390 #[test]
3391 fn hardened_security_context_drops_all_capabilities() {
3392 let ctx = hardened_security_context();
3393 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3394 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3395 assert_eq!(ctx.run_as_non_root, Some(true));
3396 let caps = ctx.capabilities.unwrap();
3397 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3398 assert_eq!(
3399 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3400 Some("RuntimeDefault")
3401 );
3402 }
3403
3404 #[test]
3405 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3406 let ctx = kanidm_job_security_context();
3407 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3408 assert!(ctx.run_as_non_root.is_none());
3409 assert!(ctx.read_only_root_filesystem.is_none());
3410 let caps = ctx.capabilities.unwrap();
3411 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3412 assert_eq!(
3413 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3414 Some("RuntimeDefault")
3415 );
3416 }
3417
3418 #[test]
3419 fn safety_backup_job_name_is_deterministic() {
3420 let source = KanidmRestoreSource {
3421 local: Some(super::KanidmRestoreLocalSource {
3422 file_name: "backup.json".to_string(),
3423 }),
3424 backup_ref: None,
3425 };
3426 let restore = make_restore(source, None);
3427 let name1 = super::safety_backup_job_name(&restore);
3428 let name2 = super::safety_backup_job_name(&restore);
3429 assert_eq!(name1, name2);
3430 assert_eq!(name1, "test-restore-safety-backup");
3431 }
3432
3433 #[test]
3434 fn source_prep_job_name_is_deterministic() {
3435 let source = KanidmRestoreSource {
3436 local: None,
3437 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3438 name: "backup-1".to_string(),
3439 }),
3440 };
3441 let restore = make_restore(source, None);
3442 let name = super::source_prep_job_name(&restore);
3443 assert_eq!(name, "test-restore-source-prep");
3444 }
3445
3446 #[test]
3447 fn break_glass_condition_is_recorded_in_status() {
3448 let mut status = KanidmRestoreStatus::default();
3449 let condition = super::restore_condition(
3450 &status.conditions,
3451 CONDITION_BREAK_GLASS,
3452 CONDITION_TRUE,
3453 "BreakGlassOverride",
3454 "Safety backup skipped",
3455 Some(1),
3456 );
3457 status.conditions.push(condition);
3458 assert!(status.conditions.iter().any(|c| {
3459 c.type_ == CONDITION_BREAK_GLASS
3460 && c.status == CONDITION_TRUE
3461 && c.reason == "BreakGlassOverride"
3462 }));
3463 }
3464
3465 #[test]
3466 fn replica_count_entry_serialization() {
3467 let entry = ReplicaCountEntry {
3468 group: "primary".to_string(),
3469 replicas: 3,
3470 };
3471 let json = serde_json::to_string(&entry).unwrap();
3472 assert!(json.contains("\"group\":\"primary\""));
3473 assert!(json.contains("\"replicas\":3"));
3474 }
3475
3476 #[test]
3477 fn local_source_requires_repository_ref_when_not_skipping() {
3478 let source = KanidmRestoreSource {
3479 local: Some(super::KanidmRestoreLocalSource {
3480 file_name: "backup.json".to_string(),
3481 }),
3482 backup_ref: None,
3483 };
3484 let safety = SafetyBackupConfig {
3485 repository_ref: None,
3486 skip: false,
3487 };
3488 let restore = make_restore(source, Some(safety));
3489 assert!(validate_safety_backup_config(&restore).is_err());
3490 }
3491
3492 #[test]
3493 fn local_source_accepts_repository_ref_when_not_skipping() {
3494 let source = KanidmRestoreSource {
3495 local: Some(super::KanidmRestoreLocalSource {
3496 file_name: "backup.json".to_string(),
3497 }),
3498 backup_ref: None,
3499 };
3500 let safety = SafetyBackupConfig {
3501 repository_ref: Some(SafetyBackupRepositoryRef {
3502 name: "offsite".to_string(),
3503 }),
3504 skip: false,
3505 };
3506 let restore = make_restore(source, Some(safety));
3507 assert!(validate_safety_backup_config(&restore).is_ok());
3508 }
3509
3510 #[test]
3511 fn skip_without_break_glass_annotations_is_rejected() {
3512 let source = KanidmRestoreSource {
3513 local: Some(super::KanidmRestoreLocalSource {
3514 file_name: "backup.json".to_string(),
3515 }),
3516 backup_ref: None,
3517 };
3518 let safety = SafetyBackupConfig {
3519 repository_ref: None,
3520 skip: true,
3521 };
3522 let restore = make_restore(source, Some(safety));
3523 assert!(validate_safety_backup_config(&restore).is_err());
3524 }
3525
3526 #[test]
3527 fn has_accepted_condition_true() {
3528 let conditions = vec![Condition {
3529 type_: "Ready".to_string(),
3530 status: "True".to_string(),
3531 reason: "Accepted".to_string(),
3532 message: String::new(),
3533 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3534 k8s_openapi::jiff::Timestamp::now(),
3535 ),
3536 observed_generation: None,
3537 }];
3538 assert!(has_accepted_condition(&conditions));
3539 }
3540
3541 #[test]
3542 fn has_accepted_condition_false_when_not_accepted_reason() {
3543 let conditions = vec![Condition {
3544 type_: "Ready".to_string(),
3545 status: "True".to_string(),
3546 reason: "SomeOtherReason".to_string(),
3547 message: String::new(),
3548 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3549 k8s_openapi::jiff::Timestamp::now(),
3550 ),
3551 observed_generation: None,
3552 }];
3553 assert!(!has_accepted_condition(&conditions));
3554 }
3555
3556 #[test]
3557 fn has_accepted_condition_false_when_status_false() {
3558 let conditions = vec![Condition {
3559 type_: "Ready".to_string(),
3560 status: "False".to_string(),
3561 reason: "Accepted".to_string(),
3562 message: String::new(),
3563 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3564 k8s_openapi::jiff::Timestamp::now(),
3565 ),
3566 observed_generation: None,
3567 }];
3568 assert!(!has_accepted_condition(&conditions));
3569 }
3570
3571 #[test]
3572 fn has_accepted_condition_empty() {
3573 assert!(!has_accepted_condition(&[]));
3574 }
3575
3576 #[test]
3577 fn safety_backup_upload_operation_doc_is_valid() {
3578 let source = KanidmRestoreSource {
3579 local: None,
3580 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3581 name: "backup-1".to_string(),
3582 }),
3583 };
3584 let mut restore = make_restore(source, None);
3585 restore.metadata.namespace = Some("test-ns".to_string());
3586 let target = super::super::crd::Kanidm::default();
3587 let doc_str = super::build_safety_upload_operation_doc(
3588 &restore,
3589 &target,
3590 "test-backup-id",
3591 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3592 "my-bucket",
3593 "prod",
3594 "https://s3.example.com",
3595 "us-east-1",
3596 false,
3597 false,
3598 None,
3599 None,
3600 )
3601 .unwrap();
3602 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3603 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3604 assert_eq!(parsed["kind"], "OperationDocument");
3605 assert_eq!(parsed["operation"], "upload");
3606 assert_eq!(parsed["bucket"], "my-bucket");
3607 assert_eq!(parsed["prefix"], "prod");
3608 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3609 assert_eq!(parsed["region"], "us-east-1");
3610 assert_eq!(parsed["backupId"], "test-backup-id");
3611 assert_eq!(parsed["consistency"], "kanidm-offline");
3612 assert_eq!(parsed["reason"], "restore-safety");
3613 assert_eq!(
3614 parsed["payloadPath"],
3615 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3616 );
3617 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3618 }
3619
3620 #[test]
3621 fn download_operation_doc_uses_real_repository_data() {
3622 let source = KanidmRestoreSource {
3623 local: None,
3624 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3625 name: "backup-1".to_string(),
3626 }),
3627 };
3628 let _restore = make_restore(source, None);
3629 let target = super::super::crd::Kanidm::default();
3630 let doc_str = super::build_download_operation_doc(
3631 &target,
3632 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3633 "019c7c76-f423-7a12-8f41-2bea7588a303",
3634 "source-uid-456",
3635 "real-bucket",
3636 "real-prefix",
3637 "https://real-endpoint.com",
3638 "eu-west-1",
3639 true,
3640 false,
3641 None,
3642 None,
3643 );
3644 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3645 assert_eq!(parsed["bucket"], "real-bucket");
3646 assert_eq!(parsed["prefix"], "real-prefix");
3647 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3648 assert_eq!(parsed["region"], "eu-west-1");
3649 assert_eq!(parsed["forcePathStyle"], true);
3650 assert_eq!(
3651 parsed["manifestKey"],
3652 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3653 );
3654 assert_eq!(
3655 parsed["expectedBackupId"],
3656 "019c7c76-f423-7a12-8f41-2bea7588a303"
3657 );
3658 assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3659 }
3660
3661 #[test]
3662 fn safety_backup_ref_is_set_after_verification() {
3663 let status = KanidmRestoreStatus {
3664 phase: KanidmRestorePhase::SafetyBackup,
3665 safety_backup_job_name: Some("test-safety-backup".to_string()),
3666 safety_backup_ref: Some("safety-test-restore".to_string()),
3667 safety_backup_expected_backup_id: Some(
3668 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3669 ),
3670 safety_backup_manifest_key: Some(
3671 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3672 ),
3673 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3674 ..Default::default()
3675 };
3676 assert!(status.safety_backup_ref.is_some());
3677 assert!(
3678 status
3679 .safety_backup_ref
3680 .as_ref()
3681 .unwrap()
3682 .starts_with("safety-")
3683 );
3684 assert!(status.safety_backup_expected_backup_id.is_some());
3685 assert!(status.safety_backup_manifest_key.is_some());
3686 assert!(status.safety_backup_payload_sha256.is_some());
3687 }
3688
3689 #[test]
3690 fn database_mutation_not_started_without_safety_backup_ref() {
3691 let status = KanidmRestoreStatus {
3692 phase: KanidmRestorePhase::PreparingSource,
3693 safety_backup_ref: None,
3694 database_mutation_started: false,
3695 ..Default::default()
3696 };
3697 assert!(!status.database_mutation_started);
3698 assert!(status.safety_backup_ref.is_none());
3699 }
3700
3701 #[test]
3702 fn pre_boundary_failure_does_not_set_mutation_started() {
3703 let status = KanidmRestoreStatus {
3704 phase: KanidmRestorePhase::Failed,
3705 message: Some(
3706 "safety backup job failed; target restored to original state".to_string(),
3707 ),
3708 database_mutation_started: false,
3709 ..Default::default()
3710 };
3711 assert!(!status.database_mutation_started);
3712 }
3713
3714 #[test]
3715 fn shared_volume_constants_are_distinct() {
3716 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3717 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3718 }
3719
3720 #[test]
3721 fn validate_backup_compatibility_matching_versions_passes() {
3722 let source = KanidmRestoreSource {
3723 local: None,
3724 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3725 name: "backup-1".to_string(),
3726 }),
3727 };
3728 let restore = make_restore(source, None);
3729 let target = kanidm_with_version("1.10.4");
3730 let backup = kaniop_backup_core::crd::KanidmBackup {
3731 metadata: kube::api::ObjectMeta::default(),
3732 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3733 backup_id: "id".to_string(),
3734 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3735 name: "k".to_string(),
3736 uid: "uid".to_string(),
3737 },
3738 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3739 name: "repo".to_string(),
3740 },
3741 manifest_key: "key".to_string(),
3742 },
3743 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3744 kanidm_version: Some("1.10.4".to_string()),
3745 ..Default::default()
3746 }),
3747 };
3748 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3749 }
3750
3751 #[test]
3752 fn validate_backup_compatibility_version_mismatch_fails() {
3753 let source = KanidmRestoreSource {
3754 local: None,
3755 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3756 name: "backup-1".to_string(),
3757 }),
3758 };
3759 let restore = make_restore(source, None);
3760 let target = kanidm_with_version("1.11.0");
3761 let backup = kaniop_backup_core::crd::KanidmBackup {
3762 metadata: kube::api::ObjectMeta::default(),
3763 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3764 backup_id: "id".to_string(),
3765 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3766 name: "k".to_string(),
3767 uid: "uid".to_string(),
3768 },
3769 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3770 name: "repo".to_string(),
3771 },
3772 manifest_key: "key".to_string(),
3773 },
3774 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3775 kanidm_version: Some("1.10.4".to_string()),
3776 ..Default::default()
3777 }),
3778 };
3779 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3780 }
3781
3782 #[test]
3783 fn validate_backup_compatibility_no_backup_version_passes() {
3784 let source = KanidmRestoreSource {
3785 local: None,
3786 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3787 name: "backup-1".to_string(),
3788 }),
3789 };
3790 let restore = make_restore(source, None);
3791 let target = super::super::crd::Kanidm::default();
3792 let backup = kaniop_backup_core::crd::KanidmBackup {
3793 metadata: kube::api::ObjectMeta::default(),
3794 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3795 backup_id: "id".to_string(),
3796 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3797 name: "k".to_string(),
3798 uid: "uid".to_string(),
3799 },
3800 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3801 name: "repo".to_string(),
3802 },
3803 manifest_key: "key".to_string(),
3804 },
3805 status: None,
3806 };
3807 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3808 }
3809
3810 #[test]
3811 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3812 let source = KanidmRestoreSource {
3813 local: None,
3814 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3815 name: "backup-1".to_string(),
3816 }),
3817 };
3818 let mut restore = make_restore(source, None);
3819 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3820 let target = super::super::crd::Kanidm::default();
3821 let backup = kaniop_backup_core::crd::KanidmBackup {
3822 metadata: kube::api::ObjectMeta::default(),
3823 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3824 backup_id: "id".to_string(),
3825 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3826 name: "k".to_string(),
3827 uid: "uid".to_string(),
3828 },
3829 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3830 name: "repo".to_string(),
3831 },
3832 manifest_key: "key".to_string(),
3833 },
3834 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3835 image_digest: Some("sha256:originaldigest".to_string()),
3836 ..Default::default()
3837 }),
3838 };
3839 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3840 }
3841
3842 #[test]
3843 fn validate_backup_compatibility_image_digest_match_passes() {
3844 let source = KanidmRestoreSource {
3845 local: None,
3846 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3847 name: "backup-1".to_string(),
3848 }),
3849 };
3850 let mut restore = make_restore(source, None);
3851 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3852 let target = super::super::crd::Kanidm::default();
3853 let backup = kaniop_backup_core::crd::KanidmBackup {
3854 metadata: kube::api::ObjectMeta::default(),
3855 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3856 backup_id: "id".to_string(),
3857 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3858 name: "k".to_string(),
3859 uid: "uid".to_string(),
3860 },
3861 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3862 name: "repo".to_string(),
3863 },
3864 manifest_key: "key".to_string(),
3865 },
3866 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3867 image_digest: Some("sha256:abc123".to_string()),
3868 ..Default::default()
3869 }),
3870 };
3871 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3872 }
3873
3874 #[test]
3875 fn safety_backup_id_is_deterministic() {
3876 let source = KanidmRestoreSource {
3877 local: None,
3878 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3879 name: "backup-1".to_string(),
3880 }),
3881 };
3882 let restore = make_restore(source, None);
3883 let id1 = super::compute_safety_backup_id(&restore).unwrap();
3884 let id2 = super::compute_safety_backup_id(&restore).unwrap();
3885 assert_eq!(id1, id2);
3886 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3887 }
3888
3889 #[test]
3890 fn safety_manifest_key_is_deterministic() {
3891 let source = KanidmRestoreSource {
3892 local: None,
3893 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3894 name: "backup-1".to_string(),
3895 }),
3896 };
3897 let restore = make_restore(source, None);
3898 let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3899 let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3900 assert_eq!(key1, key2);
3901 assert!(key1.contains("manifest.json"));
3902 assert!(key1.starts_with("v1/tenants/"));
3903 assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3906 assert!(
3907 !key1.contains("a1b2c3d4"),
3908 "safety manifest key must not key the tenant segment by object UID: {key1}"
3909 );
3910 }
3911
3912 #[test]
3920 fn safety_manifest_key_matches_deployed_keyspace_layout() {
3921 let source = KanidmRestoreSource {
3922 local: None,
3923 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3924 name: "backup-1".to_string(),
3925 }),
3926 };
3927 let restore = make_restore(source, None);
3928 let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3929 assert_eq!(
3930 super::compute_safety_backup_id(&restore).unwrap(),
3931 expected_backup_id
3932 );
3933 assert_eq!(
3934 super::compute_safety_manifest_key(&restore).unwrap(),
3935 format!(
3936 "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
3937 )
3938 );
3939 }
3940
3941 #[test]
3942 fn safety_backup_id_survives_restart() {
3943 let source = KanidmRestoreSource {
3944 local: None,
3945 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3946 name: "backup-1".to_string(),
3947 }),
3948 };
3949 let mut restore = make_restore(source, None);
3950 let original_id = super::compute_safety_backup_id(&restore).unwrap();
3951 assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
3955 restore.status = Some(KanidmRestoreStatus {
3956 phase: KanidmRestorePhase::SafetyBackup,
3957 safety_backup_expected_backup_id: Some(original_id.clone()),
3958 ..Default::default()
3959 });
3960 let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3961 assert_eq!(original_id, recomputed_id);
3962 }
3963
3964 #[test]
3965 fn safety_backup_id_derived_from_uid_not_name() {
3966 let source = KanidmRestoreSource {
3967 local: None,
3968 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3969 name: "backup-1".to_string(),
3970 }),
3971 };
3972 let restore1 = make_restore(source.clone(), None);
3973 let mut restore2 = make_restore(source, None);
3974 restore2.metadata.name = Some("different-name-same-uid".to_string());
3975 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3976 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3977 assert_eq!(id1, id2);
3978 }
3979
3980 #[test]
3981 fn safety_backup_id_differs_for_different_uids() {
3982 let source = KanidmRestoreSource {
3983 local: None,
3984 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3985 name: "backup-1".to_string(),
3986 }),
3987 };
3988 let restore1 = make_restore(source.clone(), None);
3989 let mut restore2 = make_restore(source, None);
3990 restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3991 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3992 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3993 assert_ne!(id1, id2);
3994 }
3995
3996 #[test]
3997 fn safety_backup_id_errors_without_uid() {
3998 let source = KanidmRestoreSource {
3999 local: None,
4000 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4001 name: "backup-1".to_string(),
4002 }),
4003 };
4004 let mut restore = make_restore(source, None);
4005 restore.metadata.uid = None;
4006 assert!(super::compute_safety_backup_id(&restore).is_err());
4007 }
4008
4009 #[test]
4010 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4011 let source = KanidmRestoreSource {
4012 local: None,
4013 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4014 name: "backup-1".to_string(),
4015 }),
4016 };
4017 let restore = make_restore(source, None);
4018 assert!(super::requires_safety_backup(&restore));
4019 let status_without_ref = KanidmRestoreStatus {
4020 phase: KanidmRestorePhase::RestoringPrimary,
4021 safety_backup_ref: None,
4022 database_mutation_started: false,
4023 ..Default::default()
4024 };
4025 assert!(status_without_ref.safety_backup_ref.is_none());
4026 assert!(!status_without_ref.database_mutation_started);
4027 }
4028
4029 #[test]
4030 fn mutation_boundary_allows_progression_when_safety_ref_set() {
4031 let status_with_ref = KanidmRestoreStatus {
4032 phase: KanidmRestorePhase::RestoringPrimary,
4033 safety_backup_ref: Some("safety-test-restore".to_string()),
4034 safety_backup_manifest_key: Some(
4035 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4036 ),
4037 safety_backup_payload_sha256: Some("abc123".to_string()),
4038 database_mutation_started: false,
4039 ..Default::default()
4040 };
4041 assert!(status_with_ref.safety_backup_ref.is_some());
4042 assert!(!status_with_ref.database_mutation_started);
4043 }
4044
4045 #[test]
4046 fn result_document_verification_rejects_missing_backup_id() {
4047 let result_json = r#"{
4048 "apiVersion": "backup.kaniop.rs/v1alpha1",
4049 "kind": "ResultDocument",
4050 "operation": "upload",
4051 "success": true,
4052 "exitCode": "success",
4053 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4054 "payloadSha256": "abc123"
4055 }"#;
4056 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4057 assert!(result.backup_id.is_none());
4058 }
4059
4060 #[test]
4061 fn result_document_verification_rejects_missing_manifest_key() {
4062 let result_json = r#"{
4063 "apiVersion": "backup.kaniop.rs/v1alpha1",
4064 "kind": "ResultDocument",
4065 "operation": "upload",
4066 "success": true,
4067 "exitCode": "success",
4068 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4069 "payloadSha256": "abc123"
4070 }"#;
4071 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4072 assert!(result.manifest_key.is_none());
4073 }
4074
4075 #[test]
4076 fn result_document_verification_rejects_missing_payload_sha256() {
4077 let result_json = r#"{
4078 "apiVersion": "backup.kaniop.rs/v1alpha1",
4079 "kind": "ResultDocument",
4080 "operation": "upload",
4081 "success": true,
4082 "exitCode": "success",
4083 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4084 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4085 }"#;
4086 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4087 assert!(result.payload_sha256.is_none());
4088 }
4089
4090 #[test]
4091 fn result_document_verification_rejects_failure_exit_code() {
4092 let result_json = r#"{
4093 "apiVersion": "backup.kaniop.rs/v1alpha1",
4094 "kind": "ResultDocument",
4095 "operation": "upload",
4096 "success": false,
4097 "exitCode": "integrity",
4098 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4099 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4100 "payloadSha256": "abc123"
4101 }"#;
4102 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4103 assert!(!result.success);
4104 assert_eq!(
4105 result.exit_code,
4106 kaniop_backup_core::result::ExitCode::Integrity
4107 );
4108 }
4109
4110 #[test]
4111 fn result_document_verification_rejects_wrong_operation() {
4112 let result_json = r#"{
4113 "apiVersion": "backup.kaniop.rs/v1alpha1",
4114 "kind": "ResultDocument",
4115 "operation": "download",
4116 "success": true,
4117 "exitCode": "success",
4118 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4119 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4120 "payloadSha256": "abc123"
4121 }"#;
4122 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4123 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4124 }
4125
4126 #[test]
4127 fn result_document_verification_accepts_valid_document() {
4128 let result_json = r#"{
4129 "apiVersion": "backup.kaniop.rs/v1alpha1",
4130 "kind": "ResultDocument",
4131 "operation": "upload",
4132 "success": true,
4133 "exitCode": "success",
4134 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4135 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4136 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4137 }"#;
4138 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4139 assert!(result.success);
4140 assert_eq!(
4141 result.exit_code,
4142 kaniop_backup_core::result::ExitCode::Success
4143 );
4144 assert_eq!(result.operation, "upload");
4145 assert!(result.backup_id.is_some());
4146 assert!(result.manifest_key.is_some());
4147 assert!(result.payload_sha256.is_some());
4148 }
4149
4150 #[test]
4151 fn termination_message_path_constant_is_valid() {
4152 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4153 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4154 }
4155
4156 #[test]
4157 fn status_tracks_safety_backup_evidence() {
4158 let status = KanidmRestoreStatus {
4159 phase: KanidmRestorePhase::PreparingSource,
4160 safety_backup_ref: Some("safety-test-restore".to_string()),
4161 safety_backup_expected_backup_id: Some(
4162 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4163 ),
4164 safety_backup_manifest_key: Some(
4165 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4166 ),
4167 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4168 ..Default::default()
4169 };
4170 assert!(status.safety_backup_ref.is_some());
4171 assert!(status.safety_backup_expected_backup_id.is_some());
4172 assert!(status.safety_backup_manifest_key.is_some());
4173 assert!(status.safety_backup_payload_sha256.is_some());
4174 }
4175
4176 #[test]
4177 fn fail_closed_semantics_preserved_on_result_parse_failure() {
4178 let invalid_json = "not valid json";
4179 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4180 assert!(result.is_err());
4181 }
4182
4183 #[test]
4184 fn fail_closed_semantics_preserved_on_oversized_result() {
4185 let oversized_json = format!(
4186 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4187 kaniop_backup_core::result::RESULT_DOC_VERSION,
4188 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4189 );
4190 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4191 assert!(result.is_err());
4192 }
4193
4194 #[test]
4195 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4196 let source = KanidmRestoreSource {
4197 local: None,
4198 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4199 name: "backup-1".to_string(),
4200 }),
4201 };
4202 let mut restore = make_restore(source, None);
4203 restore.metadata.namespace = Some("test-ns".to_string());
4204 let target = super::super::crd::Kanidm::default();
4205 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4206 let doc_str = super::build_safety_upload_operation_doc(
4207 &restore,
4208 &target,
4209 "test-backup-id",
4210 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4211 "my-bucket",
4212 "prod",
4213 "https://s3.example.com",
4214 "us-east-1",
4215 false,
4216 false,
4217 Some(&ca_path),
4218 None,
4219 )
4220 .unwrap();
4221 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4222 assert_eq!(parsed["caBundlePath"], ca_path);
4223 }
4224
4225 #[test]
4226 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4227 let source = KanidmRestoreSource {
4228 local: None,
4229 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4230 name: "backup-1".to_string(),
4231 }),
4232 };
4233 let mut restore = make_restore(source, None);
4234 restore.metadata.namespace = Some("test-ns".to_string());
4235 let target = super::super::crd::Kanidm::default();
4236 let doc_str = super::build_safety_upload_operation_doc(
4237 &restore,
4238 &target,
4239 "test-backup-id",
4240 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4241 "my-bucket",
4242 "prod",
4243 "https://s3.example.com",
4244 "us-east-1",
4245 false,
4246 false,
4247 None,
4248 None,
4249 )
4250 .unwrap();
4251 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4252 assert!(parsed["caBundlePath"].is_null());
4253 }
4254
4255 #[test]
4256 fn download_operation_doc_includes_ca_bundle_path_when_set() {
4257 let source = KanidmRestoreSource {
4258 local: None,
4259 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4260 name: "backup-1".to_string(),
4261 }),
4262 };
4263 let _restore = make_restore(source, None);
4264 let target = super::super::crd::Kanidm::default();
4265 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4266 let doc_str = super::build_download_operation_doc(
4267 &target,
4268 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4269 "019c7c76-f423-7a12-8f41-2bea7588a303",
4270 "source-uid-456",
4271 "real-bucket",
4272 "real-prefix",
4273 "https://real-endpoint.com",
4274 "eu-west-1",
4275 true,
4276 false,
4277 Some(&ca_path),
4278 None,
4279 );
4280 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4281 assert_eq!(parsed["caBundlePath"], ca_path);
4282 }
4283
4284 #[test]
4285 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4286 let source = KanidmRestoreSource {
4287 local: None,
4288 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4289 name: "backup-1".to_string(),
4290 }),
4291 };
4292 let _restore = make_restore(source, None);
4293 let target = super::super::crd::Kanidm::default();
4294 let doc_str = super::build_download_operation_doc(
4295 &target,
4296 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4297 "019c7c76-f423-7a12-8f41-2bea7588a303",
4298 "source-uid-456",
4299 "real-bucket",
4300 "real-prefix",
4301 "https://real-endpoint.com",
4302 "eu-west-1",
4303 true,
4304 false,
4305 None,
4306 None,
4307 );
4308 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4309 assert!(parsed["caBundlePath"].is_null());
4310 }
4311
4312 #[test]
4313 fn source_prep_result_operation_is_download() {
4314 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4315 assert_ne!(
4316 super::SOURCE_PREP_RESULT_OPERATION,
4317 super::SAFETY_BACKUP_RESULT_OPERATION
4318 );
4319 }
4320
4321 #[test]
4322 fn download_result_document_verification_accepts_valid_document() {
4323 let result_json = r#"{
4324 "apiVersion": "backup.kaniop.rs/v1alpha1",
4325 "kind": "ResultDocument",
4326 "operation": "download",
4327 "success": true,
4328 "exitCode": "success",
4329 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4330 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4331 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4332 }"#;
4333 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4334 assert!(result.success);
4335 assert_eq!(
4336 result.exit_code,
4337 kaniop_backup_core::result::ExitCode::Success
4338 );
4339 assert_eq!(result.operation, "download");
4340 assert_eq!(
4341 result.backup_id.as_deref(),
4342 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4343 );
4344 assert!(result.manifest_key.is_some());
4345 assert!(result.payload_sha256.is_some());
4346 }
4347
4348 #[test]
4349 fn download_result_document_rejects_wrong_operation() {
4350 let result_json = r#"{
4351 "apiVersion": "backup.kaniop.rs/v1alpha1",
4352 "kind": "ResultDocument",
4353 "operation": "upload",
4354 "success": true,
4355 "exitCode": "success",
4356 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4357 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4358 "payloadSha256": "abc123"
4359 }"#;
4360 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4361 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4362 }
4363
4364 #[test]
4365 fn download_result_document_rejects_failure_success_field() {
4366 let result_json = r#"{
4367 "apiVersion": "backup.kaniop.rs/v1alpha1",
4368 "kind": "ResultDocument",
4369 "operation": "download",
4370 "success": false,
4371 "exitCode": "retryable",
4372 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4373 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4374 "payloadSha256": "abc123"
4375 }"#;
4376 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4377 assert!(!result.success);
4378 assert_ne!(
4379 result.exit_code,
4380 kaniop_backup_core::result::ExitCode::Success
4381 );
4382 }
4383
4384 #[test]
4385 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4386 let status = KanidmRestoreStatus {
4387 phase: KanidmRestorePhase::PreparingSource,
4388 database_mutation_started: false,
4389 ..Default::default()
4390 };
4391 assert!(!status.database_mutation_started);
4392 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4393 }
4394
4395 #[test]
4396 fn backup_job_volume_size_default_is_10gi() {
4397 let qty = crate::controller::backup_job_volume_size();
4398 assert_eq!(qty, Quantity("10Gi".to_string()));
4399 }
4400
4401 #[test]
4402 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4403 use kaniop_backup_core::auth::build_encryption_env_vars;
4404 use kaniop_backup_core::crd::SecretRef;
4405
4406 let key_ref = SecretRef {
4407 name: "kek-secret".to_string(),
4408 };
4409 let env_vars = build_encryption_env_vars(Some(&key_ref));
4410 assert_eq!(env_vars.len(), 1);
4411 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4412 let secret_ref = env_vars[0]
4413 .value_from
4414 .as_ref()
4415 .unwrap()
4416 .secret_key_ref
4417 .as_ref()
4418 .unwrap();
4419 assert_eq!(secret_ref.name, "kek-secret");
4420 assert_eq!(secret_ref.key, "encryption-key");
4421 }
4422
4423 #[test]
4424 fn encryption_env_vars_absent_when_no_key_ref() {
4425 use kaniop_backup_core::auth::build_encryption_env_vars;
4426
4427 let env_vars = build_encryption_env_vars(None);
4428 assert!(
4429 env_vars.is_empty(),
4430 "encryption env vars must be empty when key_ref is None"
4431 );
4432 }
4433
4434 #[test]
4435 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4436 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4437
4438 let source = KanidmRestoreSource {
4439 local: None,
4440 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4441 name: "backup-1".to_string(),
4442 }),
4443 };
4444 let mut restore = make_restore(source, None);
4445 restore.metadata.namespace = Some("test-ns".to_string());
4446 let target = super::super::crd::Kanidm::default();
4447 let encryption = RepositoryEncryption {
4448 mode: EncryptionMode::ClientSide,
4449 key_id: None,
4450 key_ref: Some(SecretRef {
4451 name: "kek-secret".to_string(),
4452 }),
4453 };
4454 let doc_str = super::build_safety_upload_operation_doc(
4455 &restore,
4456 &target,
4457 "test-backup-id",
4458 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4459 "my-bucket",
4460 "prod",
4461 "https://s3.example.com",
4462 "us-east-1",
4463 false,
4464 false,
4465 None,
4466 Some(&encryption),
4467 )
4468 .unwrap();
4469 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4470 assert_eq!(parsed["encryptionMode"], "clientSide");
4471 assert!(parsed["encryptionKeyId"].is_null());
4472 }
4473
4474 #[test]
4475 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4476 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4477
4478 let source = KanidmRestoreSource {
4479 local: None,
4480 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4481 name: "backup-1".to_string(),
4482 }),
4483 };
4484 let _restore = make_restore(source, None);
4485 let target = super::super::crd::Kanidm::default();
4486 let encryption = RepositoryEncryption {
4487 mode: EncryptionMode::ProviderKms,
4488 key_id: Some("alias/kaniop-backups".to_string()),
4489 key_ref: None,
4490 };
4491 let doc_str = super::build_download_operation_doc(
4492 &target,
4493 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4494 "019c7c76-f423-7a12-8f41-2bea7588a303",
4495 "source-uid-456",
4496 "real-bucket",
4497 "real-prefix",
4498 "https://real-endpoint.com",
4499 "eu-west-1",
4500 true,
4501 false,
4502 None,
4503 Some(&encryption),
4504 );
4505 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4506 assert_eq!(parsed["encryptionMode"], "providerKms");
4507 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4508 }
4509
4510 #[test]
4511 fn post_mutation_failure_records_job_names() {
4512 let existing = KanidmRestoreStatus {
4513 phase: KanidmRestorePhase::RestoringPrimary,
4514 database_mutation_started: true,
4515 ..Default::default()
4516 };
4517
4518 let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4519 assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4520 assert_eq!(
4521 restore_failed.restore_job_name.as_deref(),
4522 Some("my-restore-job")
4523 );
4524 assert!(restore_failed.verify_job_name.is_none());
4525 assert!(restore_failed.database_mutation_started);
4526 assert_eq!(
4527 restore_failed.message.as_deref(),
4528 Some("database restore job failed")
4529 );
4530
4531 let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4532 assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4533 assert_eq!(
4534 verify_failed.verify_job_name.as_deref(),
4535 Some("my-verify-job")
4536 );
4537 assert!(verify_failed.restore_job_name.is_none());
4538 assert!(verify_failed.database_mutation_started);
4539 assert_eq!(
4540 verify_failed.message.as_deref(),
4541 Some("database verification failed")
4542 );
4543 }
4544
4545 #[test]
4546 fn restore_and_verify_job_names_are_deterministic() {
4547 let source = KanidmRestoreSource {
4548 local: Some(super::KanidmRestoreLocalSource {
4549 file_name: "backup.json".to_string(),
4550 }),
4551 backup_ref: None,
4552 };
4553 let restore = make_restore(source, None);
4554 assert_eq!(restore_job_name(&restore), "test-restore-restore");
4555 assert_eq!(verify_job_name(&restore), "test-restore-verify");
4556 }
4557}