1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{
12 KanidmBackup, KanidmBackupPhase, KanidmBackupRepository, KanidmBackupRepositoryStatus,
13};
14use kaniop_backup_core::image::data_mover_image;
15use kaniop_backup_core::operation::{
16 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
17};
18use kaniop_backup_core::result::{ExitCode, parse_result_document};
19use kaniop_k8s_util::error::{Error, Result};
20
21use std::collections::BTreeSet;
22use std::sync::Arc;
23use std::time::Duration;
24
25use futures::StreamExt;
26use k8s_openapi::api::apps::v1::StatefulSet;
27use k8s_openapi::api::batch::v1::{Job, JobSpec};
28use k8s_openapi::api::core::v1::{
29 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
30 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
31 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
32};
33use k8s_openapi::api::storage::v1::VolumeAttachment;
34use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
35use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
36use k8s_openapi::jiff::Timestamp;
37use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
38use kube::runtime::controller::{Action, Controller};
39use kube::runtime::events::{Event, EventType, Recorder, Reporter};
40use kube::runtime::finalizer::{Event as Finalizer, finalizer};
41use kube::runtime::watcher;
42use kube::{Api, Client, CustomResource, Resource, ResourceExt};
43use opentelemetry::metrics::{Counter, Gauge, Histogram};
44use opentelemetry::{KeyValue, global};
45#[cfg(feature = "schemars")]
46use schemars::JsonSchema;
47use serde::{Deserialize, Serialize};
48use serde_json::json;
49use tracing::{debug, error, info, warn};
50
51pub const CONTROLLER_ID: &str = "kanidm-restore";
52pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
53const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
54const DATA_VOLUME: &str = "kanidm-data";
55const SHARED_VOLUME: &str = "safety-backup-shared";
56const DATA_PATH: &str = "/data";
57const TLS_VOLUME: &str = "kanidm-certs";
58const TLS_PATH: &str = "/etc/kanidm/tls";
59const BACKUP_PATH: &str = "/data";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70pub const DISASTER_RECOVERY_ANNOTATION: &str = "backup.kaniop.rs/disaster-recovery";
71const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
72const CONDITION_DISASTER_RECOVERY: &str = "DisasterRecoveryOverride";
73const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
74const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
75const SOURCE_PREP_RESULT_OPERATION: &str = "download";
76
77#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
78#[cfg_attr(feature = "schemars", derive(JsonSchema))]
79#[cfg_attr(
80 feature = "schemars",
81 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
82)]
83#[kube(
84 category = "kaniop",
85 group = "kaniop.rs",
86 version = "v1beta1",
87 kind = "KanidmRestore",
88 plural = "kanidmrestores",
89 singular = "kanidmrestore",
90 shortname = "idmrestore",
91 namespaced,
92 status = "KanidmRestoreStatus",
93 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
94 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
95 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
96)]
97#[serde(rename_all = "camelCase")]
98pub struct KanidmRestoreSpec {
99 pub target_ref: KanidmRestoreTargetRef,
100 pub source: KanidmRestoreSource,
101 pub restore_image: String,
102 #[serde(skip_serializing_if = "Option::is_none")]
103 pub safety_backup: Option<SafetyBackupConfig>,
104}
105
106#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
107#[cfg_attr(feature = "schemars", derive(JsonSchema))]
108#[serde(rename_all = "camelCase")]
109pub struct KanidmRestoreTargetRef {
110 pub name: String,
111 pub uid: String,
112}
113
114#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
115#[cfg_attr(feature = "schemars", derive(JsonSchema))]
116#[cfg_attr(
117 feature = "schemars",
118 schemars(extend("x-kubernetes-validations" = [
119 {
120 "message": "exactly one of local or backupRef must be set",
121 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
122 }
123 ]))
124)]
125#[serde(rename_all = "camelCase")]
126pub struct KanidmRestoreSource {
127 #[serde(skip_serializing_if = "Option::is_none")]
129 pub local: Option<KanidmRestoreLocalSource>,
130 #[serde(skip_serializing_if = "Option::is_none")]
132 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
133}
134
135#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
136#[cfg_attr(feature = "schemars", derive(JsonSchema))]
137#[serde(rename_all = "camelCase")]
138pub struct KanidmRestoreLocalSource {
139 pub file_name: String,
141}
142
143#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
144#[cfg_attr(feature = "schemars", derive(JsonSchema))]
145#[serde(rename_all = "camelCase")]
146pub struct KanidmRestoreBackupRefSource {
147 pub name: String,
149}
150
151#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
152#[cfg_attr(feature = "schemars", derive(JsonSchema))]
153#[serde(rename_all = "camelCase")]
154pub struct SafetyBackupConfig {
155 #[serde(skip_serializing_if = "Option::is_none")]
157 pub repository_ref: Option<SafetyBackupRepositoryRef>,
158 #[serde(default)]
160 pub skip: bool,
161}
162
163#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
164#[cfg_attr(feature = "schemars", derive(JsonSchema))]
165#[serde(rename_all = "camelCase")]
166pub struct SafetyBackupRepositoryRef {
167 pub name: String,
168}
169
170#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
171#[cfg_attr(feature = "schemars", derive(JsonSchema))]
172pub enum KanidmRestorePhase {
173 #[default]
174 Pending,
175 Validating,
176 Quiescing,
177 SafetyBackup,
178 PreparingSource,
179 RestoringPrimary,
180 Verifying,
181 RebuildingReplicas,
182 Resuming,
183 Completed,
184 Failed,
185}
186
187#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
188#[cfg_attr(feature = "schemars", derive(JsonSchema))]
189#[serde(rename_all = "camelCase")]
190pub struct KanidmRestoreStatus {
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_generation: Option<i64>,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 pub observed_target_uid: Option<String>,
195 #[serde(default)]
196 pub phase: KanidmRestorePhase,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub restore_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub verify_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub safety_backup_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub source_prep_job_name: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_ref: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_expected_backup_id: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_manifest_key: Option<String>,
211 #[serde(skip_serializing_if = "Option::is_none")]
212 pub safety_backup_payload_sha256: Option<String>,
213 #[serde(default)]
214 pub replicas_cleared: bool,
215 #[serde(default)]
216 pub certificates_cleared: bool,
217 #[serde(default)]
218 pub database_mutation_started: bool,
219 #[serde(default, skip_serializing_if = "Vec::is_empty")]
220 pub original_replicas: Vec<ReplicaCountEntry>,
221 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
222 pub phase_timestamps: std::collections::BTreeMap<String, String>,
223 #[serde(default, skip_serializing_if = "Vec::is_empty")]
224 #[cfg_attr(
225 feature = "schemars",
226 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
227 )]
228 pub conditions: Vec<Condition>,
229 #[serde(skip_serializing_if = "Option::is_none")]
230 pub message: Option<String>,
231}
232
233#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
234#[cfg_attr(feature = "schemars", derive(JsonSchema))]
235#[serde(rename_all = "camelCase")]
236pub struct ReplicaCountEntry {
237 pub group: String,
238 pub replicas: i32,
239}
240
241#[derive(Clone)]
242struct RestoreMetrics {
243 attempts: Counter<u64>,
244 outcomes: Counter<u64>,
245 duration_seconds: Histogram<f64>,
246 start_timestamp_seconds: Gauge<i64>,
247 break_glass_total: Counter<u64>,
248 disaster_recovery_total: Counter<u64>,
249 safety_backup_duration_seconds: Histogram<f64>,
250}
251
252impl RestoreMetrics {
253 fn new() -> Self {
254 let meter = global::meter("kaniop");
255 Self {
256 attempts: meter
257 .u64_counter("restore_attempts")
258 .with_description("Number of Kanidm restore attempts started")
259 .build(),
260 outcomes: meter
261 .u64_counter("restore_outcomes")
262 .with_description("Number of terminal Kanidm restore outcomes")
263 .build(),
264 duration_seconds: meter
265 .f64_histogram("restore_duration_seconds")
266 .with_description("Kanidm restore duration from object creation to terminal phase")
267 .with_unit("s")
268 .build(),
269 start_timestamp_seconds: meter
270 .i64_gauge("restore_start_timestamp_seconds")
271 .with_description("Unix timestamp when an active Kanidm restore was created; zero for terminal restores")
272 .with_unit("s")
273 .build(),
274 break_glass_total: meter
275 .u64_counter("restore_break_glass")
276 .with_description("Number of break-glass safety backup overrides")
277 .build(),
278 disaster_recovery_total: meter
279 .u64_counter("restore_disaster_recovery")
280 .with_description("Number of approved disaster-recovery source identity overrides")
281 .build(),
282 safety_backup_duration_seconds: meter
283 .f64_histogram("restore_safety_backup_duration_seconds")
284 .with_description("Duration of pre-restore safety backup creation")
285 .with_unit("s")
286 .build(),
287 }
288 }
289}
290
291fn record_restore_start_timestamp(restore: &KanidmRestore, metrics: &RestoreMetrics) {
292 let phase = restore.status.as_ref().map(|status| status.phase).unwrap_or_default();
293 let active = restore.metadata.deletion_timestamp.is_none()
294 && !matches!(phase, KanidmRestorePhase::Completed | KanidmRestorePhase::Failed);
295 let timestamp = if active {
296 restore
297 .metadata
298 .creation_timestamp
299 .as_ref()
300 .map(|created| created.0.as_second())
301 .unwrap_or_else(|| Timestamp::now().as_second())
302 } else {
303 0
304 };
305 let attributes = [
306 KeyValue::new("namespace", restore.namespace().unwrap_or_default()),
307 KeyValue::new("restore", restore.name_any()),
308 ];
309 metrics.start_timestamp_seconds.record(timestamp, &attributes);
310}
311
312#[derive(Clone)]
313struct RestoreContext {
314 client: Client,
315 recorder: Recorder,
316 metrics: RestoreMetrics,
317}
318
319async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
320 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
321 match phase {
322 KanidmRestorePhase::Pending => {
323 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
324 Ok(Action::requeue(REQUEUE))
325 }
326 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
327 Ok((target, disaster_recovery_source)) => {
328 if requires_safety_backup(&restore)
331 && !safety_repository_usage_locked(&restore, &ctx).await?
332 {
333 return Ok(Action::requeue(REQUEUE));
334 }
335 mark_restoring(&restore, &target, &ctx).await?;
336 if let Some(source) = disaster_recovery_source.as_ref() {
337 record_disaster_recovery_override(&restore, &target, &ctx, source).await;
338 }
339 let mut status = restore.status.clone().unwrap_or_default();
340 if let Some(source) = disaster_recovery_source.as_ref() {
341 let target_uid = target.uid().unwrap_or_default();
342 let condition = restore_condition(
343 &status.conditions,
344 CONDITION_DISASTER_RECOVERY,
345 CONDITION_TRUE,
346 "ApprovedSourceIdentityMismatch",
347 &format!(
348 "Approved disaster recovery from source {}/{} into target {}/{}",
349 source.name,
350 source.uid,
351 target.name_any(),
352 target_uid
353 ),
354 restore.metadata.generation,
355 );
356 status
357 .conditions
358 .retain(|condition| condition.type_ != CONDITION_DISASTER_RECOVERY);
359 status.conditions.push(condition);
360 }
361 status.observed_target_uid = target.uid();
362 status.original_replicas = capture_original_replicas(&target);
363 status.phase = KanidmRestorePhase::Quiescing;
364 status.message = None;
365 patch_status(&restore, &ctx, status).await?;
366 Ok(Action::requeue(REQUEUE))
367 }
368 Err(error) => {
369 set_phase(
370 &restore,
371 &ctx,
372 KanidmRestorePhase::Failed,
373 Some(error.to_string()),
374 )
375 .await?;
376 Ok(Action::requeue(Duration::from_secs(300)))
377 }
378 },
379 KanidmRestorePhase::Quiescing => {
380 let target = get_target(&restore, &ctx).await?;
381 scale_all(&target, &ctx, 0).await?;
382 if target_pods_stopped(&target, &ctx).await?
383 && target_volumes_detached(&target, &ctx).await?
384 {
385 let next_phase = if requires_safety_backup(&restore) {
386 KanidmRestorePhase::SafetyBackup
387 } else {
388 record_break_glass(&restore, &ctx).await?;
389 KanidmRestorePhase::PreparingSource
390 };
391 if next_phase == KanidmRestorePhase::PreparingSource {
392 let refreshed = Api::<KanidmRestore>::namespaced(
393 ctx.client.clone(),
394 restore.namespace().as_deref().unwrap_or_default(),
395 )
396 .get(&restore.name_any())
397 .await
398 .map_err(|error| {
399 Error::kube_error(
400 "get",
401 "KanidmRestore",
402 restore.namespace().as_deref().unwrap_or_default(),
403 restore.name_any(),
404 error,
405 )
406 })?;
407 set_phase(&refreshed, &ctx, next_phase, None).await?;
408 } else {
409 set_phase(&restore, &ctx, next_phase, None).await?;
410 }
411 }
412 Ok(Action::requeue(REQUEUE))
413 }
414 KanidmRestorePhase::SafetyBackup => {
415 if !safety_repository_usage_locked(&restore, &ctx).await? {
418 return Ok(Action::requeue(REQUEUE));
419 }
420 let target = get_target(&restore, &ctx).await?;
421 let name = safety_backup_job_name(&restore);
422 let expected_backup_id = compute_safety_backup_id(&restore)?;
423 let expected_manifest_key = compute_safety_manifest_key(&restore)?;
424 ensure_safety_backup_job(
425 &restore,
426 &target,
427 &ctx,
428 &name,
429 &expected_backup_id,
430 &expected_manifest_key,
431 )
432 .await?;
433 let mut status = restore.status.clone().unwrap_or_default();
434 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
435 match job_state(&restore, &ctx, &name).await? {
436 JobState::Complete => {
437 let result =
438 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
439 match result {
440 Ok(verified) => {
441 status.safety_backup_job_name = Some(name);
442 status.safety_backup_ref =
443 Some(format!("safety-{}", restore.name_any()));
444 status.safety_backup_manifest_key = Some(verified.manifest_key);
445 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
446 record_safety_backup_duration(&restore, &ctx).await;
447 status.phase = KanidmRestorePhase::PreparingSource;
448 status.message = None;
449 patch_status(&restore, &ctx, status).await?;
450 }
451 Err(error) => {
452 resume_before_mutation(&restore, &ctx).await?;
453 set_phase(
454 &restore,
455 &ctx,
456 KanidmRestorePhase::Failed,
457 Some(format!("safety backup result verification failed: {error}")),
458 )
459 .await?;
460 }
461 }
462 }
463 JobState::Failed => {
464 resume_before_mutation(&restore, &ctx).await?;
465 set_phase(
466 &restore,
467 &ctx,
468 KanidmRestorePhase::Failed,
469 Some(
470 "safety backup job failed; target restored to original state"
471 .to_string(),
472 ),
473 )
474 .await?;
475 }
476 JobState::Running => {
477 patch_status(&restore, &ctx, status).await?;
478 }
479 }
480 Ok(Action::requeue(REQUEUE))
481 }
482 KanidmRestorePhase::PreparingSource => {
483 if is_remote_source(&restore) {
484 let target = get_target(&restore, &ctx).await?;
485 let name = source_prep_job_name(&restore);
486 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
487 match job_state(&restore, &ctx, &name).await? {
488 JobState::Complete => {
489 let backup_id = restore
490 .spec
491 .source
492 .backup_ref
493 .as_ref()
494 .map(|b| b.name.clone())
495 .unwrap_or_default();
496 let backup = {
497 let ns = restore.namespace().unwrap();
498 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
499 .get(&backup_id)
500 .await
501 .map_err(|e| {
502 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
503 })?
504 };
505 let expected_backup_id = &backup.spec.backup_id;
506 let result =
507 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
508 .await;
509 match result {
510 Ok(verified) => {
511 if let Some(expected_sha256) = backup
512 .status
513 .as_ref()
514 .and_then(|s| s.payload_sha256.as_deref())
515 {
516 if verified.payload_sha256 != expected_sha256 {
517 resume_before_mutation(&restore, &ctx).await?;
518 set_phase(
519 &restore,
520 &ctx,
521 KanidmRestorePhase::Failed,
522 Some(format!(
523 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
524 verified.payload_sha256
525 )),
526 )
527 .await?;
528 return Ok(Action::requeue(REQUEUE));
529 }
530 }
531 let mut status = restore.status.clone().unwrap_or_default();
532 status.source_prep_job_name = Some(name);
533 status.phase = KanidmRestorePhase::RestoringPrimary;
534 status.message = None;
535 patch_status(&restore, &ctx, status).await?;
536 }
537 Err(error) => {
538 resume_before_mutation(&restore, &ctx).await?;
539 set_phase(
540 &restore,
541 &ctx,
542 KanidmRestorePhase::Failed,
543 Some(format!(
544 "source preparation result verification failed: {error}"
545 )),
546 )
547 .await?;
548 }
549 }
550 }
551 JobState::Failed => {
552 resume_before_mutation(&restore, &ctx).await?;
553 set_phase(
554 &restore,
555 &ctx,
556 KanidmRestorePhase::Failed,
557 Some(
558 "source preparation job failed; target restored to original state"
559 .to_string(),
560 ),
561 )
562 .await?;
563 }
564 JobState::Running => {}
565 }
566 } else {
567 let target = get_target(&restore, &ctx).await?;
568 let name = source_check_job_name(&restore);
569 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
570 match job_state(&restore, &ctx, &name).await? {
571 JobState::Complete => {
572 let mut status = restore.status.clone().unwrap_or_default();
573 status.source_prep_job_name = Some(name);
574 status.phase = KanidmRestorePhase::RestoringPrimary;
575 status.message = None;
576 patch_status(&restore, &ctx, status).await?;
577 }
578 JobState::Failed => {
579 resume_before_mutation(&restore, &ctx).await?;
580 set_phase(
581 &restore,
582 &ctx,
583 KanidmRestorePhase::Failed,
584 Some(
585 "local backup file check failed; target restored to original state"
586 .to_string(),
587 ),
588 )
589 .await?;
590 }
591 JobState::Running => {}
592 }
593 }
594 Ok(Action::requeue(REQUEUE))
595 }
596 KanidmRestorePhase::RestoringPrimary => {
597 let target = get_target(&restore, &ctx).await?;
598 let status = restore.status.as_ref().cloned().unwrap_or_default();
599 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
600 return Err(Error::MissingData(
601 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
602 ));
603 }
604 if !status.database_mutation_started {
605 let mut status = status;
606 status.database_mutation_started = true;
607 status.message = Some(
608 "database mutation boundary persisted; starting primary restore".to_string(),
609 );
610 patch_status(&restore, &ctx, status).await?;
611 return Ok(Action::requeue(REQUEUE));
612 }
613 let name = restore_job_name(&restore);
614 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
615 match job_state(&restore, &ctx, &name).await? {
616 JobState::Complete => {
617 let mut status = restore.status.clone().unwrap_or_default();
618 status.restore_job_name = Some(name);
619 status.phase = KanidmRestorePhase::Verifying;
620 status.message = None;
621 patch_status(&restore, &ctx, status).await?;
622 }
623 JobState::Failed => {
624 let status = restore.status.clone().unwrap_or_default();
625 let failed_status = restore_primary_failed_status(&status, &name);
626 patch_status(&restore, &ctx, failed_status).await?;
627 }
628 JobState::Running => {}
629 }
630 Ok(Action::requeue(REQUEUE))
631 }
632 KanidmRestorePhase::Verifying => {
633 let target = get_target(&restore, &ctx).await?;
634 let name = verify_job_name(&restore);
635 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
636 match job_state(&restore, &ctx, &name).await? {
637 JobState::Complete => {
638 let mut status = restore.status.clone().unwrap_or_default();
639 status.verify_job_name = Some(name);
640 status.phase = KanidmRestorePhase::RebuildingReplicas;
641 status.message = None;
642 patch_status(&restore, &ctx, status).await?;
643 }
644 JobState::Failed => {
645 let status = restore.status.clone().unwrap_or_default();
646 let failed_status = verifying_failed_status(&status, &name);
647 patch_status(&restore, &ctx, failed_status).await?;
648 }
649 JobState::Running => {}
650 }
651 Ok(Action::requeue(REQUEUE))
652 }
653 KanidmRestorePhase::RebuildingReplicas => {
654 let target = get_target(&restore, &ctx).await?;
655 let mut status = restore.status.clone().unwrap_or_default();
656 if !status.replicas_cleared {
657 if delete_secondary_pvcs(&target, &ctx).await? {
658 status.replicas_cleared = true;
659 status.message = Some("secondary database state cleared".to_string());
660 patch_status(&restore, &ctx, status).await?;
661 }
662 return Ok(Action::requeue(REQUEUE));
663 }
664
665 scale_primary(&target, &ctx, 1).await?;
666 if !primary_ready(&target, &ctx).await? {
667 return Ok(Action::requeue(REQUEUE));
668 }
669 if !status.certificates_cleared {
670 delete_replica_cert_secrets(&target, &ctx).await?;
671 delete_admin_secret(&target, &ctx).await?;
672 status.certificates_cleared = true;
673 status.message = Some(
674 "replica certificates and admin secret cleared for regeneration".to_string(),
675 );
676 patch_status(&restore, &ctx, status).await?;
677 return Ok(Action::requeue(REQUEUE));
678 }
679 status.phase = KanidmRestorePhase::Resuming;
680 status.message = None;
681 patch_status(&restore, &ctx, status).await?;
682 Ok(Action::requeue(REQUEUE))
683 }
684 KanidmRestorePhase::Resuming => {
685 let target = get_target(&restore, &ctx).await?;
686 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
687 clear_restoring(&restore, &target, &ctx).await?;
688 Ok(Action::requeue(Duration::from_secs(3600)))
689 }
690 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
691 Ok(Action::requeue(Duration::from_secs(3600)))
692 }
693 }
694}
695
696#[derive(Clone, Debug, PartialEq, Eq)]
697struct DisasterRecoverySource {
698 name: String,
699 uid: String,
700}
701
702async fn validate(
703 restore: &KanidmRestore,
704 ctx: &RestoreContext,
705) -> Result<(Kanidm, Option<DisasterRecoverySource>)> {
706 let target = get_target(restore, ctx).await?;
707 let actual_uid = target
708 .uid()
709 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
710 if actual_uid != restore.spec.target_ref.uid {
711 return Err(Error::MissingData(format!(
712 "target UID mismatch: expected {}, got {}",
713 restore.spec.target_ref.uid, actual_uid
714 )));
715 }
716 validate_source(restore)?;
717 let disaster_recovery_source = if is_remote_source(restore) {
718 validate_backup_ref(restore, &target, ctx).await?
719 } else {
720 None
721 };
722 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
723 {
724 return Err(Error::MissingData(format!(
725 "restoreImage must be the target's pinned Kanidm image (target image is {})",
726 target.spec.image
727 )));
728 }
729 let storage = target
730 .spec
731 .storage
732 .as_ref()
733 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
734 if storage.empty_dir.is_some()
735 || storage.ephemeral.is_some()
736 || storage.volume_claim_template.is_none()
737 {
738 return Err(Error::MissingData(
739 "restore requires PVC-backed Kanidm storage".to_string(),
740 ));
741 }
742 let primaries = target
743 .spec
744 .replica_groups
745 .iter()
746 .filter(|rg| rg.primary_node)
747 .count();
748 if primaries != 1 {
749 return Err(Error::MissingData(
750 "backup/restore requires exactly one primary replica group".to_string(),
751 ));
752 }
753 let ns = restore.namespace().unwrap();
754 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
755 .list(&ListParams::default())
756 .await
757 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
758 if restores.items.iter().any(|other| {
759 other.name_any() != restore.name_any()
760 && other.spec.target_ref.name == restore.spec.target_ref.name
761 && !matches!(
762 other.status.as_ref().map(|s| s.phase),
763 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
764 )
765 }) {
766 return Err(Error::MissingData(
767 "another active restore targets this Kanidm".to_string(),
768 ));
769 }
770 validate_safety_backup_config(restore)?;
771 Ok((target, disaster_recovery_source))
772}
773
774fn safe_basename(name: &str) -> bool {
775 !name.is_empty()
776 && name != "."
777 && name != ".."
778 && !name.contains('/')
779 && !name.contains('\\')
780 && !name.contains('\0')
781}
782
783fn mutable_image(image: &str) -> bool {
784 image == "kanidm/server:latest"
785 || image.ends_with(":latest")
786 || (!image.contains('@')
787 && !image
788 .rsplit('/')
789 .next()
790 .is_some_and(|part| part.contains(':')))
791}
792
793async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
794 let ns = restore
795 .namespace()
796 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
797 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
798 .get(&restore.spec.target_ref.name)
799 .await
800 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
801}
802
803async fn mark_restoring(
804 restore: &KanidmRestore,
805 target: &Kanidm,
806 ctx: &RestoreContext,
807) -> Result<()> {
808 let ns = target.namespace().unwrap();
809 let uid = restore
810 .uid()
811 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
812 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
813 .patch(
814 &target.name_any(),
815 &PatchParams::default(),
816 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
817 )
818 .await
819 .map(|_| ())
820 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
821}
822
823async fn clear_restoring(
824 restore: &KanidmRestore,
825 target: &Kanidm,
826 ctx: &RestoreContext,
827) -> Result<()> {
828 let ns = target.namespace().unwrap();
829 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
830 return Ok(());
831 }
832 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
833 .patch(
834 &target.name_any(),
835 &PatchParams::default(),
836 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
837 )
838 .await
839 .map(|_| ())
840 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
841}
842
843async fn patch_status(
844 restore: &KanidmRestore,
845 ctx: &RestoreContext,
846 mut status: KanidmRestoreStatus,
847) -> Result<()> {
848 let ns = restore.namespace().unwrap();
849 let previous_phase = restore.status.as_ref().map(|current| current.phase);
850 let next_phase = status.phase;
851 status.observed_generation = restore.metadata.generation;
852 update_restore_conditions(&mut status, restore.metadata.generation);
853
854 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
855 .patch_status(
856 &restore.name_any(),
857 &PatchParams::default(),
858 &Patch::Merge(json!({"status": &status})),
859 )
860 .await
861 .map_err(|e| {
862 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
863 })?;
864
865 if previous_phase != Some(next_phase) {
866 record_restore_transition(
867 restore,
868 ctx,
869 previous_phase,
870 next_phase,
871 status.message.as_deref(),
872 )
873 .await;
874 }
875 Ok(())
876}
877
878fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
879 let previous = status.conditions.clone();
880 let phase = status.phase;
881 let phase_reason = format!("{phase:?}");
882 let message = status
883 .message
884 .clone()
885 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
886 let terminal_success = phase == KanidmRestorePhase::Completed;
887 let terminal_failure = phase == KanidmRestorePhase::Failed;
888 let progressing = !terminal_success && !terminal_failure;
889
890 let sticky_conditions: Vec<Condition> = previous
891 .iter()
892 .filter(|condition| {
893 condition.type_ == CONDITION_BREAK_GLASS
894 || condition.type_ == CONDITION_DISASTER_RECOVERY
895 })
896 .cloned()
897 .collect();
898 status.conditions = vec![
899 restore_condition(
900 &previous,
901 CONDITION_PROGRESSING,
902 if progressing {
903 CONDITION_TRUE
904 } else {
905 CONDITION_FALSE
906 },
907 &phase_reason,
908 &message,
909 generation,
910 ),
911 restore_condition(
912 &previous,
913 CONDITION_READY,
914 if terminal_success {
915 CONDITION_TRUE
916 } else {
917 CONDITION_FALSE
918 },
919 if terminal_success {
920 "RestoreCompleted"
921 } else {
922 "RestoreNotCompleted"
923 },
924 if terminal_success {
925 "Kanidm restore completed successfully."
926 } else {
927 "Kanidm restore has not completed successfully."
928 },
929 generation,
930 ),
931 restore_condition(
932 &previous,
933 CONDITION_FAILED,
934 if terminal_failure {
935 CONDITION_TRUE
936 } else {
937 CONDITION_FALSE
938 },
939 if terminal_failure {
940 "RestoreFailed"
941 } else {
942 "NoRestoreFailure"
943 },
944 if terminal_failure {
945 status
946 .message
947 .as_deref()
948 .unwrap_or("Kanidm restore failed.")
949 } else {
950 "No terminal restore failure has been recorded."
951 },
952 generation,
953 ),
954 ];
955 status.conditions.extend(sticky_conditions);
956}
957
958fn restore_condition(
959 previous: &[Condition],
960 condition_type: &str,
961 condition_status: &str,
962 reason: &str,
963 message: &str,
964 generation: Option<i64>,
965) -> Condition {
966 let last_transition_time = previous
967 .iter()
968 .find(|condition| {
969 condition.type_ == condition_type
970 && condition.status == condition_status
971 && condition.reason == reason
972 })
973 .map(|condition| condition.last_transition_time.clone())
974 .unwrap_or_else(|| Time(Timestamp::now()));
975 Condition {
976 type_: condition_type.to_string(),
977 status: condition_status.to_string(),
978 reason: reason.to_string(),
979 message: message.to_string(),
980 last_transition_time,
981 observed_generation: generation,
982 }
983}
984
985async fn record_restore_transition(
986 restore: &KanidmRestore,
987 ctx: &RestoreContext,
988 previous_phase: Option<KanidmRestorePhase>,
989 phase: KanidmRestorePhase,
990 message: Option<&str>,
991) {
992 if phase == KanidmRestorePhase::Validating {
993 ctx.metrics.attempts.add(1, &[]);
994 }
995
996 let result = match phase {
997 KanidmRestorePhase::Completed => Some("success"),
998 KanidmRestorePhase::Failed => Some("failure"),
999 _ => None,
1000 };
1001 if let Some(result) = result {
1002 let attributes = [
1003 KeyValue::new("result", result),
1004 KeyValue::new("phase", format!("{phase:?}")),
1005 ];
1006 ctx.metrics.outcomes.add(1, &attributes);
1007 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
1008 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
1009 ctx.metrics.duration_seconds.record(elapsed, &attributes);
1010 }
1011 }
1012
1013 let reason = if phase == KanidmRestorePhase::Failed {
1014 "RestoreFailed"
1015 } else {
1016 "RestorePhaseChanged"
1017 };
1018 let note = message.map(str::to_string).or_else(|| {
1019 Some(format!(
1020 "Kanidm restore phase changed from {} to {phase:?}.",
1021 previous_phase
1022 .map(|previous| format!("{previous:?}"))
1023 .unwrap_or_else(|| "None".to_string())
1024 ))
1025 });
1026 if let Err(error) = ctx
1027 .recorder
1028 .publish(
1029 &Event {
1030 type_: if phase == KanidmRestorePhase::Failed {
1031 EventType::Warning
1032 } else {
1033 EventType::Normal
1034 },
1035 reason: reason.to_string(),
1036 note,
1037 action: "Restore".to_string(),
1038 secondary: None,
1039 },
1040 &restore.object_ref(&()),
1041 )
1042 .await
1043 {
1044 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1045 }
1046}
1047
1048async fn set_phase(
1049 restore: &KanidmRestore,
1050 ctx: &RestoreContext,
1051 phase: KanidmRestorePhase,
1052 message: Option<String>,
1053) -> Result<()> {
1054 let mut status = restore.status.clone().unwrap_or_default();
1055 status.phase = phase;
1056 status.message = message;
1057 patch_status(restore, ctx, status).await
1058}
1059
1060async fn fail_after_mutation(
1061 restore: &KanidmRestore,
1062 ctx: &RestoreContext,
1063 message: &str,
1064) -> Result<()> {
1065 set_phase(
1066 restore,
1067 ctx,
1068 KanidmRestorePhase::Failed,
1069 Some(message.to_string()),
1070 )
1071 .await
1072}
1073
1074fn restore_primary_failed_status(
1075 existing: &KanidmRestoreStatus,
1076 job_name: &str,
1077) -> KanidmRestoreStatus {
1078 let mut status = existing.clone();
1079 status.restore_job_name = Some(job_name.to_string());
1080 status.phase = KanidmRestorePhase::Failed;
1081 status.message = Some("database restore job failed".to_string());
1082 status
1083}
1084
1085fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
1086 let mut status = existing.clone();
1087 status.verify_job_name = Some(job_name.to_string());
1088 status.phase = KanidmRestorePhase::Failed;
1089 status.message = Some("database verification failed".to_string());
1090 status
1091}
1092
1093fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1094 target
1095 .spec
1096 .replica_groups
1097 .iter()
1098 .find(|rg| rg.primary_node)
1099 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1100}
1101
1102async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1103 let ns = target.namespace().unwrap();
1104 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1105 for rg in &target.spec.replica_groups {
1106 let name = target.statefulset_name(&rg.name);
1107 api.patch(
1108 &name,
1109 &PatchParams::default(),
1110 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1111 )
1112 .await
1113 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1114 }
1115 Ok(())
1116}
1117
1118async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1119 let ns = target.namespace().unwrap();
1120 let rg = primary_group(target)?;
1121 let name = target.statefulset_name(&rg.name);
1122 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1123 .patch(
1124 &name,
1125 &PatchParams::default(),
1126 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1127 )
1128 .await
1129 .map(|_| ())
1130 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1131}
1132
1133async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1134 let ns = target.namespace().unwrap();
1135 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1136 for rg in &target.spec.replica_groups {
1137 let name = target.statefulset_name(&rg.name);
1138 api.patch(
1139 &name,
1140 &PatchParams::default(),
1141 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1142 )
1143 .await
1144 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1145 }
1146 Ok(())
1147}
1148
1149async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1150 let ns = target.namespace().unwrap();
1151 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1152 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1153 .await
1154 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1155 Ok(pods.items.is_empty())
1156}
1157
1158async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1159 let ns = target.namespace().unwrap();
1160 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1161 let mut pv_names = BTreeSet::new();
1162 let primary_name = primary_pvc_name(target)?;
1163 let primary_pvc = pvc_api
1164 .get(&primary_name)
1165 .await
1166 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1167 let primary_pv = primary_pvc
1168 .spec
1169 .as_ref()
1170 .and_then(|spec| spec.volume_name.clone())
1171 .ok_or_else(|| {
1172 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1173 })?;
1174 pv_names.insert(primary_pv);
1175
1176 for rg in &target.spec.replica_groups {
1177 let sts = target.statefulset_name(&rg.name);
1178 for ordinal in 0..rg.replicas {
1179 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1180 if name == primary_name {
1181 continue;
1182 }
1183 if let Some(pvc) = pvc_api
1184 .get_opt(&name)
1185 .await
1186 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1187 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1188 {
1189 pv_names.insert(pv_name);
1190 }
1191 }
1192 }
1193
1194 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1195 .list(&ListParams::default())
1196 .await
1197 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1198 Ok(!attachments.items.iter().any(|attachment| {
1199 attachment
1200 .spec
1201 .source
1202 .persistent_volume_name
1203 .as_ref()
1204 .is_some_and(|pv_name| pv_names.contains(pv_name))
1205 }))
1206}
1207
1208async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1209 let ns = target.namespace().unwrap();
1210 let name = target.statefulset_name(&primary_group(target)?.name);
1211 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1212 .get(&name)
1213 .await
1214 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1215 Ok(sts
1216 .status
1217 .as_ref()
1218 .and_then(|s| s.ready_replicas)
1219 .unwrap_or(0)
1220 >= 1)
1221}
1222
1223fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1224 let rg = primary_group(target)?;
1225 Ok(format!(
1226 "{DATA_VOLUME}-{}-0",
1227 target.statefulset_name(&rg.name)
1228 ))
1229}
1230
1231async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1232 let ns = target.namespace().unwrap();
1233 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1234 let mut all_absent = true;
1235 for rg in &target.spec.replica_groups {
1236 let sts = target.statefulset_name(&rg.name);
1237 for ordinal in 0..rg.replicas {
1238 if rg.primary_node && ordinal == 0 {
1239 continue;
1240 }
1241 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1242 match api
1243 .get_opt(&name)
1244 .await
1245 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1246 {
1247 None => {}
1248 Some(pvc) => {
1249 all_absent = false;
1250 if pvc.metadata.deletion_timestamp.is_none() {
1251 match api.delete(&name, &DeleteParams::default()).await {
1252 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1253 Err(kube::Error::Api(status)) if status.code == 404 => {}
1254 Err(error) => {
1255 return Err(Error::kube_error(
1256 "delete",
1257 "PersistentVolumeClaim",
1258 &ns,
1259 &name,
1260 error,
1261 ));
1262 }
1263 }
1264 }
1265 }
1266 }
1267 }
1268 }
1269 Ok(all_absent)
1270}
1271
1272async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1273 let ns = target.namespace().unwrap();
1274 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1275 let label_selector = format!(
1276 "{}={},{}={}",
1277 CLUSTER_LABEL,
1278 target.name_any(),
1279 SECRET_TYPE_LABEL,
1280 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1281 );
1282 let secrets = api
1283 .list(&ListParams::default().labels(&label_selector))
1284 .await
1285 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1286 let mut all_absent = secrets.items.is_empty();
1287 for secret in secrets.items {
1288 let name = secret.name_any();
1289 if secret.metadata.deletion_timestamp.is_none() {
1290 match api.delete(&name, &DeleteParams::default()).await {
1291 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1292 Err(kube::Error::Api(status)) if status.code == 404 => {}
1293 Err(error) => {
1294 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1295 }
1296 }
1297 all_absent = false;
1298 }
1299 }
1300 Ok(all_absent)
1301}
1302
1303async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1304 let ns = target.namespace().unwrap();
1305 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1306 let name = format!("{}-admin-passwords", target.name_any());
1307 match api
1308 .get_opt(&name)
1309 .await
1310 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1311 {
1312 None => Ok(true),
1313 Some(secret) => {
1314 if secret.metadata.deletion_timestamp.is_some() {
1315 return Ok(false);
1316 }
1317 match api.delete(&name, &DeleteParams::default()).await {
1318 Ok(_) => {
1319 debug!(secret = %name, "deleting stale admin secret");
1320 Ok(false)
1321 }
1322 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1323 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1324 }
1325 }
1326 }
1327}
1328
1329fn restore_job_name(restore: &KanidmRestore) -> String {
1330 format!("{}-restore", restore.name_any())
1331}
1332fn verify_job_name(restore: &KanidmRestore) -> String {
1333 format!("{}-verify", restore.name_any())
1334}
1335
1336fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1337 let origin = target
1338 .spec
1339 .origin
1340 .clone()
1341 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1342 vec![
1343 EnvVar {
1344 name: "KANIDM_DB_PATH".to_string(),
1345 value: Some("/data/kanidm.db".to_string()),
1346 ..Default::default()
1347 },
1348 EnvVar {
1349 name: "KANIDM_DOMAIN".to_string(),
1350 value: Some(target.spec.domain.clone()),
1351 ..Default::default()
1352 },
1353 EnvVar {
1354 name: "KANIDM_ORIGIN".to_string(),
1355 value: Some(origin),
1356 ..Default::default()
1357 },
1358 EnvVar {
1359 name: "KANIDM_TLS_CHAIN".to_string(),
1360 value: Some(format!("{TLS_PATH}/tls.crt")),
1361 ..Default::default()
1362 },
1363 EnvVar {
1364 name: "KANIDM_TLS_KEY".to_string(),
1365 value: Some(format!("{TLS_PATH}/tls.key")),
1366 ..Default::default()
1367 },
1368 ]
1369}
1370
1371async fn ensure_database_job(
1372 restore: &KanidmRestore,
1373 target: &Kanidm,
1374 ctx: &RestoreContext,
1375 name: &str,
1376 verify: bool,
1377) -> Result<()> {
1378 let ns = restore.namespace().unwrap();
1379 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1380 if api
1381 .get_opt(name)
1382 .await
1383 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1384 .is_some()
1385 {
1386 return Ok(());
1387 }
1388 let mut command = vec![
1389 "kanidmd".to_string(),
1390 "database".to_string(),
1391 if verify { "backup" } else { "restore" }.to_string(),
1392 ];
1393 if verify {
1394 command.push("/verify/verification.json.gz".to_string());
1395 } else {
1396 if is_remote_source(restore) {
1397 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1398 } else {
1399 let file_name = restore
1400 .spec
1401 .source
1402 .local
1403 .as_ref()
1404 .map(|l| l.file_name.clone())
1405 .unwrap_or_default();
1406 command.push(format!("{BACKUP_PATH}/{file_name}"));
1407 }
1408 }
1409 let job = Job {
1410 metadata: ObjectMeta {
1411 name: Some(name.to_string()),
1412 namespace: Some(ns.clone()),
1413 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1414 ..Default::default()
1415 },
1416 spec: Some(JobSpec {
1417 backoff_limit: Some(0),
1418 template: PodTemplateSpec {
1419 metadata: Some(ObjectMeta::default()),
1420 spec: Some(PodSpec {
1421 automount_service_account_token: Some(false),
1422 restart_policy: Some("Never".to_string()),
1423 security_context: Some(source_job_pod_security_context(target)),
1424 containers: vec![Container {
1425 name: if verify { "verify" } else { "restore" }.to_string(),
1426 image: Some(restore.spec.restore_image.clone()),
1427 command: Some(command),
1428 env: Some(build_kanidm_db_env_vars(target)),
1429 security_context: Some(kanidm_job_security_context()),
1430 resources: Some(job_resource_requirements()),
1431 volume_mounts: Some(vec![
1432 VolumeMount {
1433 name: DATA_VOLUME.to_string(),
1434 mount_path: DATA_PATH.to_string(),
1435 ..Default::default()
1436 },
1437 VolumeMount {
1438 name: TLS_VOLUME.to_string(),
1439 mount_path: TLS_PATH.to_string(),
1440 read_only: Some(true),
1441 ..Default::default()
1442 },
1443 VolumeMount {
1444 name: "verification-output".to_string(),
1445 mount_path: "/verify".to_string(),
1446 ..Default::default()
1447 },
1448 ]),
1449 ..Default::default()
1450 }],
1451 volumes: Some(vec![
1452 Volume {
1453 name: DATA_VOLUME.to_string(),
1454 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1455 claim_name: primary_pvc_name(target)?,
1456 read_only: Some(false),
1457 }),
1458 ..Default::default()
1459 },
1460 Volume {
1461 name: TLS_VOLUME.to_string(),
1462 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1463 secret_name: Some(target.effective_tls_secret_name()),
1464 ..Default::default()
1465 }),
1466 ..Default::default()
1467 },
1468 Volume {
1469 name: "verification-output".to_string(),
1470 empty_dir: Some(Default::default()),
1471 ..Default::default()
1472 },
1473 ]),
1474 ..Default::default()
1475 }),
1476 },
1477 ..Default::default()
1478 }),
1479 ..Default::default()
1480 };
1481 api.create(&PostParams::default(), &job)
1482 .await
1483 .map(|_| ())
1484 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1485}
1486
1487enum JobState {
1488 Running,
1489 Complete,
1490 Failed,
1491}
1492
1493async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1494 let ns = restore.namespace().unwrap();
1495 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1496 .get(name)
1497 .await
1498 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1499 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1500 return Ok(JobState::Failed);
1501 }
1502 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1503 return Ok(JobState::Complete);
1504 }
1505 Ok(JobState::Running)
1506}
1507
1508fn is_remote_source(restore: &KanidmRestore) -> bool {
1509 restore.spec.source.backup_ref.is_some()
1510}
1511
1512fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1513 let safety = restore.spec.safety_backup.as_ref();
1514 let skip = safety.is_some_and(|s| s.skip);
1515 !skip
1516}
1517
1518fn validate_source(restore: &KanidmRestore) -> Result<()> {
1519 let has_local = restore.spec.source.local.is_some();
1520 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1521 if has_local && has_backup_ref {
1522 return Err(Error::MissingData(
1523 "source.local and source.backupRef are mutually exclusive".to_string(),
1524 ));
1525 }
1526 if !has_local && !has_backup_ref {
1527 return Err(Error::MissingData(
1528 "source must specify either local or backupRef".to_string(),
1529 ));
1530 }
1531 if let Some(local) = &restore.spec.source.local {
1532 if !safe_basename(&local.file_name) {
1533 return Err(Error::MissingData(
1534 "restore source fileName must be a safe basename".to_string(),
1535 ));
1536 }
1537 }
1538 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1539 if backup_ref.name.is_empty() {
1540 return Err(Error::MissingData(
1541 "source.backupRef.name must not be empty".to_string(),
1542 ));
1543 }
1544 }
1545 Ok(())
1546}
1547
1548fn disaster_recovery_requested(restore: &KanidmRestore) -> bool {
1549 restore
1550 .metadata
1551 .annotations
1552 .as_ref()
1553 .and_then(|annotations| annotations.get(DISASTER_RECOVERY_ANNOTATION))
1554 .is_some_and(|value| value == "true")
1555}
1556
1557fn validate_disaster_recovery_approval(restore: &KanidmRestore) -> Result<()> {
1558 let annotations = restore.metadata.annotations.as_ref();
1559 let reason = annotations
1560 .and_then(|values| values.get(BREAK_GLASS_REASON_ANNOTATION))
1561 .map(String::as_str)
1562 .filter(|value| !value.trim().is_empty());
1563 let approved_by = annotations
1564 .and_then(|values| values.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1565 .map(String::as_str)
1566 .filter(|value| !value.trim().is_empty());
1567
1568 if reason.is_none() || approved_by.is_none() {
1569 return Err(Error::MissingData(format!(
1570 "disaster-recovery restore requires non-empty {} and {} annotations",
1571 BREAK_GLASS_REASON_ANNOTATION, BREAK_GLASS_APPROVED_BY_ANNOTATION
1572 )));
1573 }
1574 Ok(())
1575}
1576
1577async fn record_disaster_recovery_override(
1578 restore: &KanidmRestore,
1579 target: &Kanidm,
1580 ctx: &RestoreContext,
1581 source: &DisasterRecoverySource,
1582) {
1583 ctx.metrics.disaster_recovery_total.add(1, &[]);
1584 let reason = restore
1585 .metadata
1586 .annotations
1587 .as_ref()
1588 .and_then(|annotations| annotations.get(BREAK_GLASS_REASON_ANNOTATION))
1589 .cloned()
1590 .unwrap_or_default();
1591 let approved_by = restore
1592 .metadata
1593 .annotations
1594 .as_ref()
1595 .and_then(|annotations| annotations.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1596 .cloned()
1597 .unwrap_or_default();
1598 let target_uid = target.uid().unwrap_or_default();
1599 let note = format!(
1600 "Approved disaster recovery from source {}/{} into target {}/{}. reason={}, approvedBy={}",
1601 source.name,
1602 source.uid,
1603 target.name_any(),
1604 target_uid,
1605 reason,
1606 approved_by
1607 );
1608 if let Err(error) = ctx
1609 .recorder
1610 .publish(
1611 &Event {
1612 type_: EventType::Warning,
1613 reason: "DisasterRecoveryOverride".to_string(),
1614 note: Some(note.clone()),
1615 action: "Restore".to_string(),
1616 secondary: None,
1617 },
1618 &restore.object_ref(&()),
1619 )
1620 .await
1621 {
1622 warn!(restore = %restore.name_any(), %error, "failed to publish disaster-recovery event");
1623 }
1624 warn!(restore = %restore.name_any(), %note, "disaster-recovery source identity override activated");
1625}
1626
1627fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1628 let safety = restore.spec.safety_backup.as_ref();
1629 let skip = safety.is_some_and(|s| s.skip);
1630 if skip {
1631 let annotations = restore.metadata.annotations.as_ref();
1632 let reason = annotations
1633 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1634 .map(|s| s.trim())
1635 .unwrap_or("");
1636 let approver = annotations
1637 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1638 .map(|s| s.trim())
1639 .unwrap_or("");
1640 if reason.is_empty() {
1641 return Err(Error::MissingData(format!(
1642 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1643 )));
1644 }
1645 if approver.is_empty() {
1646 return Err(Error::MissingData(format!(
1647 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1648 )));
1649 }
1650 }
1651 if !skip {
1652 let has_repo = safety
1653 .and_then(|s| s.repository_ref.as_ref())
1654 .is_some_and(|r| !r.name.is_empty());
1655 if !has_repo {
1656 return Err(Error::MissingData(
1657 "safety backup requires safetyBackup.repositoryRef when skip is not set"
1658 .to_string(),
1659 ));
1660 }
1661 }
1662 Ok(())
1663}
1664
1665fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1666 target
1667 .spec
1668 .replica_groups
1669 .iter()
1670 .map(|rg| ReplicaCountEntry {
1671 group: rg.name.clone(),
1672 replicas: rg.replicas,
1673 })
1674 .collect()
1675}
1676
1677async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1678 if let Ok(target) = get_target(restore, ctx).await {
1679 let owns_maintenance =
1680 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1681 if owns_maintenance {
1682 scale_desired(&target, ctx).await?;
1683 clear_restoring(restore, &target, ctx).await?;
1684 }
1685 }
1686 Ok(())
1687}
1688
1689async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1690 ctx.metrics.break_glass_total.add(1, &[]);
1691 let reason = restore
1692 .metadata
1693 .annotations
1694 .as_ref()
1695 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1696 .cloned()
1697 .unwrap_or_default();
1698 let approver = restore
1699 .metadata
1700 .annotations
1701 .as_ref()
1702 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1703 .cloned()
1704 .unwrap_or_default();
1705 let mut status = restore.status.clone().unwrap_or_default();
1706 let break_glass_condition = restore_condition(
1707 &status.conditions,
1708 CONDITION_BREAK_GLASS,
1709 CONDITION_TRUE,
1710 "BreakGlassOverride",
1711 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1712 restore.metadata.generation,
1713 );
1714 status
1715 .conditions
1716 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1717 status.conditions.push(break_glass_condition);
1718 patch_status(restore, ctx, status).await?;
1719 if let Err(error) = ctx
1720 .recorder
1721 .publish(
1722 &Event {
1723 type_: EventType::Warning,
1724 reason: "SafetyBackupSkipped".to_string(),
1725 note: Some(format!(
1726 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1727 )),
1728 action: "Restore".to_string(),
1729 secondary: None,
1730 },
1731 &restore.object_ref(&()),
1732 )
1733 .await
1734 {
1735 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1736 }
1737 warn!(
1738 restore = %restore.name_any(),
1739 reason = %reason,
1740 approver = %approver,
1741 "break-glass safety backup override activated"
1742 );
1743 Ok(())
1744}
1745
1746async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1747 let status = restore.status.as_ref().cloned().unwrap_or_default();
1748 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1749 if let Ok(start_ts) = start.parse::<Timestamp>() {
1750 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1751 ctx.metrics
1752 .safety_backup_duration_seconds
1753 .record(elapsed, &[]);
1754 }
1755 }
1756}
1757
1758fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1759 format!("{}-safety-backup", restore.name_any())
1760}
1761
1762fn source_prep_job_name(restore: &KanidmRestore) -> String {
1763 format!("{}-source-prep", restore.name_any())
1764}
1765
1766fn source_check_job_name(restore: &KanidmRestore) -> String {
1767 format!("{}-source-check", restore.name_any())
1768}
1769
1770async fn ensure_source_check_job(
1771 restore: &KanidmRestore,
1772 target: &Kanidm,
1773 ctx: &RestoreContext,
1774 name: &str,
1775) -> Result<()> {
1776 let ns = restore.namespace().unwrap();
1777 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1778 if api
1779 .get_opt(name)
1780 .await
1781 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1782 .is_some()
1783 {
1784 return Ok(());
1785 }
1786 let file_name = restore
1787 .spec
1788 .source
1789 .local
1790 .as_ref()
1791 .map(|l| l.file_name.clone())
1792 .unwrap_or_default();
1793 let operation_doc = serde_json::json!({
1794 "apiVersion": "backup.kaniop.rs/v1alpha1",
1795 "kind": "OperationDocument",
1796 "operation": "check",
1797 "path": format!("{BACKUP_PATH}/{file_name}"),
1798 "resultPath": "/run/kaniop-result/result.json",
1799 "format": "kanidmJsonGzip",
1800 })
1801 .to_string();
1802 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1803 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1804 let data_mover_image = data_mover_image();
1805 let job = Job {
1806 metadata: ObjectMeta {
1807 name: Some(name.to_string()),
1808 namespace: Some(ns.clone()),
1809 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1810 ..Default::default()
1811 },
1812 spec: Some(JobSpec {
1813 backoff_limit: Some(0),
1814 template: PodTemplateSpec {
1815 metadata: Some(ObjectMeta::default()),
1816 spec: Some(PodSpec {
1817 automount_service_account_token: Some(false),
1818 restart_policy: Some("Never".to_string()),
1819 security_context: Some(source_job_pod_security_context(target)),
1820 containers: vec![Container {
1821 name: "source-check".to_string(),
1822 image: Some(data_mover_image),
1823 command: Some(vec![
1824 "/bin/kaniop-data-mover".to_string(),
1825 "check".to_string(),
1826 "--operation-doc".to_string(),
1827 "/run/kaniop/operation.json".to_string(),
1828 ]),
1829 security_context: Some(hardened_security_context()),
1830 resources: Some(job_resource_requirements()),
1831 env: Some(vec![EnvVar {
1832 name: "RUST_LOG".to_string(),
1833 value: Some("info".to_string()),
1834 ..Default::default()
1835 }]),
1836 volume_mounts: Some(vec![
1837 VolumeMount {
1838 name: DATA_VOLUME.to_string(),
1839 mount_path: DATA_PATH.to_string(),
1840 read_only: Some(true),
1841 ..Default::default()
1842 },
1843 VolumeMount {
1844 name: "operation".to_string(),
1845 mount_path: "/run/kaniop".to_string(),
1846 read_only: Some(true),
1847 ..Default::default()
1848 },
1849 VolumeMount {
1850 name: "result".to_string(),
1851 mount_path: "/run/kaniop-result".to_string(),
1852 ..Default::default()
1853 },
1854 ]),
1855 ..Default::default()
1856 }],
1857 volumes: Some(vec![
1858 Volume {
1859 name: DATA_VOLUME.to_string(),
1860 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1861 claim_name: primary_pvc_name(target)?,
1862 read_only: Some(true),
1863 }),
1864 ..Default::default()
1865 },
1866 Volume {
1867 name: "operation".to_string(),
1868 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1869 name: operation_cm_name,
1870 ..Default::default()
1871 }),
1872 ..Default::default()
1873 },
1874 Volume {
1875 name: "result".to_string(),
1876 empty_dir: Some(Default::default()),
1877 ..Default::default()
1878 },
1879 ]),
1880 ..Default::default()
1881 }),
1882 },
1883 ..Default::default()
1884 }),
1885 ..Default::default()
1886 };
1887 api.create(&PostParams::default(), &job)
1888 .await
1889 .map(|_| ())
1890 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1891}
1892
1893const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1894
1895fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1896 let ns = restore.namespace().unwrap_or_default();
1897 let uid = restore.uid().ok_or_else(|| {
1898 Error::MissingData(
1899 "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1900 .to_string(),
1901 )
1902 })?;
1903 let namespace_uuid =
1904 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1905 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1906 Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1907}
1908
1909fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1917 let ns = restore.namespace().unwrap_or_default();
1918 let kanidm_uid = &restore.spec.target_ref.uid;
1919 let backup_id = compute_safety_backup_id(restore)?;
1920 Ok(format!(
1921 "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1922 ))
1923}
1924
1925struct VerifiedSafetyBackup {
1926 manifest_key: String,
1927 payload_sha256: String,
1928}
1929
1930async fn read_safety_backup_result(
1931 restore: &KanidmRestore,
1932 ctx: &RestoreContext,
1933 job_name: &str,
1934 expected_backup_id: &str,
1935) -> Result<VerifiedSafetyBackup> {
1936 let ns = restore.namespace().unwrap();
1937 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1938 let pod_list = pods
1939 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1940 .await
1941 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1942 let pod = pod_list.items.first().ok_or_else(|| {
1943 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1944 })?;
1945 let pod_name = pod.name_any();
1946 let container_status = pod
1947 .status
1948 .as_ref()
1949 .and_then(|s| s.container_statuses.as_ref())
1950 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1951 .ok_or_else(|| {
1952 Error::MissingData(format!(
1953 "safety-upload container status not found in pod {pod_name}"
1954 ))
1955 })?;
1956 let termination_message = container_status
1957 .state
1958 .as_ref()
1959 .and_then(|state| state.terminated.as_ref())
1960 .and_then(|t| t.message.as_ref())
1961 .ok_or_else(|| {
1962 Error::MissingData(format!(
1963 "termination message not found in pod {pod_name}; result transport failed"
1964 ))
1965 })?;
1966 if termination_message.is_empty() {
1967 return Err(Error::MissingData(format!(
1968 "termination message is empty in pod {pod_name}; result document was not written"
1969 )));
1970 }
1971 let result_doc = parse_result_document(termination_message).map_err(|e| {
1972 Error::ParseError(format!(
1973 "failed to parse result document from pod {pod_name}: {e}"
1974 ))
1975 })?;
1976 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1977 return Err(Error::ParseError(format!(
1978 "unsupported result document version: {}",
1979 result_doc.api_version
1980 )));
1981 }
1982 if result_doc.kind != "ResultDocument" {
1983 return Err(Error::ParseError(format!(
1984 "invalid result document kind: {}",
1985 result_doc.kind
1986 )));
1987 }
1988 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1989 return Err(Error::ParseError(format!(
1990 "result document operation mismatch: expected '{}', got '{}'",
1991 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1992 )));
1993 }
1994 if !result_doc.success {
1995 return Err(Error::ParseError(format!(
1996 "safety backup reported failure with exit code {:?}",
1997 result_doc.exit_code
1998 )));
1999 }
2000 if result_doc.exit_code != ExitCode::Success {
2001 return Err(Error::ParseError(format!(
2002 "safety backup exit code is not Success: {:?}",
2003 result_doc.exit_code
2004 )));
2005 }
2006 let doc_backup_id = result_doc
2007 .backup_id
2008 .as_deref()
2009 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2010 if doc_backup_id != expected_backup_id {
2011 return Err(Error::ParseError(format!(
2012 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2013 )));
2014 }
2015 let manifest_key = result_doc
2016 .manifest_key
2017 .clone()
2018 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2019 if manifest_key.is_empty() {
2020 return Err(Error::ParseError(
2021 "result document manifestKey is empty".to_string(),
2022 ));
2023 }
2024 let payload_sha256 = result_doc
2025 .payload_sha256
2026 .clone()
2027 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2028 if payload_sha256.is_empty() {
2029 return Err(Error::ParseError(
2030 "result document payloadSha256 is empty".to_string(),
2031 ));
2032 }
2033 info!(
2034 restore = %restore.name_any(),
2035 backup_id = %expected_backup_id,
2036 manifest_key = %manifest_key,
2037 "safety backup result verified"
2038 );
2039 Ok(VerifiedSafetyBackup {
2040 manifest_key,
2041 payload_sha256,
2042 })
2043}
2044
2045struct VerifiedSourcePrepResult {
2046 #[allow(dead_code)]
2047 manifest_key: String,
2048 payload_sha256: String,
2049}
2050
2051async fn read_source_prep_result(
2052 restore: &KanidmRestore,
2053 ctx: &RestoreContext,
2054 job_name: &str,
2055 expected_backup_id: &str,
2056) -> Result<VerifiedSourcePrepResult> {
2057 let ns = restore.namespace().unwrap();
2058 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
2059 let pod_list = pods
2060 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
2061 .await
2062 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
2063 let pod = pod_list.items.first().ok_or_else(|| {
2064 Error::MissingData(format!("no pod found for source prep job {job_name}"))
2065 })?;
2066 let pod_name = pod.name_any();
2067 let container_status = pod
2068 .status
2069 .as_ref()
2070 .and_then(|s| s.container_statuses.as_ref())
2071 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
2072 .ok_or_else(|| {
2073 Error::MissingData(format!(
2074 "source-prep container status not found in pod {pod_name}"
2075 ))
2076 })?;
2077 let termination_message = container_status
2078 .state
2079 .as_ref()
2080 .and_then(|state| state.terminated.as_ref())
2081 .and_then(|t| t.message.as_ref())
2082 .ok_or_else(|| {
2083 Error::MissingData(format!(
2084 "termination message not found in pod {pod_name}; result transport failed"
2085 ))
2086 })?;
2087 if termination_message.is_empty() {
2088 return Err(Error::MissingData(format!(
2089 "termination message is empty in pod {pod_name}; result document was not written"
2090 )));
2091 }
2092 let result_doc = parse_result_document(termination_message).map_err(|e| {
2093 Error::ParseError(format!(
2094 "failed to parse result document from pod {pod_name}: {e}"
2095 ))
2096 })?;
2097 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
2098 return Err(Error::ParseError(format!(
2099 "unsupported result document version: {}",
2100 result_doc.api_version
2101 )));
2102 }
2103 if result_doc.kind != "ResultDocument" {
2104 return Err(Error::ParseError(format!(
2105 "invalid result document kind: {}",
2106 result_doc.kind
2107 )));
2108 }
2109 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
2110 return Err(Error::ParseError(format!(
2111 "result document operation mismatch: expected '{}', got '{}'",
2112 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
2113 )));
2114 }
2115 if !result_doc.success {
2116 return Err(Error::ParseError(format!(
2117 "source prep reported failure with exit code {:?}",
2118 result_doc.exit_code
2119 )));
2120 }
2121 if result_doc.exit_code != ExitCode::Success {
2122 return Err(Error::ParseError(format!(
2123 "source prep exit code is not Success: {:?}",
2124 result_doc.exit_code
2125 )));
2126 }
2127 let doc_backup_id = result_doc
2128 .backup_id
2129 .as_deref()
2130 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2131 if doc_backup_id != expected_backup_id {
2132 return Err(Error::ParseError(format!(
2133 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2134 )));
2135 }
2136 let manifest_key = result_doc
2137 .manifest_key
2138 .clone()
2139 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2140 if manifest_key.is_empty() {
2141 return Err(Error::ParseError(
2142 "result document manifestKey is empty".to_string(),
2143 ));
2144 }
2145 let payload_sha256 = result_doc
2146 .payload_sha256
2147 .clone()
2148 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2149 if payload_sha256.is_empty() {
2150 return Err(Error::ParseError(
2151 "result document payloadSha256 is empty".to_string(),
2152 ));
2153 }
2154 info!(
2155 restore = %restore.name_any(),
2156 backup_id = %expected_backup_id,
2157 manifest_key = %manifest_key,
2158 payload_sha256 = %payload_sha256,
2159 "source prep result verified"
2160 );
2161 Ok(VerifiedSourcePrepResult {
2162 manifest_key,
2163 payload_sha256,
2164 })
2165}
2166
2167fn source_job_pod_security_context(
2168 target: &Kanidm,
2169) -> k8s_openapi::api::core::v1::PodSecurityContext {
2170 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2171 security_context.seccomp_profile = Some(SeccompProfile {
2172 type_: "RuntimeDefault".to_string(),
2173 ..Default::default()
2174 });
2175 security_context
2176}
2177
2178fn hardened_security_context() -> SecurityContext {
2179 SecurityContext {
2180 allow_privilege_escalation: Some(false),
2181 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2182 drop: Some(vec!["ALL".to_string()]),
2183 ..Default::default()
2184 }),
2185 read_only_root_filesystem: Some(true),
2186 run_as_non_root: Some(true),
2187 seccomp_profile: Some(SeccompProfile {
2188 type_: "RuntimeDefault".to_string(),
2189 ..Default::default()
2190 }),
2191 ..Default::default()
2192 }
2193}
2194
2195fn kanidm_job_security_context() -> SecurityContext {
2196 SecurityContext {
2197 allow_privilege_escalation: Some(false),
2198 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2199 drop: Some(vec!["ALL".to_string()]),
2200 ..Default::default()
2201 }),
2202 seccomp_profile: Some(SeccompProfile {
2203 type_: "RuntimeDefault".to_string(),
2204 ..Default::default()
2205 }),
2206 ..Default::default()
2207 }
2208}
2209
2210fn job_resource_requirements() -> ResourceRequirements {
2211 ResourceRequirements {
2212 requests: Some(std::collections::BTreeMap::from([
2213 ("cpu".to_string(), Quantity("100m".to_string())),
2214 ("memory".to_string(), Quantity("128Mi".to_string())),
2215 ])),
2216 limits: Some(std::collections::BTreeMap::from([
2217 ("cpu".to_string(), Quantity("2".to_string())),
2218 ("memory".to_string(), Quantity("2Gi".to_string())),
2219 ])),
2220 ..Default::default()
2221 }
2222}
2223
2224fn is_usage_lock_persisted(status: Option<&KanidmBackupRepositoryStatus>) -> bool {
2227 status.is_some_and(|status| {
2228 status.usage_started == Some(true)
2229 || (status.usage_started.is_none() && status.observed_generation.is_some())
2230 })
2231}
2232
2233async fn safety_repository_usage_locked(
2234 restore: &KanidmRestore,
2235 ctx: &RestoreContext,
2236) -> Result<bool> {
2237 let repository_name = restore
2238 .spec
2239 .safety_backup
2240 .as_ref()
2241 .and_then(|safety| safety.repository_ref.as_ref())
2242 .map(|reference| reference.name.as_str())
2243 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2244 let namespace = restore.namespace().unwrap_or_default();
2245 let repository = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &namespace)
2246 .get(repository_name)
2247 .await
2248 .map_err(|error| {
2249 Error::kube_error(
2250 "get",
2251 "KanidmBackupRepository",
2252 &namespace,
2253 repository_name,
2254 error,
2255 )
2256 })?;
2257 Ok(is_usage_lock_persisted(repository.status.as_ref()))
2258}
2259
2260async fn ensure_safety_backup_job(
2261 restore: &KanidmRestore,
2262 target: &Kanidm,
2263 ctx: &RestoreContext,
2264 name: &str,
2265 backup_id: &str,
2266 manifest_key: &str,
2267) -> Result<()> {
2268 let ns = restore.namespace().unwrap();
2269 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2270 if api
2271 .get_opt(name)
2272 .await
2273 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2274 .is_some()
2275 {
2276 return Ok(());
2277 }
2278 let safety = restore
2279 .spec
2280 .safety_backup
2281 .as_ref()
2282 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2283 let repo_ref = safety
2284 .repository_ref
2285 .as_ref()
2286 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2287 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2288 .get(&repo_ref.name)
2289 .await
2290 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2291 let _kanidm_uid = &restore.spec.target_ref.uid;
2292 let endpoint = &repo.spec.s3.endpoint;
2293 let region = &repo.spec.s3.region;
2294 let operation_doc = build_safety_upload_operation_doc(
2295 restore,
2296 target,
2297 backup_id,
2298 manifest_key,
2299 &repo.spec.s3.bucket,
2300 &repo.spec.s3.prefix,
2301 endpoint,
2302 region,
2303 repo.spec.s3.force_path_style,
2304 repo.spec.s3.insecure,
2305 repo.spec
2306 .s3
2307 .ca_bundle_ref
2308 .as_ref()
2309 .map(|_| ca_bundle_path())
2310 .as_deref(),
2311 repo.spec.encryption.as_ref(),
2312 )?;
2313 let operation_cm_name = format!("{name}-op");
2314 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2315 let auth_method = &repo.spec.authentication.writer;
2316 let data_mover_image = data_mover_image();
2317 let init_command = vec![
2318 "kanidmd".to_string(),
2319 "database".to_string(),
2320 "backup".to_string(),
2321 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2322 ];
2323 let upload_script = format!(
2324 r#"set -eu
2325/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2326RESULT_FILE="/run/kaniop-result/result.json"
2327TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2328if [ ! -f "$RESULT_FILE" ]; then
2329 echo "ERROR: result document not found at $RESULT_FILE"
2330 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2331 exit 1
2332fi
2333if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2334 echo "ERROR: safety backup upload reported failure"
2335 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2336 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2337 exit 1
2338fi
2339dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2340echo "safety backup upload completed successfully"
2341"#
2342 );
2343 let job = Job {
2344 metadata: ObjectMeta {
2345 name: Some(name.to_string()),
2346 namespace: Some(ns.clone()),
2347 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2348 ..Default::default()
2349 },
2350 spec: Some(JobSpec {
2351 backoff_limit: Some(0),
2352 template: PodTemplateSpec {
2353 metadata: Some(ObjectMeta::default()),
2354 spec: Some(PodSpec {
2355 automount_service_account_token: Some(false),
2356 restart_policy: Some("Never".to_string()),
2357 security_context: Some(source_job_pod_security_context(target)),
2358 init_containers: Some(vec![Container {
2359 name: "safety-backup".to_string(),
2360 image: Some(restore.spec.restore_image.clone()),
2361 command: Some(init_command),
2362 security_context: Some(SecurityContext {
2363 allow_privilege_escalation: Some(false),
2364 capabilities: Some(Capabilities {
2365 drop: Some(vec!["ALL".to_string()]),
2366 ..Default::default()
2367 }),
2368 read_only_root_filesystem: Some(true),
2369 seccomp_profile: Some(SeccompProfile {
2370 type_: "RuntimeDefault".to_string(),
2371 ..Default::default()
2372 }),
2373 ..Default::default()
2374 }),
2375 resources: Some(job_resource_requirements()),
2376 env: Some(build_kanidm_db_env_vars(target)),
2377 volume_mounts: Some(vec![
2378 VolumeMount {
2379 name: DATA_VOLUME.to_string(),
2380 mount_path: DATA_PATH.to_string(),
2381 read_only: Some(false),
2382 ..Default::default()
2383 },
2384 VolumeMount {
2385 name: TLS_VOLUME.to_string(),
2386 mount_path: TLS_PATH.to_string(),
2387 read_only: Some(true),
2388 ..Default::default()
2389 },
2390 VolumeMount {
2391 name: SHARED_VOLUME.to_string(),
2392 mount_path: SHARED_VOL_PATH.to_string(),
2393 ..Default::default()
2394 },
2395 ]),
2396 ..Default::default()
2397 }]),
2398 containers: vec![Container {
2399 name: "safety-upload".to_string(),
2400 image: Some(data_mover_image),
2401 command: Some(vec!["/bin/sh".to_string()]),
2402 args: Some(vec!["-c".to_string(), upload_script]),
2403 security_context: Some(hardened_security_context()),
2404 resources: Some(job_resource_requirements()),
2405 env: {
2406 let mut env_vars =
2407 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2408 env_vars.push(EnvVar {
2409 name: "RUST_LOG".to_string(),
2410 value: Some("info".to_string()),
2411 ..Default::default()
2412 });
2413 if repo.spec.s3.ca_bundle_ref.is_some() {
2414 env_vars.push(ca_bundle_env_var());
2415 }
2416 env_vars.extend(build_encryption_env_vars(
2417 repo.spec
2418 .encryption
2419 .as_ref()
2420 .and_then(|e| e.key_ref.as_ref()),
2421 ));
2422 Some(env_vars)
2423 },
2424 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2425 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2426 volume_mounts: {
2427 let mut mounts = vec![
2428 VolumeMount {
2429 name: SHARED_VOLUME.to_string(),
2430 mount_path: SHARED_VOL_PATH.to_string(),
2431 read_only: Some(true),
2432 ..Default::default()
2433 },
2434 VolumeMount {
2435 name: "operation".to_string(),
2436 mount_path: "/run/kaniop".to_string(),
2437 read_only: Some(true),
2438 ..Default::default()
2439 },
2440 VolumeMount {
2441 name: "result".to_string(),
2442 mount_path: "/run/kaniop-result".to_string(),
2443 ..Default::default()
2444 },
2445 ];
2446 mounts.extend(build_auth_volume_mounts(auth_method));
2447 if repo.spec.s3.ca_bundle_ref.is_some() {
2448 mounts.push(build_ca_bundle_volume_mount());
2449 }
2450 Some(mounts)
2451 },
2452 ..Default::default()
2453 }],
2454 volumes: {
2455 let mut vols = vec![
2456 Volume {
2457 name: DATA_VOLUME.to_string(),
2458 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2459 claim_name: primary_pvc_name(target)?,
2460 read_only: Some(false),
2461 }),
2462 ..Default::default()
2463 },
2464 Volume {
2465 name: TLS_VOLUME.to_string(),
2466 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2467 secret_name: Some(target.effective_tls_secret_name()),
2468 ..Default::default()
2469 }),
2470 ..Default::default()
2471 },
2472 Volume {
2473 name: SHARED_VOLUME.to_string(),
2474 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2475 size_limit: Some(crate::controller::backup_job_volume_size()),
2476 ..Default::default()
2477 }),
2478 ..Default::default()
2479 },
2480 Volume {
2481 name: "operation".to_string(),
2482 config_map: Some(ConfigMapVolumeSource {
2483 name: operation_cm_name,
2484 ..Default::default()
2485 }),
2486 ..Default::default()
2487 },
2488 Volume {
2489 name: "result".to_string(),
2490 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2491 size_limit: Some(Quantity("16Mi".to_string())),
2492 ..Default::default()
2493 }),
2494 ..Default::default()
2495 },
2496 ];
2497 vols.extend(build_auth_volumes(auth_method));
2498 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2499 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2500 }
2501 Some(vols)
2502 },
2503 ..Default::default()
2504 }),
2505 },
2506 ..Default::default()
2507 }),
2508 ..Default::default()
2509 };
2510 api.create(&PostParams::default(), &job)
2511 .await
2512 .map(|_| ())
2513 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2514}
2515
2516async fn ensure_source_prep_job(
2517 restore: &KanidmRestore,
2518 target: &Kanidm,
2519 ctx: &RestoreContext,
2520 name: &str,
2521) -> Result<()> {
2522 let ns = restore.namespace().unwrap();
2523 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2524 if api
2525 .get_opt(name)
2526 .await
2527 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2528 .is_some()
2529 {
2530 return Ok(());
2531 }
2532 let backup_name = restore
2533 .spec
2534 .source
2535 .backup_ref
2536 .as_ref()
2537 .map(|b| b.name.clone())
2538 .unwrap_or_default();
2539 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2540 .get(&backup_name)
2541 .await
2542 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2543 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2544 .get(&backup.spec.repository_ref.name)
2545 .await
2546 .map_err(|e| {
2547 Error::kube_error(
2548 "get",
2549 "KanidmBackupRepository",
2550 &ns,
2551 &backup.spec.repository_ref.name,
2552 e,
2553 )
2554 })?;
2555 let endpoint = &repo.spec.s3.endpoint;
2556 let region = &repo.spec.s3.region;
2557 let operation_doc = build_download_operation_doc(
2558 target,
2559 &backup.spec.manifest_key,
2560 &backup.spec.backup_id,
2561 &backup.spec.kanidm_ref.uid,
2562 &repo.spec.s3.bucket,
2563 &repo.spec.s3.prefix,
2564 endpoint,
2565 region,
2566 repo.spec.s3.force_path_style,
2567 repo.spec.s3.insecure,
2568 repo.spec
2569 .s3
2570 .ca_bundle_ref
2571 .as_ref()
2572 .map(|_| ca_bundle_path())
2573 .as_deref(),
2574 repo.spec.encryption.as_ref(),
2575 );
2576 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2577 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2578 let auth_method = &repo.spec.authentication.reader;
2579 let data_mover_image = data_mover_image();
2580 let download_script = format!(
2581 r#"set -eu
2582/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2583RESULT_FILE="/run/kaniop-result/result.json"
2584TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2585if [ ! -f "$RESULT_FILE" ]; then
2586 echo "ERROR: result document not found at $RESULT_FILE"
2587 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2588 exit 1
2589fi
2590if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2591 echo "ERROR: source preparation download reported failure"
2592 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2593 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2594 exit 1
2595fi
2596dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2597echo "source preparation download completed successfully"
2598"#
2599 );
2600 let job = Job {
2601 metadata: ObjectMeta {
2602 name: Some(name.to_string()),
2603 namespace: Some(ns.clone()),
2604 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2605 ..Default::default()
2606 },
2607 spec: Some(JobSpec {
2608 backoff_limit: Some(0),
2609 template: PodTemplateSpec {
2610 metadata: Some(ObjectMeta::default()),
2611 spec: Some(PodSpec {
2612 automount_service_account_token: Some(false),
2613 restart_policy: Some("Never".to_string()),
2614 security_context: Some(source_job_pod_security_context(target)),
2615 containers: vec![Container {
2616 name: "source-prep".to_string(),
2617 image: Some(data_mover_image),
2618 command: Some(vec!["/bin/sh".to_string()]),
2619 args: Some(vec!["-c".to_string(), download_script]),
2620 security_context: Some(hardened_security_context()),
2621 resources: Some(job_resource_requirements()),
2622 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2623 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2624 env: {
2625 let mut env_vars = build_auth_env_vars(
2626 auth_method,
2627 &backup.spec.repository_ref.name,
2628 AuthRole::Reader,
2629 );
2630 env_vars.push(EnvVar {
2631 name: "RUST_LOG".to_string(),
2632 value: Some("info".to_string()),
2633 ..Default::default()
2634 });
2635 if repo.spec.s3.ca_bundle_ref.is_some() {
2636 env_vars.push(ca_bundle_env_var());
2637 }
2638 env_vars.extend(build_encryption_env_vars(
2639 repo.spec
2640 .encryption
2641 .as_ref()
2642 .and_then(|e| e.key_ref.as_ref()),
2643 ));
2644 Some(env_vars)
2645 },
2646 volume_mounts: {
2647 let mut mounts = vec![
2648 VolumeMount {
2649 name: DATA_VOLUME.to_string(),
2650 mount_path: DATA_PATH.to_string(),
2651 ..Default::default()
2652 },
2653 VolumeMount {
2654 name: "operation".to_string(),
2655 mount_path: "/run/kaniop".to_string(),
2656 read_only: Some(true),
2657 ..Default::default()
2658 },
2659 VolumeMount {
2660 name: "result".to_string(),
2661 mount_path: "/run/kaniop-result".to_string(),
2662 ..Default::default()
2663 },
2664 ];
2665 mounts.extend(build_auth_volume_mounts(auth_method));
2666 if repo.spec.s3.ca_bundle_ref.is_some() {
2667 mounts.push(build_ca_bundle_volume_mount());
2668 }
2669 Some(mounts)
2670 },
2671 ..Default::default()
2672 }],
2673 volumes: {
2674 let mut vols = vec![
2675 Volume {
2676 name: DATA_VOLUME.to_string(),
2677 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2678 claim_name: primary_pvc_name(target)?,
2679 read_only: Some(false),
2680 }),
2681 ..Default::default()
2682 },
2683 Volume {
2684 name: "operation".to_string(),
2685 config_map: Some(ConfigMapVolumeSource {
2686 name: operation_cm_name,
2687 ..Default::default()
2688 }),
2689 ..Default::default()
2690 },
2691 Volume {
2692 name: "result".to_string(),
2693 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2694 size_limit: Some(Quantity("16Mi".to_string())),
2695 ..Default::default()
2696 }),
2697 ..Default::default()
2698 },
2699 ];
2700 vols.extend(build_auth_volumes(auth_method));
2701 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2702 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2703 }
2704 Some(vols)
2705 },
2706 ..Default::default()
2707 }),
2708 },
2709 ..Default::default()
2710 }),
2711 ..Default::default()
2712 };
2713 api.create(&PostParams::default(), &job)
2714 .await
2715 .map(|_| ())
2716 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2717}
2718
2719#[allow(clippy::too_many_arguments)]
2720fn build_safety_upload_operation_doc(
2721 restore: &KanidmRestore,
2722 target: &Kanidm,
2723 backup_id: &str,
2724 manifest_key: &str,
2725 bucket: &str,
2726 prefix: &str,
2727 endpoint: &str,
2728 region: &str,
2729 force_path_style: bool,
2730 insecure: bool,
2731 ca_bundle_path: Option<&str>,
2732 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2733) -> Result<String> {
2734 let enc_mode =
2735 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2736 let enc_key_id = encryption
2737 .and_then(|e| e.key_id.as_ref())
2738 .map(|s| serde_json::Value::String(s.clone()));
2739 let op = UploadOperation {
2740 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2741 bucket: bucket.to_string(),
2742 prefix: prefix.to_string(),
2743 endpoint: endpoint.to_string(),
2744 region: region.to_string(),
2745 force_path_style,
2746 insecure,
2747 ca_bundle_path: ca_bundle_path.map(str::to_string),
2748 backup_id: backup_id.to_string(),
2749 namespace_uid: restore.namespace().unwrap_or_default(),
2753 kanidm_uid: restore.spec.target_ref.uid.clone(),
2754 kanidm_name: restore.spec.target_ref.name.clone(),
2755 domain: target.spec.domain.clone(),
2756 kanidm_version: target
2757 .status
2758 .as_ref()
2759 .and_then(|s| s.version.as_ref())
2760 .map(|v| v.image_tag.clone())
2761 .unwrap_or_default(),
2762 image_digest: None,
2763 consistency: "kanidm-offline".to_string(),
2764 reason: "restore-safety".to_string(),
2765 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2766 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2767 result_path: "/run/kaniop-result/result.json".to_string(),
2768 max_concurrent_parts: 4,
2769 max_retries: 3,
2770 };
2771 let _ = manifest_key;
2772 let doc = OperationDocument {
2773 api_version: OPERATION_DOC_VERSION.to_string(),
2774 kind: "OperationDocument".to_string(),
2775 spec: OperationSpec::Upload(op),
2776 };
2777 serde_json::to_string(&doc)
2778 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2779}
2780
2781async fn validate_backup_ref(
2782 restore: &KanidmRestore,
2783 target: &Kanidm,
2784 ctx: &RestoreContext,
2785) -> Result<Option<DisasterRecoverySource>> {
2786 let backup_ref = restore
2787 .spec
2788 .source
2789 .backup_ref
2790 .as_ref()
2791 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2792 let ns = restore.namespace().unwrap();
2793 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2794 .get(&backup_ref.name)
2795 .await
2796 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2797 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2798 return Err(Error::MissingData(format!(
2799 "KanidmBackup '{}' is not Ready (phase: {:?})",
2800 backup_ref.name,
2801 backup.status.as_ref().map(|s| &s.phase)
2802 )));
2803 }
2804 let source_identity_mismatch = backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid
2805 || backup.spec.kanidm_ref.name != restore.spec.target_ref.name;
2806 let disaster_recovery_source = if source_identity_mismatch {
2807 if !disaster_recovery_requested(restore) {
2808 return Err(Error::MissingData(format!(
2809 "KanidmBackup source identity {}/{} does not match target {}/{}; set {}=true with non-empty {} and {} annotations for an audited disaster-recovery restore",
2810 backup.spec.kanidm_ref.name,
2811 backup.spec.kanidm_ref.uid,
2812 restore.spec.target_ref.name,
2813 restore.spec.target_ref.uid,
2814 DISASTER_RECOVERY_ANNOTATION,
2815 BREAK_GLASS_REASON_ANNOTATION,
2816 BREAK_GLASS_APPROVED_BY_ANNOTATION,
2817 )));
2818 }
2819 validate_disaster_recovery_approval(restore)?;
2820 Some(DisasterRecoverySource {
2821 name: backup.spec.kanidm_ref.name.clone(),
2822 uid: backup.spec.kanidm_ref.uid.clone(),
2823 })
2824 } else {
2825 None
2826 };
2827 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2828 .get(&backup.spec.repository_ref.name)
2829 .await
2830 .map_err(|e| {
2831 Error::kube_error(
2832 "get",
2833 "KanidmBackupRepository",
2834 &ns,
2835 &backup.spec.repository_ref.name,
2836 e,
2837 )
2838 })?;
2839 if !has_accepted_condition(
2840 &repo
2841 .status
2842 .as_ref()
2843 .map(|s| &s.conditions)
2844 .cloned()
2845 .unwrap_or_default(),
2846 ) {
2847 return Err(Error::MissingData(format!(
2848 "KanidmBackupRepository '{}' configuration has not been accepted",
2849 backup.spec.repository_ref.name
2850 )));
2851 }
2852 validate_backup_compatibility(restore, target, &backup)?;
2853 Ok(disaster_recovery_source)
2854}
2855
2856fn has_accepted_condition(conditions: &[Condition]) -> bool {
2857 conditions
2858 .iter()
2859 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2860}
2861
2862fn validate_backup_compatibility(
2863 restore: &KanidmRestore,
2864 target: &Kanidm,
2865 backup: &KanidmBackup,
2866) -> Result<()> {
2867 let status = backup.status.as_ref();
2868 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2869 let target_version = target
2870 .status
2871 .as_ref()
2872 .and_then(|s| s.version.as_ref())
2873 .map(|v| v.image_tag.as_str());
2874 if let Some(tv) = target_version {
2875 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2876 return Err(Error::MissingData(format!(
2877 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2878 )));
2879 }
2880 }
2881 }
2882 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2883 if !backup_digest.is_empty()
2884 && restore.spec.restore_image.contains('@')
2885 && !restore.spec.restore_image.contains(backup_digest)
2886 {
2887 return Err(Error::MissingData(format!(
2888 "restore image digest does not match backup image digest '{backup_digest}'"
2889 )));
2890 }
2891 }
2892 Ok(())
2893}
2894
2895#[allow(clippy::too_many_arguments)]
2896fn build_download_operation_doc(
2897 target: &Kanidm,
2898 manifest_key: &str,
2899 expected_backup_id: &str,
2900 expected_source_uid: &str,
2901 bucket: &str,
2902 prefix: &str,
2903 endpoint: &str,
2904 region: &str,
2905 force_path_style: bool,
2906 insecure: bool,
2907 ca_bundle_path: Option<&str>,
2908 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2909) -> String {
2910 let enc_mode =
2911 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2912 let enc_key_id = encryption
2913 .and_then(|e| e.key_id.as_ref())
2914 .map(|s| serde_json::Value::String(s.clone()));
2915 serde_json::json!({
2916 "apiVersion": "backup.kaniop.rs/v1alpha1",
2917 "kind": "OperationDocument",
2918 "operation": "download",
2919 "manifestKey": manifest_key,
2920 "bucket": bucket,
2921 "prefix": prefix,
2922 "endpoint": endpoint,
2923 "region": region,
2924 "forcePathStyle": force_path_style,
2925 "insecure": insecure,
2926 "caBundlePath": ca_bundle_path,
2927 "expectedBackupId": expected_backup_id,
2928 "expectedKanidmUid": expected_source_uid,
2929 "expectedDomain": target.spec.domain,
2930 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2931 "resultPath": "/run/kaniop-result/result.json",
2932 "maxRetries": 3,
2933 "encryptionMode": enc_mode,
2934 "encryptionKeyId": enc_key_id,
2935 })
2936 .to_string()
2937}
2938
2939async fn ensure_operation_configmap(
2940 restore: &KanidmRestore,
2941 name: &str,
2942 data: &str,
2943 ns: &str,
2944 ctx: &RestoreContext,
2945) -> Result<()> {
2946 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2947 if api
2948 .get_opt(name)
2949 .await
2950 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2951 .is_some()
2952 {
2953 return Ok(());
2954 }
2955 let cm = ConfigMap {
2956 metadata: ObjectMeta {
2957 name: Some(name.to_string()),
2958 namespace: Some(ns.to_string()),
2959 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2960 ..Default::default()
2961 },
2962 data: Some(std::collections::BTreeMap::from([(
2963 "operation.json".to_string(),
2964 data.to_string(),
2965 )])),
2966 ..Default::default()
2967 };
2968 api.create(&PostParams::default(), &cm)
2969 .await
2970 .map(|_| ())
2971 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2972}
2973
2974#[cfg(test)]
2975mod tests {
2976 use super::{
2977 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2978 CONDITION_DISASTER_RECOVERY, CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE,
2979 DISASTER_RECOVERY_ANNOTATION,
2980 KanidmRestore, KanidmRestorePhase,
2981 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2982 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2983 disaster_recovery_requested, hardened_security_context, has_accepted_condition,
2984 is_usage_lock_persisted,
2985 is_remote_source, kanidm_job_security_context, mutable_image, requires_safety_backup,
2986 restore_job_name, safe_basename, validate_disaster_recovery_approval,
2987 validate_safety_backup_config, validate_source, verify_job_name,
2988 };
2989 use kaniop_backup_core::crd::KanidmBackupRepositoryStatus;
2990 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2991 use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
2992 use k8s_openapi::jiff::Timestamp;
2993 use kube::api::ObjectMeta as ApiObjectMeta;
2994 use std::collections::BTreeMap;
2995
2996 fn make_restore(
2997 source: KanidmRestoreSource,
2998 safety: Option<SafetyBackupConfig>,
2999 ) -> KanidmRestore {
3000 KanidmRestore {
3001 metadata: ApiObjectMeta {
3002 name: Some("test-restore".to_string()),
3003 namespace: Some("default".to_string()),
3004 uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
3005 ..Default::default()
3006 },
3007 spec: KanidmRestoreSpec {
3008 target_ref: KanidmRestoreTargetRef {
3009 name: "corp-idm".to_string(),
3010 uid: "test-uid-123".to_string(),
3011 },
3012 source,
3013 restore_image: "kanidm/server@sha256:abc".to_string(),
3014 safety_backup: safety,
3015 },
3016 status: None,
3017 }
3018 }
3019
3020 #[test]
3021 fn safety_backup_waits_for_persisted_repository_lock() {
3022 assert!(!is_usage_lock_persisted(None));
3023 let mut status = KanidmBackupRepositoryStatus {
3024 observed_generation: Some(1),
3025 usage_started: Some(false),
3026 ..Default::default()
3027 };
3028 assert!(!is_usage_lock_persisted(Some(&status)));
3029 status.usage_started = Some(true);
3030 assert!(is_usage_lock_persisted(Some(&status)));
3031 status.usage_started = None;
3033 assert!(is_usage_lock_persisted(Some(&status)));
3034 status.observed_generation = None;
3035 assert!(!is_usage_lock_persisted(Some(&status)));
3036 }
3037
3038 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
3039 super::super::crd::Kanidm {
3040 status: Some(super::super::crd::KanidmStatus {
3041 available_replicas: 1,
3042 replicas: 1,
3043 unavailable_replicas: 0,
3044 updated_replicas: 1,
3045 replica_statuses: vec![],
3046 replica_column: "1/1".to_string(),
3047 secret_name: None,
3048 version: Some(super::super::crd::KanidmVersionStatus {
3049 image_tag: image_tag.to_string(),
3050 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
3051 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
3052 }),
3053 domain_appearance_image: None,
3054 mail_sender: None,
3055 conditions: None,
3056 }),
3057 ..Default::default()
3058 }
3059 }
3060
3061 #[test]
3062 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
3063 assert!(!KanidmRestoreStatus::default().database_mutation_started);
3064 }
3065
3066 #[test]
3067 fn completed_restore_sets_ready_condition() {
3068 let mut status = KanidmRestoreStatus {
3069 phase: KanidmRestorePhase::Completed,
3070 ..Default::default()
3071 };
3072 super::update_restore_conditions(&mut status, Some(7));
3073 assert!(status.conditions.iter().any(|condition| {
3074 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
3075 }));
3076 assert!(!status.conditions.iter().any(|condition| {
3077 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3078 }));
3079 }
3080
3081 #[test]
3082 fn disaster_recovery_condition_survives_phase_updates() {
3083 let dr_condition = Condition {
3084 type_: CONDITION_DISASTER_RECOVERY.to_string(),
3085 status: CONDITION_TRUE.to_string(),
3086 reason: "ApprovedSourceIdentityMismatch".to_string(),
3087 message: "approved DR".to_string(),
3088 last_transition_time: Time(Timestamp::now()),
3089 observed_generation: Some(7),
3090 };
3091 let mut status = KanidmRestoreStatus {
3092 phase: KanidmRestorePhase::Quiescing,
3093 conditions: vec![dr_condition.clone()],
3094 ..Default::default()
3095 };
3096
3097 super::update_restore_conditions(&mut status, Some(8));
3098
3099 assert!(status.conditions.iter().any(|condition| condition == &dr_condition));
3100 }
3101
3102 #[test]
3103 fn failed_restore_sets_failed_condition() {
3104 let mut status = KanidmRestoreStatus {
3105 phase: KanidmRestorePhase::Failed,
3106 message: Some("verification failed".to_string()),
3107 ..Default::default()
3108 };
3109 super::update_restore_conditions(&mut status, Some(8));
3110 assert!(status.conditions.iter().any(|condition| {
3111 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
3112 }));
3113 }
3114
3115 #[test]
3116 fn restore_filename_is_confined_to_backup_directory() {
3117 assert!(safe_basename("backup.json.gz"));
3118 assert!(!safe_basename("../backup.json.gz"));
3119 assert!(!safe_basename("a/b"));
3120 assert!(!safe_basename(""));
3121 }
3122
3123 #[test]
3124 fn restore_image_rejects_latest_and_unpinned_names() {
3125 assert!(mutable_image("kanidm/server:latest"));
3126 assert!(mutable_image("kanidm/server"));
3127 assert!(!mutable_image("kanidm/server:1.10.0"));
3128 assert!(!mutable_image("kanidm/server:1.11.2@sha256:serverdigest"));
3129 assert!(!mutable_image("kanidm/server@sha256:abc"));
3130 assert!(mutable_image("registry.example.com:5000/kanidm/server"));
3131 }
3132
3133 #[test]
3134 fn source_validation_rejects_both_local_and_backup_ref() {
3135 let source = KanidmRestoreSource {
3136 local: Some(super::KanidmRestoreLocalSource {
3137 file_name: "backup.json".to_string(),
3138 }),
3139 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3140 name: "some-backup".to_string(),
3141 }),
3142 };
3143 let restore = make_restore(source, None);
3144 assert!(validate_source(&restore).is_err());
3145 }
3146
3147 #[test]
3148 fn source_validation_rejects_empty_source() {
3149 let source = KanidmRestoreSource {
3150 local: None,
3151 backup_ref: None,
3152 };
3153 let restore = make_restore(source, None);
3154 assert!(validate_source(&restore).is_err());
3155 }
3156
3157 #[test]
3158 fn source_validation_accepts_local_source() {
3159 let source = KanidmRestoreSource {
3160 local: Some(super::KanidmRestoreLocalSource {
3161 file_name: "backup.json".to_string(),
3162 }),
3163 backup_ref: None,
3164 };
3165 let restore = make_restore(source, None);
3166 assert!(validate_source(&restore).is_ok());
3167 }
3168
3169 #[test]
3170 fn source_validation_accepts_backup_ref_source() {
3171 let source = KanidmRestoreSource {
3172 local: None,
3173 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3174 name: "corp-idm-019c7c76".to_string(),
3175 }),
3176 };
3177 let restore = make_restore(source, None);
3178 assert!(validate_source(&restore).is_ok());
3179 }
3180
3181 #[test]
3182 fn source_validation_rejects_empty_backup_ref_name() {
3183 let source = KanidmRestoreSource {
3184 local: None,
3185 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3186 name: "".to_string(),
3187 }),
3188 };
3189 let restore = make_restore(source, None);
3190 assert!(validate_source(&restore).is_err());
3191 }
3192
3193 #[test]
3194 fn is_remote_source_true_for_backup_ref() {
3195 let source = KanidmRestoreSource {
3196 local: None,
3197 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3198 name: "backup-1".to_string(),
3199 }),
3200 };
3201 let restore = make_restore(source, None);
3202 assert!(is_remote_source(&restore));
3203 }
3204
3205 #[test]
3206 fn is_remote_source_false_for_local() {
3207 let source = KanidmRestoreSource {
3208 local: Some(super::KanidmRestoreLocalSource {
3209 file_name: "backup.json".to_string(),
3210 }),
3211 backup_ref: None,
3212 };
3213 let restore = make_restore(source, None);
3214 assert!(!is_remote_source(&restore));
3215 }
3216
3217 #[test]
3218 fn requires_safety_backup_default_true() {
3219 let source = KanidmRestoreSource {
3220 local: None,
3221 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3222 name: "backup-1".to_string(),
3223 }),
3224 };
3225 let restore = make_restore(source, None);
3226 assert!(requires_safety_backup(&restore));
3227 }
3228
3229 #[test]
3230 fn requires_safety_backup_false_when_skip() {
3231 let source = KanidmRestoreSource {
3232 local: None,
3233 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3234 name: "backup-1".to_string(),
3235 }),
3236 };
3237 let safety = SafetyBackupConfig {
3238 repository_ref: Some(SafetyBackupRepositoryRef {
3239 name: "offsite".to_string(),
3240 }),
3241 skip: true,
3242 };
3243 let restore = make_restore(source, Some(safety));
3244 assert!(!requires_safety_backup(&restore));
3245 }
3246
3247 #[test]
3248 fn break_glass_validation_rejects_skip_without_reason() {
3249 let source = KanidmRestoreSource {
3250 local: None,
3251 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3252 name: "backup-1".to_string(),
3253 }),
3254 };
3255 let safety = SafetyBackupConfig {
3256 repository_ref: Some(SafetyBackupRepositoryRef {
3257 name: "offsite".to_string(),
3258 }),
3259 skip: true,
3260 };
3261 let mut restore = make_restore(source, Some(safety));
3262 restore.metadata.annotations = Some(BTreeMap::from([(
3263 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3264 "commander@example.com".to_string(),
3265 )]));
3266 assert!(validate_safety_backup_config(&restore).is_err());
3267 }
3268
3269 #[test]
3270 fn break_glass_validation_rejects_skip_without_approver() {
3271 let source = KanidmRestoreSource {
3272 local: None,
3273 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3274 name: "backup-1".to_string(),
3275 }),
3276 };
3277 let safety = SafetyBackupConfig {
3278 repository_ref: Some(SafetyBackupRepositoryRef {
3279 name: "offsite".to_string(),
3280 }),
3281 skip: true,
3282 };
3283 let mut restore = make_restore(source, Some(safety));
3284 restore.metadata.annotations = Some(BTreeMap::from([(
3285 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3286 "PVC is unreadable".to_string(),
3287 )]));
3288 assert!(validate_safety_backup_config(&restore).is_err());
3289 }
3290
3291 #[test]
3292 fn break_glass_validation_accepts_valid_annotations() {
3293 let source = KanidmRestoreSource {
3294 local: None,
3295 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3296 name: "backup-1".to_string(),
3297 }),
3298 };
3299 let safety = SafetyBackupConfig {
3300 repository_ref: Some(SafetyBackupRepositoryRef {
3301 name: "offsite".to_string(),
3302 }),
3303 skip: true,
3304 };
3305 let mut restore = make_restore(source, Some(safety));
3306 restore.metadata.annotations = Some(BTreeMap::from([
3307 (
3308 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3309 "PVC is unreadable".to_string(),
3310 ),
3311 (
3312 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3313 "commander@example.com".to_string(),
3314 ),
3315 ]));
3316 assert!(validate_safety_backup_config(&restore).is_ok());
3317 }
3318
3319 #[test]
3320 fn disaster_recovery_requires_explicit_approval_annotations() {
3321 let source = KanidmRestoreSource {
3322 local: None,
3323 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3324 name: "backup-1".to_string(),
3325 }),
3326 };
3327 let mut restore = make_restore(source, None);
3328 restore.metadata.annotations = Some(BTreeMap::from([(
3329 DISASTER_RECOVERY_ANNOTATION.to_string(),
3330 "true".to_string(),
3331 )]));
3332 assert!(disaster_recovery_requested(&restore));
3333 assert!(validate_disaster_recovery_approval(&restore).is_err());
3334 }
3335
3336 #[test]
3337 fn disaster_recovery_accepts_reason_and_approver() {
3338 let source = KanidmRestoreSource {
3339 local: None,
3340 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3341 name: "backup-1".to_string(),
3342 }),
3343 };
3344 let mut restore = make_restore(source, None);
3345 restore.metadata.annotations = Some(BTreeMap::from([
3346 (
3347 DISASTER_RECOVERY_ANNOTATION.to_string(),
3348 "true".to_string(),
3349 ),
3350 (
3351 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3352 "original cluster lost".to_string(),
3353 ),
3354 (
3355 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3356 "incident-commander".to_string(),
3357 ),
3358 ]));
3359 assert!(disaster_recovery_requested(&restore));
3360 assert!(validate_disaster_recovery_approval(&restore).is_ok());
3361 }
3362
3363 #[test]
3364 fn remote_source_requires_repository_ref_when_not_skipping() {
3365 let source = KanidmRestoreSource {
3366 local: None,
3367 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3368 name: "backup-1".to_string(),
3369 }),
3370 };
3371 let safety = SafetyBackupConfig {
3372 repository_ref: None,
3373 skip: false,
3374 };
3375 let restore = make_restore(source, Some(safety));
3376 assert!(validate_safety_backup_config(&restore).is_err());
3377 }
3378
3379 #[test]
3380 fn remote_source_accepts_repository_ref() {
3381 let source = KanidmRestoreSource {
3382 local: None,
3383 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3384 name: "backup-1".to_string(),
3385 }),
3386 };
3387 let safety = SafetyBackupConfig {
3388 repository_ref: Some(SafetyBackupRepositoryRef {
3389 name: "offsite".to_string(),
3390 }),
3391 skip: false,
3392 };
3393 let restore = make_restore(source, Some(safety));
3394 assert!(validate_safety_backup_config(&restore).is_ok());
3395 }
3396
3397 #[test]
3398 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3399 let phases = vec![
3400 KanidmRestorePhase::Pending,
3401 KanidmRestorePhase::Validating,
3402 KanidmRestorePhase::Quiescing,
3403 KanidmRestorePhase::SafetyBackup,
3404 KanidmRestorePhase::PreparingSource,
3405 KanidmRestorePhase::RestoringPrimary,
3406 KanidmRestorePhase::Verifying,
3407 KanidmRestorePhase::RebuildingReplicas,
3408 KanidmRestorePhase::Resuming,
3409 KanidmRestorePhase::Completed,
3410 ];
3411 assert_eq!(phases.len(), 10);
3412 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3413 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3414 }
3415
3416 #[test]
3417 fn status_default_has_empty_phase_timestamps() {
3418 let status = KanidmRestoreStatus::default();
3419 assert!(status.phase_timestamps.is_empty());
3420 }
3421
3422 #[test]
3423 fn status_default_has_empty_original_replicas() {
3424 let status = KanidmRestoreStatus::default();
3425 assert!(status.original_replicas.is_empty());
3426 }
3427
3428 #[test]
3429 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3430 let mut target = super::super::crd::Kanidm::default();
3431 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3432 run_as_non_root: Some(false),
3433 run_as_user: Some(389),
3434 run_as_group: Some(389),
3435 fs_group: Some(389),
3436 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3437 supplemental_groups: Some(vec![390, 391]),
3438 ..Default::default()
3439 });
3440
3441 let ctx = super::source_job_pod_security_context(&target);
3442
3443 assert_eq!(ctx.run_as_non_root, Some(false));
3444 assert_eq!(ctx.run_as_user, Some(389));
3445 assert_eq!(ctx.run_as_group, Some(389));
3446 assert_eq!(ctx.fs_group, Some(389));
3447 assert_eq!(
3448 ctx.fs_group_change_policy.as_deref(),
3449 Some("OnRootMismatch")
3450 );
3451 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3452 assert_eq!(
3453 ctx.seccomp_profile
3454 .as_ref()
3455 .map(|profile| profile.type_.as_str()),
3456 Some("RuntimeDefault")
3457 );
3458 }
3459
3460 #[test]
3461 fn hardened_security_context_drops_all_capabilities() {
3462 let ctx = hardened_security_context();
3463 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3464 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3465 assert_eq!(ctx.run_as_non_root, Some(true));
3466 let caps = ctx.capabilities.unwrap();
3467 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3468 assert_eq!(
3469 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3470 Some("RuntimeDefault")
3471 );
3472 }
3473
3474 #[test]
3475 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3476 let ctx = kanidm_job_security_context();
3477 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3478 assert!(ctx.run_as_non_root.is_none());
3479 assert!(ctx.read_only_root_filesystem.is_none());
3480 let caps = ctx.capabilities.unwrap();
3481 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3482 assert_eq!(
3483 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3484 Some("RuntimeDefault")
3485 );
3486 }
3487
3488 #[test]
3489 fn safety_backup_job_name_is_deterministic() {
3490 let source = KanidmRestoreSource {
3491 local: Some(super::KanidmRestoreLocalSource {
3492 file_name: "backup.json".to_string(),
3493 }),
3494 backup_ref: None,
3495 };
3496 let restore = make_restore(source, None);
3497 let name1 = super::safety_backup_job_name(&restore);
3498 let name2 = super::safety_backup_job_name(&restore);
3499 assert_eq!(name1, name2);
3500 assert_eq!(name1, "test-restore-safety-backup");
3501 }
3502
3503 #[test]
3504 fn source_prep_job_name_is_deterministic() {
3505 let source = KanidmRestoreSource {
3506 local: None,
3507 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3508 name: "backup-1".to_string(),
3509 }),
3510 };
3511 let restore = make_restore(source, None);
3512 let name = super::source_prep_job_name(&restore);
3513 assert_eq!(name, "test-restore-source-prep");
3514 }
3515
3516 #[test]
3517 fn break_glass_condition_is_recorded_in_status() {
3518 let mut status = KanidmRestoreStatus::default();
3519 let condition = super::restore_condition(
3520 &status.conditions,
3521 CONDITION_BREAK_GLASS,
3522 CONDITION_TRUE,
3523 "BreakGlassOverride",
3524 "Safety backup skipped",
3525 Some(1),
3526 );
3527 status.conditions.push(condition);
3528 assert!(status.conditions.iter().any(|c| {
3529 c.type_ == CONDITION_BREAK_GLASS
3530 && c.status == CONDITION_TRUE
3531 && c.reason == "BreakGlassOverride"
3532 }));
3533 }
3534
3535 #[test]
3536 fn replica_count_entry_serialization() {
3537 let entry = ReplicaCountEntry {
3538 group: "primary".to_string(),
3539 replicas: 3,
3540 };
3541 let json = serde_json::to_string(&entry).unwrap();
3542 assert!(json.contains("\"group\":\"primary\""));
3543 assert!(json.contains("\"replicas\":3"));
3544 }
3545
3546 #[test]
3547 fn local_source_requires_repository_ref_when_not_skipping() {
3548 let source = KanidmRestoreSource {
3549 local: Some(super::KanidmRestoreLocalSource {
3550 file_name: "backup.json".to_string(),
3551 }),
3552 backup_ref: None,
3553 };
3554 let safety = SafetyBackupConfig {
3555 repository_ref: None,
3556 skip: false,
3557 };
3558 let restore = make_restore(source, Some(safety));
3559 assert!(validate_safety_backup_config(&restore).is_err());
3560 }
3561
3562 #[test]
3563 fn local_source_accepts_repository_ref_when_not_skipping() {
3564 let source = KanidmRestoreSource {
3565 local: Some(super::KanidmRestoreLocalSource {
3566 file_name: "backup.json".to_string(),
3567 }),
3568 backup_ref: None,
3569 };
3570 let safety = SafetyBackupConfig {
3571 repository_ref: Some(SafetyBackupRepositoryRef {
3572 name: "offsite".to_string(),
3573 }),
3574 skip: false,
3575 };
3576 let restore = make_restore(source, Some(safety));
3577 assert!(validate_safety_backup_config(&restore).is_ok());
3578 }
3579
3580 #[test]
3581 fn skip_without_break_glass_annotations_is_rejected() {
3582 let source = KanidmRestoreSource {
3583 local: Some(super::KanidmRestoreLocalSource {
3584 file_name: "backup.json".to_string(),
3585 }),
3586 backup_ref: None,
3587 };
3588 let safety = SafetyBackupConfig {
3589 repository_ref: None,
3590 skip: true,
3591 };
3592 let restore = make_restore(source, Some(safety));
3593 assert!(validate_safety_backup_config(&restore).is_err());
3594 }
3595
3596 #[test]
3597 fn has_accepted_condition_true() {
3598 let conditions = vec![Condition {
3599 type_: "Ready".to_string(),
3600 status: "True".to_string(),
3601 reason: "Accepted".to_string(),
3602 message: String::new(),
3603 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3604 k8s_openapi::jiff::Timestamp::now(),
3605 ),
3606 observed_generation: None,
3607 }];
3608 assert!(has_accepted_condition(&conditions));
3609 }
3610
3611 #[test]
3612 fn has_accepted_condition_false_when_not_accepted_reason() {
3613 let conditions = vec![Condition {
3614 type_: "Ready".to_string(),
3615 status: "True".to_string(),
3616 reason: "SomeOtherReason".to_string(),
3617 message: String::new(),
3618 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3619 k8s_openapi::jiff::Timestamp::now(),
3620 ),
3621 observed_generation: None,
3622 }];
3623 assert!(!has_accepted_condition(&conditions));
3624 }
3625
3626 #[test]
3627 fn has_accepted_condition_false_when_status_false() {
3628 let conditions = vec![Condition {
3629 type_: "Ready".to_string(),
3630 status: "False".to_string(),
3631 reason: "Accepted".to_string(),
3632 message: String::new(),
3633 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3634 k8s_openapi::jiff::Timestamp::now(),
3635 ),
3636 observed_generation: None,
3637 }];
3638 assert!(!has_accepted_condition(&conditions));
3639 }
3640
3641 #[test]
3642 fn has_accepted_condition_empty() {
3643 assert!(!has_accepted_condition(&[]));
3644 }
3645
3646 #[test]
3647 fn safety_backup_upload_operation_doc_is_valid() {
3648 let source = KanidmRestoreSource {
3649 local: None,
3650 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3651 name: "backup-1".to_string(),
3652 }),
3653 };
3654 let mut restore = make_restore(source, None);
3655 restore.metadata.namespace = Some("test-ns".to_string());
3656 let target = super::super::crd::Kanidm::default();
3657 let doc_str = super::build_safety_upload_operation_doc(
3658 &restore,
3659 &target,
3660 "test-backup-id",
3661 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3662 "my-bucket",
3663 "prod",
3664 "https://s3.example.com",
3665 "us-east-1",
3666 false,
3667 false,
3668 None,
3669 None,
3670 )
3671 .unwrap();
3672 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3673 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3674 assert_eq!(parsed["kind"], "OperationDocument");
3675 assert_eq!(parsed["operation"], "upload");
3676 assert_eq!(parsed["bucket"], "my-bucket");
3677 assert_eq!(parsed["prefix"], "prod");
3678 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3679 assert_eq!(parsed["region"], "us-east-1");
3680 assert_eq!(parsed["backupId"], "test-backup-id");
3681 assert_eq!(parsed["consistency"], "kanidm-offline");
3682 assert_eq!(parsed["reason"], "restore-safety");
3683 assert_eq!(
3684 parsed["payloadPath"],
3685 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3686 );
3687 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3688 }
3689
3690 #[test]
3691 fn download_operation_doc_uses_real_repository_data() {
3692 let source = KanidmRestoreSource {
3693 local: None,
3694 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3695 name: "backup-1".to_string(),
3696 }),
3697 };
3698 let _restore = make_restore(source, None);
3699 let target = super::super::crd::Kanidm::default();
3700 let doc_str = super::build_download_operation_doc(
3701 &target,
3702 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3703 "019c7c76-f423-7a12-8f41-2bea7588a303",
3704 "source-uid-456",
3705 "real-bucket",
3706 "real-prefix",
3707 "https://real-endpoint.com",
3708 "eu-west-1",
3709 true,
3710 false,
3711 None,
3712 None,
3713 );
3714 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3715 assert_eq!(parsed["bucket"], "real-bucket");
3716 assert_eq!(parsed["prefix"], "real-prefix");
3717 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3718 assert_eq!(parsed["region"], "eu-west-1");
3719 assert_eq!(parsed["forcePathStyle"], true);
3720 assert_eq!(
3721 parsed["manifestKey"],
3722 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3723 );
3724 assert_eq!(
3725 parsed["expectedBackupId"],
3726 "019c7c76-f423-7a12-8f41-2bea7588a303"
3727 );
3728 assert_eq!(parsed["expectedKanidmUid"], "source-uid-456");
3729 }
3730
3731 #[test]
3732 fn safety_backup_ref_is_set_after_verification() {
3733 let status = KanidmRestoreStatus {
3734 phase: KanidmRestorePhase::SafetyBackup,
3735 safety_backup_job_name: Some("test-safety-backup".to_string()),
3736 safety_backup_ref: Some("safety-test-restore".to_string()),
3737 safety_backup_expected_backup_id: Some(
3738 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3739 ),
3740 safety_backup_manifest_key: Some(
3741 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3742 ),
3743 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3744 ..Default::default()
3745 };
3746 assert!(status.safety_backup_ref.is_some());
3747 assert!(
3748 status
3749 .safety_backup_ref
3750 .as_ref()
3751 .unwrap()
3752 .starts_with("safety-")
3753 );
3754 assert!(status.safety_backup_expected_backup_id.is_some());
3755 assert!(status.safety_backup_manifest_key.is_some());
3756 assert!(status.safety_backup_payload_sha256.is_some());
3757 }
3758
3759 #[test]
3760 fn database_mutation_not_started_without_safety_backup_ref() {
3761 let status = KanidmRestoreStatus {
3762 phase: KanidmRestorePhase::PreparingSource,
3763 safety_backup_ref: None,
3764 database_mutation_started: false,
3765 ..Default::default()
3766 };
3767 assert!(!status.database_mutation_started);
3768 assert!(status.safety_backup_ref.is_none());
3769 }
3770
3771 #[test]
3772 fn pre_boundary_failure_does_not_set_mutation_started() {
3773 let status = KanidmRestoreStatus {
3774 phase: KanidmRestorePhase::Failed,
3775 message: Some(
3776 "safety backup job failed; target restored to original state".to_string(),
3777 ),
3778 database_mutation_started: false,
3779 ..Default::default()
3780 };
3781 assert!(!status.database_mutation_started);
3782 }
3783
3784 #[test]
3785 fn shared_volume_constants_are_distinct() {
3786 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3787 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3788 }
3789
3790 #[test]
3791 fn validate_backup_compatibility_matching_versions_passes() {
3792 let source = KanidmRestoreSource {
3793 local: None,
3794 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3795 name: "backup-1".to_string(),
3796 }),
3797 };
3798 let restore = make_restore(source, None);
3799 let target = kanidm_with_version("1.10.4");
3800 let backup = kaniop_backup_core::crd::KanidmBackup {
3801 metadata: kube::api::ObjectMeta::default(),
3802 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3803 backup_id: "id".to_string(),
3804 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3805 name: "k".to_string(),
3806 uid: "uid".to_string(),
3807 },
3808 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3809 name: "repo".to_string(),
3810 },
3811 manifest_key: "key".to_string(),
3812 },
3813 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3814 kanidm_version: Some("1.10.4".to_string()),
3815 ..Default::default()
3816 }),
3817 };
3818 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3819 }
3820
3821 #[test]
3822 fn validate_backup_compatibility_version_mismatch_fails() {
3823 let source = KanidmRestoreSource {
3824 local: None,
3825 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3826 name: "backup-1".to_string(),
3827 }),
3828 };
3829 let restore = make_restore(source, None);
3830 let target = kanidm_with_version("1.11.0");
3831 let backup = kaniop_backup_core::crd::KanidmBackup {
3832 metadata: kube::api::ObjectMeta::default(),
3833 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3834 backup_id: "id".to_string(),
3835 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3836 name: "k".to_string(),
3837 uid: "uid".to_string(),
3838 },
3839 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3840 name: "repo".to_string(),
3841 },
3842 manifest_key: "key".to_string(),
3843 },
3844 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3845 kanidm_version: Some("1.10.4".to_string()),
3846 ..Default::default()
3847 }),
3848 };
3849 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3850 }
3851
3852 #[test]
3853 fn validate_backup_compatibility_no_backup_version_passes() {
3854 let source = KanidmRestoreSource {
3855 local: None,
3856 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3857 name: "backup-1".to_string(),
3858 }),
3859 };
3860 let restore = make_restore(source, None);
3861 let target = super::super::crd::Kanidm::default();
3862 let backup = kaniop_backup_core::crd::KanidmBackup {
3863 metadata: kube::api::ObjectMeta::default(),
3864 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3865 backup_id: "id".to_string(),
3866 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3867 name: "k".to_string(),
3868 uid: "uid".to_string(),
3869 },
3870 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3871 name: "repo".to_string(),
3872 },
3873 manifest_key: "key".to_string(),
3874 },
3875 status: None,
3876 };
3877 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3878 }
3879
3880 #[test]
3881 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3882 let source = KanidmRestoreSource {
3883 local: None,
3884 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3885 name: "backup-1".to_string(),
3886 }),
3887 };
3888 let mut restore = make_restore(source, None);
3889 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3890 let target = super::super::crd::Kanidm::default();
3891 let backup = kaniop_backup_core::crd::KanidmBackup {
3892 metadata: kube::api::ObjectMeta::default(),
3893 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3894 backup_id: "id".to_string(),
3895 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3896 name: "k".to_string(),
3897 uid: "uid".to_string(),
3898 },
3899 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3900 name: "repo".to_string(),
3901 },
3902 manifest_key: "key".to_string(),
3903 },
3904 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3905 image_digest: Some("sha256:originaldigest".to_string()),
3906 ..Default::default()
3907 }),
3908 };
3909 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3910 }
3911
3912 #[test]
3913 fn validate_backup_compatibility_image_digest_match_passes() {
3914 let source = KanidmRestoreSource {
3915 local: None,
3916 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3917 name: "backup-1".to_string(),
3918 }),
3919 };
3920 let mut restore = make_restore(source, None);
3921 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3922 let target = super::super::crd::Kanidm::default();
3923 let backup = kaniop_backup_core::crd::KanidmBackup {
3924 metadata: kube::api::ObjectMeta::default(),
3925 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3926 backup_id: "id".to_string(),
3927 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3928 name: "k".to_string(),
3929 uid: "uid".to_string(),
3930 },
3931 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3932 name: "repo".to_string(),
3933 },
3934 manifest_key: "key".to_string(),
3935 },
3936 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3937 image_digest: Some("sha256:abc123".to_string()),
3938 ..Default::default()
3939 }),
3940 };
3941 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3942 }
3943
3944 #[test]
3945 fn safety_backup_id_is_deterministic() {
3946 let source = KanidmRestoreSource {
3947 local: None,
3948 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3949 name: "backup-1".to_string(),
3950 }),
3951 };
3952 let restore = make_restore(source, None);
3953 let id1 = super::compute_safety_backup_id(&restore).unwrap();
3954 let id2 = super::compute_safety_backup_id(&restore).unwrap();
3955 assert_eq!(id1, id2);
3956 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3957 }
3958
3959 #[test]
3960 fn safety_manifest_key_is_deterministic() {
3961 let source = KanidmRestoreSource {
3962 local: None,
3963 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3964 name: "backup-1".to_string(),
3965 }),
3966 };
3967 let restore = make_restore(source, None);
3968 let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3969 let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3970 assert_eq!(key1, key2);
3971 assert!(key1.contains("manifest.json"));
3972 assert!(key1.starts_with("v1/tenants/"));
3973 assert!(key1.contains("tenants/default/"), "unexpected key {key1}");
3976 assert!(
3977 !key1.contains("a1b2c3d4"),
3978 "safety manifest key must not key the tenant segment by object UID: {key1}"
3979 );
3980 }
3981
3982 #[test]
3990 fn safety_manifest_key_matches_deployed_keyspace_layout() {
3991 let source = KanidmRestoreSource {
3992 local: None,
3993 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3994 name: "backup-1".to_string(),
3995 }),
3996 };
3997 let restore = make_restore(source, None);
3998 let expected_backup_id = "8dc66173-2443-5117-b56e-8a4c60b821a3";
3999 assert_eq!(
4000 super::compute_safety_backup_id(&restore).unwrap(),
4001 expected_backup_id
4002 );
4003 assert_eq!(
4004 super::compute_safety_manifest_key(&restore).unwrap(),
4005 format!(
4006 "v1/tenants/default/clusters/test-uid-123/backups/{expected_backup_id}/manifest.json"
4007 )
4008 );
4009 }
4010
4011 #[test]
4012 fn safety_backup_id_survives_restart() {
4013 let source = KanidmRestoreSource {
4014 local: None,
4015 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4016 name: "backup-1".to_string(),
4017 }),
4018 };
4019 let mut restore = make_restore(source, None);
4020 let original_id = super::compute_safety_backup_id(&restore).unwrap();
4021 assert_eq!(original_id, "8dc66173-2443-5117-b56e-8a4c60b821a3");
4025 restore.status = Some(KanidmRestoreStatus {
4026 phase: KanidmRestorePhase::SafetyBackup,
4027 safety_backup_expected_backup_id: Some(original_id.clone()),
4028 ..Default::default()
4029 });
4030 let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
4031 assert_eq!(original_id, recomputed_id);
4032 }
4033
4034 #[test]
4035 fn safety_backup_id_derived_from_uid_not_name() {
4036 let source = KanidmRestoreSource {
4037 local: None,
4038 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4039 name: "backup-1".to_string(),
4040 }),
4041 };
4042 let restore1 = make_restore(source.clone(), None);
4043 let mut restore2 = make_restore(source, None);
4044 restore2.metadata.name = Some("different-name-same-uid".to_string());
4045 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
4046 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
4047 assert_eq!(id1, id2);
4048 }
4049
4050 #[test]
4051 fn safety_backup_id_differs_for_different_uids() {
4052 let source = KanidmRestoreSource {
4053 local: None,
4054 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4055 name: "backup-1".to_string(),
4056 }),
4057 };
4058 let restore1 = make_restore(source.clone(), None);
4059 let mut restore2 = make_restore(source, None);
4060 restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
4061 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
4062 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
4063 assert_ne!(id1, id2);
4064 }
4065
4066 #[test]
4067 fn safety_backup_id_errors_without_uid() {
4068 let source = KanidmRestoreSource {
4069 local: None,
4070 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4071 name: "backup-1".to_string(),
4072 }),
4073 };
4074 let mut restore = make_restore(source, None);
4075 restore.metadata.uid = None;
4076 assert!(super::compute_safety_backup_id(&restore).is_err());
4077 }
4078
4079 #[test]
4080 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
4081 let source = KanidmRestoreSource {
4082 local: None,
4083 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4084 name: "backup-1".to_string(),
4085 }),
4086 };
4087 let restore = make_restore(source, None);
4088 assert!(super::requires_safety_backup(&restore));
4089 let status_without_ref = KanidmRestoreStatus {
4090 phase: KanidmRestorePhase::RestoringPrimary,
4091 safety_backup_ref: None,
4092 database_mutation_started: false,
4093 ..Default::default()
4094 };
4095 assert!(status_without_ref.safety_backup_ref.is_none());
4096 assert!(!status_without_ref.database_mutation_started);
4097 }
4098
4099 #[test]
4100 fn mutation_boundary_allows_progression_when_safety_ref_set() {
4101 let status_with_ref = KanidmRestoreStatus {
4102 phase: KanidmRestorePhase::RestoringPrimary,
4103 safety_backup_ref: Some("safety-test-restore".to_string()),
4104 safety_backup_manifest_key: Some(
4105 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
4106 ),
4107 safety_backup_payload_sha256: Some("abc123".to_string()),
4108 database_mutation_started: false,
4109 ..Default::default()
4110 };
4111 assert!(status_with_ref.safety_backup_ref.is_some());
4112 assert!(!status_with_ref.database_mutation_started);
4113 }
4114
4115 #[test]
4116 fn result_document_verification_rejects_missing_backup_id() {
4117 let result_json = r#"{
4118 "apiVersion": "backup.kaniop.rs/v1alpha1",
4119 "kind": "ResultDocument",
4120 "operation": "upload",
4121 "success": true,
4122 "exitCode": "success",
4123 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4124 "payloadSha256": "abc123"
4125 }"#;
4126 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4127 assert!(result.backup_id.is_none());
4128 }
4129
4130 #[test]
4131 fn result_document_verification_rejects_missing_manifest_key() {
4132 let result_json = r#"{
4133 "apiVersion": "backup.kaniop.rs/v1alpha1",
4134 "kind": "ResultDocument",
4135 "operation": "upload",
4136 "success": true,
4137 "exitCode": "success",
4138 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4139 "payloadSha256": "abc123"
4140 }"#;
4141 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4142 assert!(result.manifest_key.is_none());
4143 }
4144
4145 #[test]
4146 fn result_document_verification_rejects_missing_payload_sha256() {
4147 let result_json = r#"{
4148 "apiVersion": "backup.kaniop.rs/v1alpha1",
4149 "kind": "ResultDocument",
4150 "operation": "upload",
4151 "success": true,
4152 "exitCode": "success",
4153 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4154 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
4155 }"#;
4156 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4157 assert!(result.payload_sha256.is_none());
4158 }
4159
4160 #[test]
4161 fn result_document_verification_rejects_failure_exit_code() {
4162 let result_json = r#"{
4163 "apiVersion": "backup.kaniop.rs/v1alpha1",
4164 "kind": "ResultDocument",
4165 "operation": "upload",
4166 "success": false,
4167 "exitCode": "integrity",
4168 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4169 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4170 "payloadSha256": "abc123"
4171 }"#;
4172 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4173 assert!(!result.success);
4174 assert_eq!(
4175 result.exit_code,
4176 kaniop_backup_core::result::ExitCode::Integrity
4177 );
4178 }
4179
4180 #[test]
4181 fn result_document_verification_rejects_wrong_operation() {
4182 let result_json = r#"{
4183 "apiVersion": "backup.kaniop.rs/v1alpha1",
4184 "kind": "ResultDocument",
4185 "operation": "download",
4186 "success": true,
4187 "exitCode": "success",
4188 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4189 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4190 "payloadSha256": "abc123"
4191 }"#;
4192 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4193 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
4194 }
4195
4196 #[test]
4197 fn result_document_verification_accepts_valid_document() {
4198 let result_json = r#"{
4199 "apiVersion": "backup.kaniop.rs/v1alpha1",
4200 "kind": "ResultDocument",
4201 "operation": "upload",
4202 "success": true,
4203 "exitCode": "success",
4204 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4205 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4206 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4207 }"#;
4208 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4209 assert!(result.success);
4210 assert_eq!(
4211 result.exit_code,
4212 kaniop_backup_core::result::ExitCode::Success
4213 );
4214 assert_eq!(result.operation, "upload");
4215 assert!(result.backup_id.is_some());
4216 assert!(result.manifest_key.is_some());
4217 assert!(result.payload_sha256.is_some());
4218 }
4219
4220 #[test]
4221 fn termination_message_path_constant_is_valid() {
4222 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
4223 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
4224 }
4225
4226 #[test]
4227 fn status_tracks_safety_backup_evidence() {
4228 let status = KanidmRestoreStatus {
4229 phase: KanidmRestorePhase::PreparingSource,
4230 safety_backup_ref: Some("safety-test-restore".to_string()),
4231 safety_backup_expected_backup_id: Some(
4232 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
4233 ),
4234 safety_backup_manifest_key: Some(
4235 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
4236 ),
4237 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
4238 ..Default::default()
4239 };
4240 assert!(status.safety_backup_ref.is_some());
4241 assert!(status.safety_backup_expected_backup_id.is_some());
4242 assert!(status.safety_backup_manifest_key.is_some());
4243 assert!(status.safety_backup_payload_sha256.is_some());
4244 }
4245
4246 #[test]
4247 fn fail_closed_semantics_preserved_on_result_parse_failure() {
4248 let invalid_json = "not valid json";
4249 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
4250 assert!(result.is_err());
4251 }
4252
4253 #[test]
4254 fn fail_closed_semantics_preserved_on_oversized_result() {
4255 let oversized_json = format!(
4256 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
4257 kaniop_backup_core::result::RESULT_DOC_VERSION,
4258 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
4259 );
4260 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
4261 assert!(result.is_err());
4262 }
4263
4264 #[test]
4265 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
4266 let source = KanidmRestoreSource {
4267 local: None,
4268 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4269 name: "backup-1".to_string(),
4270 }),
4271 };
4272 let mut restore = make_restore(source, None);
4273 restore.metadata.namespace = Some("test-ns".to_string());
4274 let target = super::super::crd::Kanidm::default();
4275 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4276 let doc_str = super::build_safety_upload_operation_doc(
4277 &restore,
4278 &target,
4279 "test-backup-id",
4280 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4281 "my-bucket",
4282 "prod",
4283 "https://s3.example.com",
4284 "us-east-1",
4285 false,
4286 false,
4287 Some(&ca_path),
4288 None,
4289 )
4290 .unwrap();
4291 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4292 assert_eq!(parsed["caBundlePath"], ca_path);
4293 }
4294
4295 #[test]
4296 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
4297 let source = KanidmRestoreSource {
4298 local: None,
4299 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4300 name: "backup-1".to_string(),
4301 }),
4302 };
4303 let mut restore = make_restore(source, None);
4304 restore.metadata.namespace = Some("test-ns".to_string());
4305 let target = super::super::crd::Kanidm::default();
4306 let doc_str = super::build_safety_upload_operation_doc(
4307 &restore,
4308 &target,
4309 "test-backup-id",
4310 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4311 "my-bucket",
4312 "prod",
4313 "https://s3.example.com",
4314 "us-east-1",
4315 false,
4316 false,
4317 None,
4318 None,
4319 )
4320 .unwrap();
4321 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4322 assert!(parsed["caBundlePath"].is_null());
4323 }
4324
4325 #[test]
4326 fn download_operation_doc_includes_ca_bundle_path_when_set() {
4327 let source = KanidmRestoreSource {
4328 local: None,
4329 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4330 name: "backup-1".to_string(),
4331 }),
4332 };
4333 let _restore = make_restore(source, None);
4334 let target = super::super::crd::Kanidm::default();
4335 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
4336 let doc_str = super::build_download_operation_doc(
4337 &target,
4338 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4339 "019c7c76-f423-7a12-8f41-2bea7588a303",
4340 "source-uid-456",
4341 "real-bucket",
4342 "real-prefix",
4343 "https://real-endpoint.com",
4344 "eu-west-1",
4345 true,
4346 false,
4347 Some(&ca_path),
4348 None,
4349 );
4350 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4351 assert_eq!(parsed["caBundlePath"], ca_path);
4352 }
4353
4354 #[test]
4355 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4356 let source = KanidmRestoreSource {
4357 local: None,
4358 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4359 name: "backup-1".to_string(),
4360 }),
4361 };
4362 let _restore = make_restore(source, None);
4363 let target = super::super::crd::Kanidm::default();
4364 let doc_str = super::build_download_operation_doc(
4365 &target,
4366 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4367 "019c7c76-f423-7a12-8f41-2bea7588a303",
4368 "source-uid-456",
4369 "real-bucket",
4370 "real-prefix",
4371 "https://real-endpoint.com",
4372 "eu-west-1",
4373 true,
4374 false,
4375 None,
4376 None,
4377 );
4378 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4379 assert!(parsed["caBundlePath"].is_null());
4380 }
4381
4382 #[test]
4383 fn source_prep_result_operation_is_download() {
4384 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4385 assert_ne!(
4386 super::SOURCE_PREP_RESULT_OPERATION,
4387 super::SAFETY_BACKUP_RESULT_OPERATION
4388 );
4389 }
4390
4391 #[test]
4392 fn download_result_document_verification_accepts_valid_document() {
4393 let result_json = r#"{
4394 "apiVersion": "backup.kaniop.rs/v1alpha1",
4395 "kind": "ResultDocument",
4396 "operation": "download",
4397 "success": true,
4398 "exitCode": "success",
4399 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4400 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4401 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4402 }"#;
4403 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4404 assert!(result.success);
4405 assert_eq!(
4406 result.exit_code,
4407 kaniop_backup_core::result::ExitCode::Success
4408 );
4409 assert_eq!(result.operation, "download");
4410 assert_eq!(
4411 result.backup_id.as_deref(),
4412 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4413 );
4414 assert!(result.manifest_key.is_some());
4415 assert!(result.payload_sha256.is_some());
4416 }
4417
4418 #[test]
4419 fn download_result_document_rejects_wrong_operation() {
4420 let result_json = r#"{
4421 "apiVersion": "backup.kaniop.rs/v1alpha1",
4422 "kind": "ResultDocument",
4423 "operation": "upload",
4424 "success": true,
4425 "exitCode": "success",
4426 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4427 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4428 "payloadSha256": "abc123"
4429 }"#;
4430 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4431 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4432 }
4433
4434 #[test]
4435 fn download_result_document_rejects_failure_success_field() {
4436 let result_json = r#"{
4437 "apiVersion": "backup.kaniop.rs/v1alpha1",
4438 "kind": "ResultDocument",
4439 "operation": "download",
4440 "success": false,
4441 "exitCode": "retryable",
4442 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4443 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4444 "payloadSha256": "abc123"
4445 }"#;
4446 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4447 assert!(!result.success);
4448 assert_ne!(
4449 result.exit_code,
4450 kaniop_backup_core::result::ExitCode::Success
4451 );
4452 }
4453
4454 #[test]
4455 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4456 let status = KanidmRestoreStatus {
4457 phase: KanidmRestorePhase::PreparingSource,
4458 database_mutation_started: false,
4459 ..Default::default()
4460 };
4461 assert!(!status.database_mutation_started);
4462 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4463 }
4464
4465 #[test]
4466 fn backup_job_volume_size_default_is_10gi() {
4467 let qty = crate::controller::backup_job_volume_size();
4468 assert_eq!(qty, Quantity("10Gi".to_string()));
4469 }
4470
4471 #[test]
4472 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4473 use kaniop_backup_core::auth::build_encryption_env_vars;
4474 use kaniop_backup_core::crd::EncryptionKeyRef;
4475
4476 let key_ref = EncryptionKeyRef {
4477 name: "kek-secret".to_string(),
4478 key: "encryption-key".to_string(),
4479 };
4480 let env_vars = build_encryption_env_vars(Some(&key_ref));
4481 assert_eq!(env_vars.len(), 1);
4482 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4483 let secret_ref = env_vars[0]
4484 .value_from
4485 .as_ref()
4486 .unwrap()
4487 .secret_key_ref
4488 .as_ref()
4489 .unwrap();
4490 assert_eq!(secret_ref.name, "kek-secret");
4491 assert_eq!(secret_ref.key, "encryption-key");
4492 }
4493
4494 #[test]
4495 fn encryption_env_vars_absent_when_no_key_ref() {
4496 use kaniop_backup_core::auth::build_encryption_env_vars;
4497
4498 let env_vars = build_encryption_env_vars(None);
4499 assert!(
4500 env_vars.is_empty(),
4501 "encryption env vars must be empty when key_ref is None"
4502 );
4503 }
4504
4505 #[test]
4506 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4507 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, EncryptionKeyRef};
4508
4509 let source = KanidmRestoreSource {
4510 local: None,
4511 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4512 name: "backup-1".to_string(),
4513 }),
4514 };
4515 let mut restore = make_restore(source, None);
4516 restore.metadata.namespace = Some("test-ns".to_string());
4517 let target = super::super::crd::Kanidm::default();
4518 let encryption = RepositoryEncryption {
4519 mode: EncryptionMode::ClientSide,
4520 key_id: None,
4521 key_ref: Some(EncryptionKeyRef {
4522 name: "kek-secret".to_string(),
4523 key: "encryption-key".to_string(),
4524 }),
4525 };
4526 let doc_str = super::build_safety_upload_operation_doc(
4527 &restore,
4528 &target,
4529 "test-backup-id",
4530 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4531 "my-bucket",
4532 "prod",
4533 "https://s3.example.com",
4534 "us-east-1",
4535 false,
4536 false,
4537 None,
4538 Some(&encryption),
4539 )
4540 .unwrap();
4541 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4542 assert_eq!(parsed["encryptionMode"], "clientSide");
4543 assert!(parsed["encryptionKeyId"].is_null());
4544 }
4545
4546 #[test]
4547 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4548 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4549
4550 let source = KanidmRestoreSource {
4551 local: None,
4552 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4553 name: "backup-1".to_string(),
4554 }),
4555 };
4556 let _restore = make_restore(source, None);
4557 let target = super::super::crd::Kanidm::default();
4558 let encryption = RepositoryEncryption {
4559 mode: EncryptionMode::ProviderKms,
4560 key_id: Some("alias/kaniop-backups".to_string()),
4561 key_ref: None,
4562 };
4563 let doc_str = super::build_download_operation_doc(
4564 &target,
4565 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4566 "019c7c76-f423-7a12-8f41-2bea7588a303",
4567 "source-uid-456",
4568 "real-bucket",
4569 "real-prefix",
4570 "https://real-endpoint.com",
4571 "eu-west-1",
4572 true,
4573 false,
4574 None,
4575 Some(&encryption),
4576 );
4577 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4578 assert_eq!(parsed["encryptionMode"], "providerKms");
4579 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4580 }
4581
4582 #[test]
4583 fn post_mutation_failure_records_job_names() {
4584 let existing = KanidmRestoreStatus {
4585 phase: KanidmRestorePhase::RestoringPrimary,
4586 database_mutation_started: true,
4587 ..Default::default()
4588 };
4589
4590 let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4591 assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4592 assert_eq!(
4593 restore_failed.restore_job_name.as_deref(),
4594 Some("my-restore-job")
4595 );
4596 assert!(restore_failed.verify_job_name.is_none());
4597 assert!(restore_failed.database_mutation_started);
4598 assert_eq!(
4599 restore_failed.message.as_deref(),
4600 Some("database restore job failed")
4601 );
4602
4603 let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4604 assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4605 assert_eq!(
4606 verify_failed.verify_job_name.as_deref(),
4607 Some("my-verify-job")
4608 );
4609 assert!(verify_failed.restore_job_name.is_none());
4610 assert!(verify_failed.database_mutation_started);
4611 assert_eq!(
4612 verify_failed.message.as_deref(),
4613 Some("database verification failed")
4614 );
4615 }
4616
4617 #[test]
4618 fn restore_and_verify_job_names_are_deterministic() {
4619 let source = KanidmRestoreSource {
4620 local: Some(super::KanidmRestoreLocalSource {
4621 file_name: "backup.json".to_string(),
4622 }),
4623 backup_ref: None,
4624 };
4625 let restore = make_restore(source, None);
4626 assert_eq!(restore_job_name(&restore), "test-restore-restore");
4627 assert_eq!(verify_job_name(&restore), "test-restore-verify");
4628 }
4629}