1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76 feature = "schemars",
77 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80 category = "kaniop",
81 group = "kaniop.rs",
82 version = "v1beta1",
83 kind = "KanidmRestore",
84 plural = "kanidmrestores",
85 singular = "kanidmrestore",
86 shortname = "idmrestore",
87 namespaced,
88 status = "KanidmRestoreStatus",
89 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95 pub target_ref: KanidmRestoreTargetRef,
96 pub source: KanidmRestoreSource,
97 pub restore_image: String,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106 pub name: String,
107 pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113 feature = "schemars",
114 schemars(extend("x-kubernetes-validations" = [
115 {
116 "message": "exactly one of local or backupRef must be set",
117 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118 }
119 ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123 #[serde(skip_serializing_if = "Option::is_none")]
125 pub local: Option<KanidmRestoreLocalSource>,
126 #[serde(skip_serializing_if = "Option::is_none")]
128 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135 pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143 pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151 #[serde(skip_serializing_if = "Option::is_none")]
153 pub repository_ref: Option<SafetyBackupRepositoryRef>,
154 #[serde(default)]
156 pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163 pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169 #[default]
170 Pending,
171 Validating,
172 Quiescing,
173 SafetyBackup,
174 PreparingSource,
175 RestoringPrimary,
176 Verifying,
177 RebuildingReplicas,
178 Resuming,
179 Completed,
180 Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187 #[serde(skip_serializing_if = "Option::is_none")]
188 pub observed_generation: Option<i64>,
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_target_uid: Option<String>,
191 #[serde(default)]
192 pub phase: KanidmRestorePhase,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 pub restore_job_name: Option<String>,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub verify_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub safety_backup_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub source_prep_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub safety_backup_ref: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_expected_backup_id: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_manifest_key: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_payload_sha256: Option<String>,
209 #[serde(default)]
210 pub replicas_cleared: bool,
211 #[serde(default)]
212 pub certificates_cleared: bool,
213 #[serde(default)]
214 pub database_mutation_started: bool,
215 #[serde(default, skip_serializing_if = "Vec::is_empty")]
216 pub original_replicas: Vec<ReplicaCountEntry>,
217 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218 pub phase_timestamps: std::collections::BTreeMap<String, String>,
219 #[serde(default, skip_serializing_if = "Vec::is_empty")]
220 #[cfg_attr(
221 feature = "schemars",
222 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223 )]
224 pub conditions: Vec<Condition>,
225 #[serde(skip_serializing_if = "Option::is_none")]
226 pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233 pub group: String,
234 pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239 attempts: Counter<u64>,
240 outcomes: Counter<u64>,
241 duration_seconds: Histogram<f64>,
242 break_glass_total: Counter<u64>,
243 safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247 fn new() -> Self {
248 let meter = global::meter("kaniop");
249 Self {
250 attempts: meter
251 .u64_counter("kaniop_restore_attempts_total")
252 .with_description("Number of Kanidm restore attempts started")
253 .build(),
254 outcomes: meter
255 .u64_counter("kaniop_restore_outcomes_total")
256 .with_description("Number of terminal Kanidm restore outcomes")
257 .build(),
258 duration_seconds: meter
259 .f64_histogram("kaniop_restore_duration_seconds")
260 .with_description("Kanidm restore duration from object creation to terminal phase")
261 .with_unit("s")
262 .build(),
263 break_glass_total: meter
264 .u64_counter("kaniop_restore_break_glass_total")
265 .with_description("Number of break-glass safety backup overrides")
266 .build(),
267 safety_backup_duration_seconds: meter
268 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269 .with_description("Duration of pre-restore safety backup creation")
270 .with_unit("s")
271 .build(),
272 }
273 }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278 client: Client,
279 recorder: Recorder,
280 metrics: RestoreMetrics,
281}
282
283async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
284 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
285 match phase {
286 KanidmRestorePhase::Pending => {
287 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
288 Ok(Action::requeue(REQUEUE))
289 }
290 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
291 Ok(target) => {
292 mark_restoring(&restore, &target, &ctx).await?;
293 let mut status = restore.status.clone().unwrap_or_default();
294 status.observed_target_uid = target.uid();
295 status.original_replicas = capture_original_replicas(&target);
296 status.phase = KanidmRestorePhase::Quiescing;
297 status.message = None;
298 patch_status(&restore, &ctx, status).await?;
299 Ok(Action::requeue(REQUEUE))
300 }
301 Err(error) => {
302 set_phase(
303 &restore,
304 &ctx,
305 KanidmRestorePhase::Failed,
306 Some(error.to_string()),
307 )
308 .await?;
309 Ok(Action::requeue(Duration::from_secs(300)))
310 }
311 },
312 KanidmRestorePhase::Quiescing => {
313 let target = get_target(&restore, &ctx).await?;
314 scale_all(&target, &ctx, 0).await?;
315 if target_pods_stopped(&target, &ctx).await?
316 && target_volumes_detached(&target, &ctx).await?
317 {
318 let next_phase = if requires_safety_backup(&restore) {
319 KanidmRestorePhase::SafetyBackup
320 } else {
321 record_break_glass(&restore, &ctx).await?;
322 KanidmRestorePhase::PreparingSource
323 };
324 if next_phase == KanidmRestorePhase::PreparingSource {
325 let refreshed = Api::<KanidmRestore>::namespaced(
326 ctx.client.clone(),
327 restore.namespace().as_deref().unwrap_or_default(),
328 )
329 .get(&restore.name_any())
330 .await
331 .map_err(|error| {
332 Error::kube_error(
333 "get",
334 "KanidmRestore",
335 restore.namespace().as_deref().unwrap_or_default(),
336 restore.name_any(),
337 error,
338 )
339 })?;
340 set_phase(&refreshed, &ctx, next_phase, None).await?;
341 } else {
342 set_phase(&restore, &ctx, next_phase, None).await?;
343 }
344 }
345 Ok(Action::requeue(REQUEUE))
346 }
347 KanidmRestorePhase::SafetyBackup => {
348 let target = get_target(&restore, &ctx).await?;
349 let name = safety_backup_job_name(&restore);
350 let expected_backup_id = compute_safety_backup_id(&restore);
351 let expected_manifest_key = compute_safety_manifest_key(&restore);
352 ensure_safety_backup_job(
353 &restore,
354 &target,
355 &ctx,
356 &name,
357 &expected_backup_id,
358 &expected_manifest_key,
359 )
360 .await?;
361 let mut status = restore.status.clone().unwrap_or_default();
362 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
363 match job_state(&restore, &ctx, &name).await? {
364 JobState::Complete => {
365 let result =
366 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
367 match result {
368 Ok(verified) => {
369 status.safety_backup_job_name = Some(name);
370 status.safety_backup_ref =
371 Some(format!("safety-{}", restore.name_any()));
372 status.safety_backup_manifest_key = Some(verified.manifest_key);
373 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
374 record_safety_backup_duration(&restore, &ctx).await;
375 status.phase = KanidmRestorePhase::PreparingSource;
376 status.message = None;
377 patch_status(&restore, &ctx, status).await?;
378 }
379 Err(error) => {
380 resume_before_mutation(&restore, &ctx).await?;
381 set_phase(
382 &restore,
383 &ctx,
384 KanidmRestorePhase::Failed,
385 Some(format!("safety backup result verification failed: {error}")),
386 )
387 .await?;
388 }
389 }
390 }
391 JobState::Failed => {
392 resume_before_mutation(&restore, &ctx).await?;
393 set_phase(
394 &restore,
395 &ctx,
396 KanidmRestorePhase::Failed,
397 Some(
398 "safety backup job failed; target restored to original state"
399 .to_string(),
400 ),
401 )
402 .await?;
403 }
404 JobState::Running => {
405 patch_status(&restore, &ctx, status).await?;
406 }
407 }
408 Ok(Action::requeue(REQUEUE))
409 }
410 KanidmRestorePhase::PreparingSource => {
411 if is_remote_source(&restore) {
412 let target = get_target(&restore, &ctx).await?;
413 let name = source_prep_job_name(&restore);
414 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
415 match job_state(&restore, &ctx, &name).await? {
416 JobState::Complete => {
417 let backup_id = restore
418 .spec
419 .source
420 .backup_ref
421 .as_ref()
422 .map(|b| b.name.clone())
423 .unwrap_or_default();
424 let backup = {
425 let ns = restore.namespace().unwrap();
426 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
427 .get(&backup_id)
428 .await
429 .map_err(|e| {
430 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
431 })?
432 };
433 let expected_backup_id = &backup.spec.backup_id;
434 let result =
435 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
436 .await;
437 match result {
438 Ok(verified) => {
439 if let Some(expected_sha256) = backup
440 .status
441 .as_ref()
442 .and_then(|s| s.payload_sha256.as_deref())
443 {
444 if verified.payload_sha256 != expected_sha256 {
445 resume_before_mutation(&restore, &ctx).await?;
446 set_phase(
447 &restore,
448 &ctx,
449 KanidmRestorePhase::Failed,
450 Some(format!(
451 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
452 verified.payload_sha256
453 )),
454 )
455 .await?;
456 return Ok(Action::requeue(REQUEUE));
457 }
458 }
459 let mut status = restore.status.clone().unwrap_or_default();
460 status.source_prep_job_name = Some(name);
461 status.phase = KanidmRestorePhase::RestoringPrimary;
462 status.message = None;
463 patch_status(&restore, &ctx, status).await?;
464 }
465 Err(error) => {
466 resume_before_mutation(&restore, &ctx).await?;
467 set_phase(
468 &restore,
469 &ctx,
470 KanidmRestorePhase::Failed,
471 Some(format!(
472 "source preparation result verification failed: {error}"
473 )),
474 )
475 .await?;
476 }
477 }
478 }
479 JobState::Failed => {
480 resume_before_mutation(&restore, &ctx).await?;
481 set_phase(
482 &restore,
483 &ctx,
484 KanidmRestorePhase::Failed,
485 Some(
486 "source preparation job failed; target restored to original state"
487 .to_string(),
488 ),
489 )
490 .await?;
491 }
492 JobState::Running => {}
493 }
494 } else {
495 let target = get_target(&restore, &ctx).await?;
496 let name = source_check_job_name(&restore);
497 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
498 match job_state(&restore, &ctx, &name).await? {
499 JobState::Complete => {
500 let mut status = restore.status.clone().unwrap_or_default();
501 status.source_prep_job_name = Some(name);
502 status.phase = KanidmRestorePhase::RestoringPrimary;
503 status.message = None;
504 patch_status(&restore, &ctx, status).await?;
505 }
506 JobState::Failed => {
507 resume_before_mutation(&restore, &ctx).await?;
508 set_phase(
509 &restore,
510 &ctx,
511 KanidmRestorePhase::Failed,
512 Some(
513 "local backup file check failed; target restored to original state"
514 .to_string(),
515 ),
516 )
517 .await?;
518 }
519 JobState::Running => {}
520 }
521 }
522 Ok(Action::requeue(REQUEUE))
523 }
524 KanidmRestorePhase::RestoringPrimary => {
525 let target = get_target(&restore, &ctx).await?;
526 let status = restore.status.as_ref().cloned().unwrap_or_default();
527 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
528 return Err(Error::MissingData(
529 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
530 ));
531 }
532 if !status.database_mutation_started {
533 let mut status = status;
534 status.database_mutation_started = true;
535 status.message = Some(
536 "database mutation boundary persisted; starting primary restore".to_string(),
537 );
538 patch_status(&restore, &ctx, status).await?;
539 return Ok(Action::requeue(REQUEUE));
540 }
541 let name = restore_job_name(&restore);
542 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
543 match job_state(&restore, &ctx, &name).await? {
544 JobState::Complete => {
545 let mut status = restore.status.clone().unwrap_or_default();
546 status.restore_job_name = Some(name);
547 status.phase = KanidmRestorePhase::Verifying;
548 status.message = None;
549 patch_status(&restore, &ctx, status).await?;
550 }
551 JobState::Failed => {
552 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
553 }
554 JobState::Running => {}
555 }
556 Ok(Action::requeue(REQUEUE))
557 }
558 KanidmRestorePhase::Verifying => {
559 let target = get_target(&restore, &ctx).await?;
560 let name = verify_job_name(&restore);
561 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
562 match job_state(&restore, &ctx, &name).await? {
563 JobState::Complete => {
564 let mut status = restore.status.clone().unwrap_or_default();
565 status.verify_job_name = Some(name);
566 status.phase = KanidmRestorePhase::RebuildingReplicas;
567 status.message = None;
568 patch_status(&restore, &ctx, status).await?;
569 }
570 JobState::Failed => {
571 fail_after_mutation(&restore, &ctx, "database verification failed").await?
572 }
573 JobState::Running => {}
574 }
575 Ok(Action::requeue(REQUEUE))
576 }
577 KanidmRestorePhase::RebuildingReplicas => {
578 let target = get_target(&restore, &ctx).await?;
579 let mut status = restore.status.clone().unwrap_or_default();
580 if !status.replicas_cleared {
581 if delete_secondary_pvcs(&target, &ctx).await? {
582 status.replicas_cleared = true;
583 status.message = Some("secondary database state cleared".to_string());
584 patch_status(&restore, &ctx, status).await?;
585 }
586 return Ok(Action::requeue(REQUEUE));
587 }
588
589 scale_primary(&target, &ctx, 1).await?;
590 if !primary_ready(&target, &ctx).await? {
591 return Ok(Action::requeue(REQUEUE));
592 }
593 if !status.certificates_cleared {
594 delete_replica_cert_secrets(&target, &ctx).await?;
595 delete_admin_secret(&target, &ctx).await?;
596 status.certificates_cleared = true;
597 status.message = Some(
598 "replica certificates and admin secret cleared for regeneration".to_string(),
599 );
600 patch_status(&restore, &ctx, status).await?;
601 return Ok(Action::requeue(REQUEUE));
602 }
603 status.phase = KanidmRestorePhase::Resuming;
604 status.message = None;
605 patch_status(&restore, &ctx, status).await?;
606 Ok(Action::requeue(REQUEUE))
607 }
608 KanidmRestorePhase::Resuming => {
609 let target = get_target(&restore, &ctx).await?;
610 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
611 clear_restoring(&restore, &target, &ctx).await?;
612 Ok(Action::requeue(Duration::from_secs(3600)))
613 }
614 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
615 Ok(Action::requeue(Duration::from_secs(3600)))
616 }
617 }
618}
619
620async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
621 let target = get_target(restore, ctx).await?;
622 let actual_uid = target
623 .uid()
624 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
625 if actual_uid != restore.spec.target_ref.uid {
626 return Err(Error::MissingData(format!(
627 "target UID mismatch: expected {}, got {}",
628 restore.spec.target_ref.uid, actual_uid
629 )));
630 }
631 validate_source(restore)?;
632 if is_remote_source(restore) {
633 validate_backup_ref(restore, &target, ctx).await?;
634 }
635 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
636 {
637 return Err(Error::MissingData(format!(
638 "restoreImage must be the target's pinned Kanidm image (target image is {})",
639 target.spec.image
640 )));
641 }
642 let storage = target
643 .spec
644 .storage
645 .as_ref()
646 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
647 if storage.empty_dir.is_some()
648 || storage.ephemeral.is_some()
649 || storage.volume_claim_template.is_none()
650 {
651 return Err(Error::MissingData(
652 "restore requires PVC-backed Kanidm storage".to_string(),
653 ));
654 }
655 let primaries = target
656 .spec
657 .replica_groups
658 .iter()
659 .filter(|rg| rg.primary_node)
660 .count();
661 if primaries != 1 {
662 return Err(Error::MissingData(
663 "backup/restore requires exactly one primary replica group".to_string(),
664 ));
665 }
666 let ns = restore.namespace().unwrap();
667 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
668 .list(&ListParams::default())
669 .await
670 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
671 if restores.items.iter().any(|other| {
672 other.name_any() != restore.name_any()
673 && other.spec.target_ref.name == restore.spec.target_ref.name
674 && !matches!(
675 other.status.as_ref().map(|s| s.phase),
676 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
677 )
678 }) {
679 return Err(Error::MissingData(
680 "another active restore targets this Kanidm".to_string(),
681 ));
682 }
683 validate_safety_backup_config(restore)?;
684 Ok(target)
685}
686
687fn safe_basename(name: &str) -> bool {
688 !name.is_empty()
689 && name != "."
690 && name != ".."
691 && !name.contains('/')
692 && !name.contains('\\')
693 && !name.contains('\0')
694}
695
696fn mutable_image(image: &str) -> bool {
697 image == "kanidm/server:latest"
698 || image.ends_with(":latest")
699 || (!image.contains('@')
700 && !image
701 .rsplit('/')
702 .next()
703 .is_some_and(|part| part.contains(':')))
704}
705
706async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
707 let ns = restore
708 .namespace()
709 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
710 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
711 .get(&restore.spec.target_ref.name)
712 .await
713 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
714}
715
716async fn mark_restoring(
717 restore: &KanidmRestore,
718 target: &Kanidm,
719 ctx: &RestoreContext,
720) -> Result<()> {
721 let ns = target.namespace().unwrap();
722 let uid = restore
723 .uid()
724 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
725 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
726 .patch(
727 &target.name_any(),
728 &PatchParams::default(),
729 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
730 )
731 .await
732 .map(|_| ())
733 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
734}
735
736async fn clear_restoring(
737 restore: &KanidmRestore,
738 target: &Kanidm,
739 ctx: &RestoreContext,
740) -> Result<()> {
741 let ns = target.namespace().unwrap();
742 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
743 return Ok(());
744 }
745 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
746 .patch(
747 &target.name_any(),
748 &PatchParams::default(),
749 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
750 )
751 .await
752 .map(|_| ())
753 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
754}
755
756async fn patch_status(
757 restore: &KanidmRestore,
758 ctx: &RestoreContext,
759 mut status: KanidmRestoreStatus,
760) -> Result<()> {
761 let ns = restore.namespace().unwrap();
762 let previous_phase = restore.status.as_ref().map(|current| current.phase);
763 let next_phase = status.phase;
764 status.observed_generation = restore.metadata.generation;
765 update_restore_conditions(&mut status, restore.metadata.generation);
766
767 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
768 .patch_status(
769 &restore.name_any(),
770 &PatchParams::default(),
771 &Patch::Merge(json!({"status": &status})),
772 )
773 .await
774 .map_err(|e| {
775 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
776 })?;
777
778 if previous_phase != Some(next_phase) {
779 record_restore_transition(
780 restore,
781 ctx,
782 previous_phase,
783 next_phase,
784 status.message.as_deref(),
785 )
786 .await;
787 }
788 Ok(())
789}
790
791fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
792 let previous = status.conditions.clone();
793 let phase = status.phase;
794 let phase_reason = format!("{phase:?}");
795 let message = status
796 .message
797 .clone()
798 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
799 let terminal_success = phase == KanidmRestorePhase::Completed;
800 let terminal_failure = phase == KanidmRestorePhase::Failed;
801 let progressing = !terminal_success && !terminal_failure;
802
803 let break_glass = previous
804 .iter()
805 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
806 .cloned();
807 status.conditions = vec![
808 restore_condition(
809 &previous,
810 CONDITION_PROGRESSING,
811 if progressing {
812 CONDITION_TRUE
813 } else {
814 CONDITION_FALSE
815 },
816 &phase_reason,
817 &message,
818 generation,
819 ),
820 restore_condition(
821 &previous,
822 CONDITION_READY,
823 if terminal_success {
824 CONDITION_TRUE
825 } else {
826 CONDITION_FALSE
827 },
828 if terminal_success {
829 "RestoreCompleted"
830 } else {
831 "RestoreNotCompleted"
832 },
833 if terminal_success {
834 "Kanidm restore completed successfully."
835 } else {
836 "Kanidm restore has not completed successfully."
837 },
838 generation,
839 ),
840 restore_condition(
841 &previous,
842 CONDITION_FAILED,
843 if terminal_failure {
844 CONDITION_TRUE
845 } else {
846 CONDITION_FALSE
847 },
848 if terminal_failure {
849 "RestoreFailed"
850 } else {
851 "NoRestoreFailure"
852 },
853 if terminal_failure {
854 status
855 .message
856 .as_deref()
857 .unwrap_or("Kanidm restore failed.")
858 } else {
859 "No terminal restore failure has been recorded."
860 },
861 generation,
862 ),
863 ];
864 if let Some(condition) = break_glass {
865 status.conditions.push(condition);
866 }
867}
868
869fn restore_condition(
870 previous: &[Condition],
871 condition_type: &str,
872 condition_status: &str,
873 reason: &str,
874 message: &str,
875 generation: Option<i64>,
876) -> Condition {
877 let last_transition_time = previous
878 .iter()
879 .find(|condition| {
880 condition.type_ == condition_type
881 && condition.status == condition_status
882 && condition.reason == reason
883 })
884 .map(|condition| condition.last_transition_time.clone())
885 .unwrap_or_else(|| Time(Timestamp::now()));
886 Condition {
887 type_: condition_type.to_string(),
888 status: condition_status.to_string(),
889 reason: reason.to_string(),
890 message: message.to_string(),
891 last_transition_time,
892 observed_generation: generation,
893 }
894}
895
896async fn record_restore_transition(
897 restore: &KanidmRestore,
898 ctx: &RestoreContext,
899 previous_phase: Option<KanidmRestorePhase>,
900 phase: KanidmRestorePhase,
901 message: Option<&str>,
902) {
903 if phase == KanidmRestorePhase::Validating {
904 ctx.metrics.attempts.add(1, &[]);
905 }
906
907 let result = match phase {
908 KanidmRestorePhase::Completed => Some("success"),
909 KanidmRestorePhase::Failed => Some("failure"),
910 _ => None,
911 };
912 if let Some(result) = result {
913 let attributes = [KeyValue::new("result", result)];
914 ctx.metrics.outcomes.add(1, &attributes);
915 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
916 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
917 ctx.metrics.duration_seconds.record(elapsed, &attributes);
918 }
919 }
920
921 let reason = if phase == KanidmRestorePhase::Failed {
922 "RestoreFailed"
923 } else {
924 "RestorePhaseChanged"
925 };
926 let note = message.map(str::to_string).or_else(|| {
927 Some(format!(
928 "Kanidm restore phase changed from {} to {phase:?}.",
929 previous_phase
930 .map(|previous| format!("{previous:?}"))
931 .unwrap_or_else(|| "None".to_string())
932 ))
933 });
934 if let Err(error) = ctx
935 .recorder
936 .publish(
937 &Event {
938 type_: if phase == KanidmRestorePhase::Failed {
939 EventType::Warning
940 } else {
941 EventType::Normal
942 },
943 reason: reason.to_string(),
944 note,
945 action: "Restore".to_string(),
946 secondary: None,
947 },
948 &restore.object_ref(&()),
949 )
950 .await
951 {
952 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
953 }
954}
955
956async fn set_phase(
957 restore: &KanidmRestore,
958 ctx: &RestoreContext,
959 phase: KanidmRestorePhase,
960 message: Option<String>,
961) -> Result<()> {
962 let mut status = restore.status.clone().unwrap_or_default();
963 status.phase = phase;
964 status.message = message;
965 patch_status(restore, ctx, status).await
966}
967
968async fn fail_after_mutation(
969 restore: &KanidmRestore,
970 ctx: &RestoreContext,
971 message: &str,
972) -> Result<()> {
973 set_phase(
974 restore,
975 ctx,
976 KanidmRestorePhase::Failed,
977 Some(message.to_string()),
978 )
979 .await
980}
981
982fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
983 target
984 .spec
985 .replica_groups
986 .iter()
987 .find(|rg| rg.primary_node)
988 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
989}
990
991async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
992 let ns = target.namespace().unwrap();
993 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
994 for rg in &target.spec.replica_groups {
995 let name = target.statefulset_name(&rg.name);
996 api.patch(
997 &name,
998 &PatchParams::default(),
999 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1000 )
1001 .await
1002 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1003 }
1004 Ok(())
1005}
1006
1007async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1008 let ns = target.namespace().unwrap();
1009 let rg = primary_group(target)?;
1010 let name = target.statefulset_name(&rg.name);
1011 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1012 .patch(
1013 &name,
1014 &PatchParams::default(),
1015 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1016 )
1017 .await
1018 .map(|_| ())
1019 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1020}
1021
1022async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1023 let ns = target.namespace().unwrap();
1024 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1025 for rg in &target.spec.replica_groups {
1026 let name = target.statefulset_name(&rg.name);
1027 api.patch(
1028 &name,
1029 &PatchParams::default(),
1030 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1031 )
1032 .await
1033 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1034 }
1035 Ok(())
1036}
1037
1038async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1039 let ns = target.namespace().unwrap();
1040 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1041 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1042 .await
1043 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1044 Ok(pods.items.is_empty())
1045}
1046
1047async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1048 let ns = target.namespace().unwrap();
1049 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1050 let mut pv_names = BTreeSet::new();
1051 let primary_name = primary_pvc_name(target)?;
1052 let primary_pvc = pvc_api
1053 .get(&primary_name)
1054 .await
1055 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1056 let primary_pv = primary_pvc
1057 .spec
1058 .as_ref()
1059 .and_then(|spec| spec.volume_name.clone())
1060 .ok_or_else(|| {
1061 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1062 })?;
1063 pv_names.insert(primary_pv);
1064
1065 for rg in &target.spec.replica_groups {
1066 let sts = target.statefulset_name(&rg.name);
1067 for ordinal in 0..rg.replicas {
1068 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1069 if name == primary_name {
1070 continue;
1071 }
1072 if let Some(pvc) = pvc_api
1073 .get_opt(&name)
1074 .await
1075 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1076 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1077 {
1078 pv_names.insert(pv_name);
1079 }
1080 }
1081 }
1082
1083 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1084 .list(&ListParams::default())
1085 .await
1086 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1087 Ok(!attachments.items.iter().any(|attachment| {
1088 attachment
1089 .spec
1090 .source
1091 .persistent_volume_name
1092 .as_ref()
1093 .is_some_and(|pv_name| pv_names.contains(pv_name))
1094 }))
1095}
1096
1097async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098 let ns = target.namespace().unwrap();
1099 let name = target.statefulset_name(&primary_group(target)?.name);
1100 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1101 .get(&name)
1102 .await
1103 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1104 Ok(sts
1105 .status
1106 .as_ref()
1107 .and_then(|s| s.ready_replicas)
1108 .unwrap_or(0)
1109 >= 1)
1110}
1111
1112fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1113 let rg = primary_group(target)?;
1114 Ok(format!(
1115 "{DATA_VOLUME}-{}-0",
1116 target.statefulset_name(&rg.name)
1117 ))
1118}
1119
1120async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1121 let ns = target.namespace().unwrap();
1122 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1123 let mut all_absent = true;
1124 for rg in &target.spec.replica_groups {
1125 let sts = target.statefulset_name(&rg.name);
1126 for ordinal in 0..rg.replicas {
1127 if rg.primary_node && ordinal == 0 {
1128 continue;
1129 }
1130 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1131 match api
1132 .get_opt(&name)
1133 .await
1134 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135 {
1136 None => {}
1137 Some(pvc) => {
1138 all_absent = false;
1139 if pvc.metadata.deletion_timestamp.is_none() {
1140 match api.delete(&name, &DeleteParams::default()).await {
1141 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1142 Err(kube::Error::Api(status)) if status.code == 404 => {}
1143 Err(error) => {
1144 return Err(Error::kube_error(
1145 "delete",
1146 "PersistentVolumeClaim",
1147 &ns,
1148 &name,
1149 error,
1150 ));
1151 }
1152 }
1153 }
1154 }
1155 }
1156 }
1157 }
1158 Ok(all_absent)
1159}
1160
1161async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1162 let ns = target.namespace().unwrap();
1163 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1164 let label_selector = format!(
1165 "{}={},{}={}",
1166 CLUSTER_LABEL,
1167 target.name_any(),
1168 SECRET_TYPE_LABEL,
1169 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1170 );
1171 let secrets = api
1172 .list(&ListParams::default().labels(&label_selector))
1173 .await
1174 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1175 let mut all_absent = secrets.items.is_empty();
1176 for secret in secrets.items {
1177 let name = secret.name_any();
1178 if secret.metadata.deletion_timestamp.is_none() {
1179 match api.delete(&name, &DeleteParams::default()).await {
1180 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1181 Err(kube::Error::Api(status)) if status.code == 404 => {}
1182 Err(error) => {
1183 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1184 }
1185 }
1186 all_absent = false;
1187 }
1188 }
1189 Ok(all_absent)
1190}
1191
1192async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1193 let ns = target.namespace().unwrap();
1194 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1195 let name = format!("{}-admin-passwords", target.name_any());
1196 match api
1197 .get_opt(&name)
1198 .await
1199 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1200 {
1201 None => Ok(true),
1202 Some(secret) => {
1203 if secret.metadata.deletion_timestamp.is_some() {
1204 return Ok(false);
1205 }
1206 match api.delete(&name, &DeleteParams::default()).await {
1207 Ok(_) => {
1208 debug!(secret = %name, "deleting stale admin secret");
1209 Ok(false)
1210 }
1211 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1212 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1213 }
1214 }
1215 }
1216}
1217
1218fn restore_job_name(restore: &KanidmRestore) -> String {
1219 format!("{}-restore", restore.name_any())
1220}
1221fn verify_job_name(restore: &KanidmRestore) -> String {
1222 format!("{}-verify", restore.name_any())
1223}
1224
1225fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1226 let origin = target
1227 .spec
1228 .origin
1229 .clone()
1230 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1231 vec![
1232 EnvVar {
1233 name: "KANIDM_DB_PATH".to_string(),
1234 value: Some("/data/kanidm.db".to_string()),
1235 ..Default::default()
1236 },
1237 EnvVar {
1238 name: "KANIDM_DOMAIN".to_string(),
1239 value: Some(target.spec.domain.clone()),
1240 ..Default::default()
1241 },
1242 EnvVar {
1243 name: "KANIDM_ORIGIN".to_string(),
1244 value: Some(origin),
1245 ..Default::default()
1246 },
1247 EnvVar {
1248 name: "KANIDM_TLS_CHAIN".to_string(),
1249 value: Some(format!("{TLS_PATH}/tls.crt")),
1250 ..Default::default()
1251 },
1252 EnvVar {
1253 name: "KANIDM_TLS_KEY".to_string(),
1254 value: Some(format!("{TLS_PATH}/tls.key")),
1255 ..Default::default()
1256 },
1257 ]
1258}
1259
1260async fn ensure_database_job(
1261 restore: &KanidmRestore,
1262 target: &Kanidm,
1263 ctx: &RestoreContext,
1264 name: &str,
1265 verify: bool,
1266) -> Result<()> {
1267 let ns = restore.namespace().unwrap();
1268 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1269 if api
1270 .get_opt(name)
1271 .await
1272 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1273 .is_some()
1274 {
1275 return Ok(());
1276 }
1277 let mut command = vec![
1278 "kanidmd".to_string(),
1279 "database".to_string(),
1280 if verify { "backup" } else { "restore" }.to_string(),
1281 ];
1282 if verify {
1283 command.push("/verify/verification.json.gz".to_string());
1284 } else {
1285 if is_remote_source(restore) {
1286 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1287 } else {
1288 let file_name = restore
1289 .spec
1290 .source
1291 .local
1292 .as_ref()
1293 .map(|l| l.file_name.clone())
1294 .unwrap_or_default();
1295 command.push(format!("{BACKUP_PATH}/{file_name}"));
1296 }
1297 }
1298 let job = Job {
1299 metadata: ObjectMeta {
1300 name: Some(name.to_string()),
1301 namespace: Some(ns.clone()),
1302 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1303 ..Default::default()
1304 },
1305 spec: Some(JobSpec {
1306 backoff_limit: Some(0),
1307 template: PodTemplateSpec {
1308 metadata: Some(ObjectMeta::default()),
1309 spec: Some(PodSpec {
1310 automount_service_account_token: Some(false),
1311 restart_policy: Some("Never".to_string()),
1312 security_context: Some(source_job_pod_security_context(target)),
1313 containers: vec![Container {
1314 name: if verify { "verify" } else { "restore" }.to_string(),
1315 image: Some(restore.spec.restore_image.clone()),
1316 command: Some(command),
1317 env: Some(build_kanidm_db_env_vars(target)),
1318 security_context: Some(kanidm_job_security_context()),
1319 resources: Some(job_resource_requirements()),
1320 volume_mounts: Some(vec![
1321 VolumeMount {
1322 name: DATA_VOLUME.to_string(),
1323 mount_path: DATA_PATH.to_string(),
1324 ..Default::default()
1325 },
1326 VolumeMount {
1327 name: TLS_VOLUME.to_string(),
1328 mount_path: TLS_PATH.to_string(),
1329 read_only: Some(true),
1330 ..Default::default()
1331 },
1332 VolumeMount {
1333 name: "verification-output".to_string(),
1334 mount_path: "/verify".to_string(),
1335 ..Default::default()
1336 },
1337 ]),
1338 ..Default::default()
1339 }],
1340 volumes: Some(vec![
1341 Volume {
1342 name: DATA_VOLUME.to_string(),
1343 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1344 claim_name: primary_pvc_name(target)?,
1345 read_only: Some(false),
1346 }),
1347 ..Default::default()
1348 },
1349 Volume {
1350 name: TLS_VOLUME.to_string(),
1351 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1352 secret_name: Some(target.effective_tls_secret_name()),
1353 ..Default::default()
1354 }),
1355 ..Default::default()
1356 },
1357 Volume {
1358 name: "verification-output".to_string(),
1359 empty_dir: Some(Default::default()),
1360 ..Default::default()
1361 },
1362 ]),
1363 ..Default::default()
1364 }),
1365 },
1366 ..Default::default()
1367 }),
1368 ..Default::default()
1369 };
1370 api.create(&PostParams::default(), &job)
1371 .await
1372 .map(|_| ())
1373 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1374}
1375
1376enum JobState {
1377 Running,
1378 Complete,
1379 Failed,
1380}
1381
1382async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1383 let ns = restore.namespace().unwrap();
1384 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1385 .get(name)
1386 .await
1387 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1388 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1389 return Ok(JobState::Failed);
1390 }
1391 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1392 return Ok(JobState::Complete);
1393 }
1394 Ok(JobState::Running)
1395}
1396
1397fn is_remote_source(restore: &KanidmRestore) -> bool {
1398 restore.spec.source.backup_ref.is_some()
1399}
1400
1401fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1402 let safety = restore.spec.safety_backup.as_ref();
1403 let skip = safety.is_some_and(|s| s.skip);
1404 !skip
1405}
1406
1407fn validate_source(restore: &KanidmRestore) -> Result<()> {
1408 let has_local = restore.spec.source.local.is_some();
1409 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1410 if has_local && has_backup_ref {
1411 return Err(Error::MissingData(
1412 "source.local and source.backupRef are mutually exclusive".to_string(),
1413 ));
1414 }
1415 if !has_local && !has_backup_ref {
1416 return Err(Error::MissingData(
1417 "source must specify either local or backupRef".to_string(),
1418 ));
1419 }
1420 if let Some(local) = &restore.spec.source.local {
1421 if !safe_basename(&local.file_name) {
1422 return Err(Error::MissingData(
1423 "restore source fileName must be a safe basename".to_string(),
1424 ));
1425 }
1426 }
1427 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1428 if backup_ref.name.is_empty() {
1429 return Err(Error::MissingData(
1430 "source.backupRef.name must not be empty".to_string(),
1431 ));
1432 }
1433 }
1434 Ok(())
1435}
1436
1437fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1438 let safety = restore.spec.safety_backup.as_ref();
1439 let skip = safety.is_some_and(|s| s.skip);
1440 if skip {
1441 let annotations = restore.metadata.annotations.as_ref();
1442 let reason = annotations
1443 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1444 .map(|s| s.trim())
1445 .unwrap_or("");
1446 let approver = annotations
1447 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1448 .map(|s| s.trim())
1449 .unwrap_or("");
1450 if reason.is_empty() {
1451 return Err(Error::MissingData(format!(
1452 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1453 )));
1454 }
1455 if approver.is_empty() {
1456 return Err(Error::MissingData(format!(
1457 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1458 )));
1459 }
1460 }
1461 if is_remote_source(restore) && !skip {
1462 let has_repo = safety
1463 .and_then(|s| s.repository_ref.as_ref())
1464 .is_some_and(|r| !r.name.is_empty());
1465 if !has_repo {
1466 return Err(Error::MissingData(
1467 "remote restore requires safetyBackup.repositoryRef".to_string(),
1468 ));
1469 }
1470 }
1471 Ok(())
1472}
1473
1474fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1475 target
1476 .spec
1477 .replica_groups
1478 .iter()
1479 .map(|rg| ReplicaCountEntry {
1480 group: rg.name.clone(),
1481 replicas: rg.replicas,
1482 })
1483 .collect()
1484}
1485
1486async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1487 if let Ok(target) = get_target(restore, ctx).await {
1488 let owns_maintenance =
1489 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1490 if owns_maintenance {
1491 scale_desired(&target, ctx).await?;
1492 clear_restoring(restore, &target, ctx).await?;
1493 }
1494 }
1495 Ok(())
1496}
1497
1498async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1499 ctx.metrics.break_glass_total.add(1, &[]);
1500 let reason = restore
1501 .metadata
1502 .annotations
1503 .as_ref()
1504 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1505 .cloned()
1506 .unwrap_or_default();
1507 let approver = restore
1508 .metadata
1509 .annotations
1510 .as_ref()
1511 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1512 .cloned()
1513 .unwrap_or_default();
1514 let mut status = restore.status.clone().unwrap_or_default();
1515 let break_glass_condition = restore_condition(
1516 &status.conditions,
1517 CONDITION_BREAK_GLASS,
1518 CONDITION_TRUE,
1519 "BreakGlassOverride",
1520 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1521 restore.metadata.generation,
1522 );
1523 status
1524 .conditions
1525 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1526 status.conditions.push(break_glass_condition);
1527 patch_status(restore, ctx, status).await?;
1528 if let Err(error) = ctx
1529 .recorder
1530 .publish(
1531 &Event {
1532 type_: EventType::Warning,
1533 reason: "SafetyBackupSkipped".to_string(),
1534 note: Some(format!(
1535 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1536 )),
1537 action: "Restore".to_string(),
1538 secondary: None,
1539 },
1540 &restore.object_ref(&()),
1541 )
1542 .await
1543 {
1544 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1545 }
1546 warn!(
1547 restore = %restore.name_any(),
1548 reason = %reason,
1549 approver = %approver,
1550 "break-glass safety backup override activated"
1551 );
1552 Ok(())
1553}
1554
1555async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1556 let status = restore.status.as_ref().cloned().unwrap_or_default();
1557 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1558 if let Ok(start_ts) = start.parse::<Timestamp>() {
1559 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1560 ctx.metrics
1561 .safety_backup_duration_seconds
1562 .record(elapsed, &[]);
1563 }
1564 }
1565}
1566
1567fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1568 format!("{}-safety-backup", restore.name_any())
1569}
1570
1571fn source_prep_job_name(restore: &KanidmRestore) -> String {
1572 format!("{}-source-prep", restore.name_any())
1573}
1574
1575fn source_check_job_name(restore: &KanidmRestore) -> String {
1576 format!("{}-source-check", restore.name_any())
1577}
1578
1579async fn ensure_source_check_job(
1580 restore: &KanidmRestore,
1581 target: &Kanidm,
1582 ctx: &RestoreContext,
1583 name: &str,
1584) -> Result<()> {
1585 let ns = restore.namespace().unwrap();
1586 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1587 if api
1588 .get_opt(name)
1589 .await
1590 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1591 .is_some()
1592 {
1593 return Ok(());
1594 }
1595 let file_name = restore
1596 .spec
1597 .source
1598 .local
1599 .as_ref()
1600 .map(|l| l.file_name.clone())
1601 .unwrap_or_default();
1602 let operation_doc = serde_json::json!({
1603 "apiVersion": "backup.kaniop.rs/v1alpha1",
1604 "kind": "OperationDocument",
1605 "operation": "check",
1606 "path": format!("{BACKUP_PATH}/{file_name}"),
1607 "resultPath": "/run/kaniop-result/result.json",
1608 "format": "kanidmJsonGzip",
1609 })
1610 .to_string();
1611 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1612 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1613 let data_mover_image = data_mover_image();
1614 let job = Job {
1615 metadata: ObjectMeta {
1616 name: Some(name.to_string()),
1617 namespace: Some(ns.clone()),
1618 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1619 ..Default::default()
1620 },
1621 spec: Some(JobSpec {
1622 backoff_limit: Some(0),
1623 template: PodTemplateSpec {
1624 metadata: Some(ObjectMeta::default()),
1625 spec: Some(PodSpec {
1626 automount_service_account_token: Some(false),
1627 restart_policy: Some("Never".to_string()),
1628 security_context: Some(source_job_pod_security_context(target)),
1629 containers: vec![Container {
1630 name: "source-check".to_string(),
1631 image: Some(data_mover_image),
1632 command: Some(vec![
1633 "/bin/kaniop-data-mover".to_string(),
1634 "check".to_string(),
1635 "--operation-doc".to_string(),
1636 "/run/kaniop/operation.json".to_string(),
1637 ]),
1638 security_context: Some(hardened_security_context()),
1639 resources: Some(job_resource_requirements()),
1640 env: Some(vec![EnvVar {
1641 name: "RUST_LOG".to_string(),
1642 value: Some("info".to_string()),
1643 ..Default::default()
1644 }]),
1645 volume_mounts: Some(vec![
1646 VolumeMount {
1647 name: DATA_VOLUME.to_string(),
1648 mount_path: DATA_PATH.to_string(),
1649 read_only: Some(true),
1650 ..Default::default()
1651 },
1652 VolumeMount {
1653 name: "operation".to_string(),
1654 mount_path: "/run/kaniop".to_string(),
1655 read_only: Some(true),
1656 ..Default::default()
1657 },
1658 VolumeMount {
1659 name: "result".to_string(),
1660 mount_path: "/run/kaniop-result".to_string(),
1661 ..Default::default()
1662 },
1663 ]),
1664 ..Default::default()
1665 }],
1666 volumes: Some(vec![
1667 Volume {
1668 name: DATA_VOLUME.to_string(),
1669 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1670 claim_name: primary_pvc_name(target)?,
1671 read_only: Some(true),
1672 }),
1673 ..Default::default()
1674 },
1675 Volume {
1676 name: "operation".to_string(),
1677 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1678 name: operation_cm_name,
1679 ..Default::default()
1680 }),
1681 ..Default::default()
1682 },
1683 Volume {
1684 name: "result".to_string(),
1685 empty_dir: Some(Default::default()),
1686 ..Default::default()
1687 },
1688 ]),
1689 ..Default::default()
1690 }),
1691 },
1692 ..Default::default()
1693 }),
1694 ..Default::default()
1695 };
1696 api.create(&PostParams::default(), &job)
1697 .await
1698 .map(|_| ())
1699 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1700}
1701
1702const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1703
1704fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1705 let ns = restore.namespace().unwrap_or_default();
1706 let name = restore.name_any();
1707 let namespace_uuid =
1708 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1709 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1710 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1711}
1712
1713fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1714 let ns = restore.namespace().unwrap_or_default();
1715 let kanidm_uid = &restore.spec.target_ref.uid;
1716 let backup_id = compute_safety_backup_id(restore);
1717 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1718}
1719
1720struct VerifiedSafetyBackup {
1721 manifest_key: String,
1722 payload_sha256: String,
1723}
1724
1725async fn read_safety_backup_result(
1726 restore: &KanidmRestore,
1727 ctx: &RestoreContext,
1728 job_name: &str,
1729 expected_backup_id: &str,
1730) -> Result<VerifiedSafetyBackup> {
1731 let ns = restore.namespace().unwrap();
1732 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1733 let pod_list = pods
1734 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1735 .await
1736 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1737 let pod = pod_list.items.first().ok_or_else(|| {
1738 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1739 })?;
1740 let pod_name = pod.name_any();
1741 let container_status = pod
1742 .status
1743 .as_ref()
1744 .and_then(|s| s.container_statuses.as_ref())
1745 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1746 .ok_or_else(|| {
1747 Error::MissingData(format!(
1748 "safety-upload container status not found in pod {pod_name}"
1749 ))
1750 })?;
1751 let termination_message = container_status
1752 .state
1753 .as_ref()
1754 .and_then(|state| state.terminated.as_ref())
1755 .and_then(|t| t.message.as_ref())
1756 .ok_or_else(|| {
1757 Error::MissingData(format!(
1758 "termination message not found in pod {pod_name}; result transport failed"
1759 ))
1760 })?;
1761 if termination_message.is_empty() {
1762 return Err(Error::MissingData(format!(
1763 "termination message is empty in pod {pod_name}; result document was not written"
1764 )));
1765 }
1766 let result_doc = parse_result_document(termination_message).map_err(|e| {
1767 Error::ParseError(format!(
1768 "failed to parse result document from pod {pod_name}: {e}"
1769 ))
1770 })?;
1771 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1772 return Err(Error::ParseError(format!(
1773 "unsupported result document version: {}",
1774 result_doc.api_version
1775 )));
1776 }
1777 if result_doc.kind != "ResultDocument" {
1778 return Err(Error::ParseError(format!(
1779 "invalid result document kind: {}",
1780 result_doc.kind
1781 )));
1782 }
1783 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1784 return Err(Error::ParseError(format!(
1785 "result document operation mismatch: expected '{}', got '{}'",
1786 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1787 )));
1788 }
1789 if !result_doc.success {
1790 return Err(Error::ParseError(format!(
1791 "safety backup reported failure with exit code {:?}",
1792 result_doc.exit_code
1793 )));
1794 }
1795 if result_doc.exit_code != ExitCode::Success {
1796 return Err(Error::ParseError(format!(
1797 "safety backup exit code is not Success: {:?}",
1798 result_doc.exit_code
1799 )));
1800 }
1801 let doc_backup_id = result_doc
1802 .backup_id
1803 .as_deref()
1804 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1805 if doc_backup_id != expected_backup_id {
1806 return Err(Error::ParseError(format!(
1807 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1808 )));
1809 }
1810 let manifest_key = result_doc
1811 .manifest_key
1812 .clone()
1813 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1814 if manifest_key.is_empty() {
1815 return Err(Error::ParseError(
1816 "result document manifestKey is empty".to_string(),
1817 ));
1818 }
1819 let payload_sha256 = result_doc
1820 .payload_sha256
1821 .clone()
1822 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1823 if payload_sha256.is_empty() {
1824 return Err(Error::ParseError(
1825 "result document payloadSha256 is empty".to_string(),
1826 ));
1827 }
1828 info!(
1829 restore = %restore.name_any(),
1830 backup_id = %expected_backup_id,
1831 manifest_key = %manifest_key,
1832 "safety backup result verified"
1833 );
1834 Ok(VerifiedSafetyBackup {
1835 manifest_key,
1836 payload_sha256,
1837 })
1838}
1839
1840struct VerifiedSourcePrepResult {
1841 #[allow(dead_code)]
1842 manifest_key: String,
1843 payload_sha256: String,
1844}
1845
1846async fn read_source_prep_result(
1847 restore: &KanidmRestore,
1848 ctx: &RestoreContext,
1849 job_name: &str,
1850 expected_backup_id: &str,
1851) -> Result<VerifiedSourcePrepResult> {
1852 let ns = restore.namespace().unwrap();
1853 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1854 let pod_list = pods
1855 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1856 .await
1857 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1858 let pod = pod_list.items.first().ok_or_else(|| {
1859 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1860 })?;
1861 let pod_name = pod.name_any();
1862 let container_status = pod
1863 .status
1864 .as_ref()
1865 .and_then(|s| s.container_statuses.as_ref())
1866 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1867 .ok_or_else(|| {
1868 Error::MissingData(format!(
1869 "source-prep container status not found in pod {pod_name}"
1870 ))
1871 })?;
1872 let termination_message = container_status
1873 .state
1874 .as_ref()
1875 .and_then(|state| state.terminated.as_ref())
1876 .and_then(|t| t.message.as_ref())
1877 .ok_or_else(|| {
1878 Error::MissingData(format!(
1879 "termination message not found in pod {pod_name}; result transport failed"
1880 ))
1881 })?;
1882 if termination_message.is_empty() {
1883 return Err(Error::MissingData(format!(
1884 "termination message is empty in pod {pod_name}; result document was not written"
1885 )));
1886 }
1887 let result_doc = parse_result_document(termination_message).map_err(|e| {
1888 Error::ParseError(format!(
1889 "failed to parse result document from pod {pod_name}: {e}"
1890 ))
1891 })?;
1892 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1893 return Err(Error::ParseError(format!(
1894 "unsupported result document version: {}",
1895 result_doc.api_version
1896 )));
1897 }
1898 if result_doc.kind != "ResultDocument" {
1899 return Err(Error::ParseError(format!(
1900 "invalid result document kind: {}",
1901 result_doc.kind
1902 )));
1903 }
1904 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1905 return Err(Error::ParseError(format!(
1906 "result document operation mismatch: expected '{}', got '{}'",
1907 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1908 )));
1909 }
1910 if !result_doc.success {
1911 return Err(Error::ParseError(format!(
1912 "source prep reported failure with exit code {:?}",
1913 result_doc.exit_code
1914 )));
1915 }
1916 if result_doc.exit_code != ExitCode::Success {
1917 return Err(Error::ParseError(format!(
1918 "source prep exit code is not Success: {:?}",
1919 result_doc.exit_code
1920 )));
1921 }
1922 let doc_backup_id = result_doc
1923 .backup_id
1924 .as_deref()
1925 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1926 if doc_backup_id != expected_backup_id {
1927 return Err(Error::ParseError(format!(
1928 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1929 )));
1930 }
1931 let manifest_key = result_doc
1932 .manifest_key
1933 .clone()
1934 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1935 if manifest_key.is_empty() {
1936 return Err(Error::ParseError(
1937 "result document manifestKey is empty".to_string(),
1938 ));
1939 }
1940 let payload_sha256 = result_doc
1941 .payload_sha256
1942 .clone()
1943 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1944 if payload_sha256.is_empty() {
1945 return Err(Error::ParseError(
1946 "result document payloadSha256 is empty".to_string(),
1947 ));
1948 }
1949 info!(
1950 restore = %restore.name_any(),
1951 backup_id = %expected_backup_id,
1952 manifest_key = %manifest_key,
1953 payload_sha256 = %payload_sha256,
1954 "source prep result verified"
1955 );
1956 Ok(VerifiedSourcePrepResult {
1957 manifest_key,
1958 payload_sha256,
1959 })
1960}
1961
1962fn source_job_pod_security_context(
1963 target: &Kanidm,
1964) -> k8s_openapi::api::core::v1::PodSecurityContext {
1965 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
1966 security_context.seccomp_profile = Some(SeccompProfile {
1967 type_: "RuntimeDefault".to_string(),
1968 ..Default::default()
1969 });
1970 security_context
1971}
1972
1973fn hardened_security_context() -> SecurityContext {
1974 SecurityContext {
1975 allow_privilege_escalation: Some(false),
1976 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
1977 drop: Some(vec!["ALL".to_string()]),
1978 ..Default::default()
1979 }),
1980 read_only_root_filesystem: Some(true),
1981 run_as_non_root: Some(true),
1982 seccomp_profile: Some(SeccompProfile {
1983 type_: "RuntimeDefault".to_string(),
1984 ..Default::default()
1985 }),
1986 ..Default::default()
1987 }
1988}
1989
1990fn kanidm_job_security_context() -> SecurityContext {
1991 SecurityContext {
1992 allow_privilege_escalation: Some(false),
1993 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
1994 drop: Some(vec!["ALL".to_string()]),
1995 ..Default::default()
1996 }),
1997 seccomp_profile: Some(SeccompProfile {
1998 type_: "RuntimeDefault".to_string(),
1999 ..Default::default()
2000 }),
2001 ..Default::default()
2002 }
2003}
2004
2005fn job_resource_requirements() -> ResourceRequirements {
2006 ResourceRequirements {
2007 requests: Some(std::collections::BTreeMap::from([
2008 ("cpu".to_string(), Quantity("100m".to_string())),
2009 ("memory".to_string(), Quantity("128Mi".to_string())),
2010 ])),
2011 limits: Some(std::collections::BTreeMap::from([
2012 ("cpu".to_string(), Quantity("2".to_string())),
2013 ("memory".to_string(), Quantity("2Gi".to_string())),
2014 ])),
2015 ..Default::default()
2016 }
2017}
2018
2019async fn ensure_safety_backup_job(
2020 restore: &KanidmRestore,
2021 target: &Kanidm,
2022 ctx: &RestoreContext,
2023 name: &str,
2024 backup_id: &str,
2025 manifest_key: &str,
2026) -> Result<()> {
2027 let ns = restore.namespace().unwrap();
2028 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2029 if api
2030 .get_opt(name)
2031 .await
2032 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2033 .is_some()
2034 {
2035 return Ok(());
2036 }
2037 let safety = restore
2038 .spec
2039 .safety_backup
2040 .as_ref()
2041 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2042 let repo_ref = safety
2043 .repository_ref
2044 .as_ref()
2045 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2046 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2047 .get(&repo_ref.name)
2048 .await
2049 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2050 let _namespace_uid = &ns;
2051 let _kanidm_uid = &restore.spec.target_ref.uid;
2052 let endpoint = &repo.spec.s3.endpoint;
2053 let region = &repo.spec.s3.region;
2054 let operation_doc = build_safety_upload_operation_doc(
2055 restore,
2056 target,
2057 backup_id,
2058 manifest_key,
2059 &repo.spec.s3.bucket,
2060 &repo.spec.s3.prefix,
2061 endpoint,
2062 region,
2063 repo.spec.s3.force_path_style,
2064 repo.spec.s3.insecure,
2065 repo.spec
2066 .s3
2067 .ca_bundle_ref
2068 .as_deref()
2069 .map(|_| ca_bundle_path())
2070 .as_deref(),
2071 repo.spec.encryption.as_ref(),
2072 )?;
2073 let operation_cm_name = format!("{name}-op");
2074 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2075 let auth_method = &repo.spec.authentication.writer;
2076 let data_mover_image = data_mover_image();
2077 let init_command = vec![
2078 "kanidmd".to_string(),
2079 "database".to_string(),
2080 "backup".to_string(),
2081 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2082 ];
2083 let upload_script = format!(
2084 r#"set -eu
2085/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2086RESULT_FILE="/run/kaniop-result/result.json"
2087TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2088if [ ! -f "$RESULT_FILE" ]; then
2089 echo "ERROR: result document not found at $RESULT_FILE"
2090 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2091 exit 1
2092fi
2093if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2094 echo "ERROR: safety backup upload reported failure"
2095 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2096 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2097 exit 1
2098fi
2099dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2100echo "safety backup upload completed successfully"
2101"#
2102 );
2103 let job = Job {
2104 metadata: ObjectMeta {
2105 name: Some(name.to_string()),
2106 namespace: Some(ns.clone()),
2107 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2108 ..Default::default()
2109 },
2110 spec: Some(JobSpec {
2111 backoff_limit: Some(0),
2112 template: PodTemplateSpec {
2113 metadata: Some(ObjectMeta::default()),
2114 spec: Some(PodSpec {
2115 automount_service_account_token: Some(false),
2116 restart_policy: Some("Never".to_string()),
2117 security_context: Some(source_job_pod_security_context(target)),
2118 init_containers: Some(vec![Container {
2119 name: "safety-backup".to_string(),
2120 image: Some(restore.spec.restore_image.clone()),
2121 command: Some(init_command),
2122 security_context: Some(SecurityContext {
2123 allow_privilege_escalation: Some(false),
2124 capabilities: Some(Capabilities {
2125 drop: Some(vec!["ALL".to_string()]),
2126 ..Default::default()
2127 }),
2128 read_only_root_filesystem: Some(true),
2129 seccomp_profile: Some(SeccompProfile {
2130 type_: "RuntimeDefault".to_string(),
2131 ..Default::default()
2132 }),
2133 ..Default::default()
2134 }),
2135 resources: Some(job_resource_requirements()),
2136 env: Some(build_kanidm_db_env_vars(target)),
2137 volume_mounts: Some(vec![
2138 VolumeMount {
2139 name: DATA_VOLUME.to_string(),
2140 mount_path: DATA_PATH.to_string(),
2141 read_only: Some(false),
2142 ..Default::default()
2143 },
2144 VolumeMount {
2145 name: TLS_VOLUME.to_string(),
2146 mount_path: TLS_PATH.to_string(),
2147 read_only: Some(true),
2148 ..Default::default()
2149 },
2150 VolumeMount {
2151 name: SHARED_VOLUME.to_string(),
2152 mount_path: SHARED_VOL_PATH.to_string(),
2153 ..Default::default()
2154 },
2155 ]),
2156 ..Default::default()
2157 }]),
2158 containers: vec![Container {
2159 name: "safety-upload".to_string(),
2160 image: Some(data_mover_image),
2161 command: Some(vec!["/bin/sh".to_string()]),
2162 args: Some(vec!["-c".to_string(), upload_script]),
2163 security_context: Some(hardened_security_context()),
2164 resources: Some(job_resource_requirements()),
2165 env: {
2166 let mut env_vars =
2167 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2168 env_vars.push(EnvVar {
2169 name: "RUST_LOG".to_string(),
2170 value: Some("info".to_string()),
2171 ..Default::default()
2172 });
2173 if repo.spec.s3.ca_bundle_ref.is_some() {
2174 env_vars.push(ca_bundle_env_var());
2175 }
2176 env_vars.extend(build_encryption_env_vars(
2177 repo.spec
2178 .encryption
2179 .as_ref()
2180 .and_then(|e| e.key_ref.as_ref()),
2181 ));
2182 Some(env_vars)
2183 },
2184 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2185 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2186 volume_mounts: {
2187 let mut mounts = vec![
2188 VolumeMount {
2189 name: SHARED_VOLUME.to_string(),
2190 mount_path: SHARED_VOL_PATH.to_string(),
2191 read_only: Some(true),
2192 ..Default::default()
2193 },
2194 VolumeMount {
2195 name: "operation".to_string(),
2196 mount_path: "/run/kaniop".to_string(),
2197 read_only: Some(true),
2198 ..Default::default()
2199 },
2200 VolumeMount {
2201 name: "result".to_string(),
2202 mount_path: "/run/kaniop-result".to_string(),
2203 ..Default::default()
2204 },
2205 ];
2206 mounts.extend(build_auth_volume_mounts(auth_method));
2207 if repo.spec.s3.ca_bundle_ref.is_some() {
2208 mounts.push(build_ca_bundle_volume_mount());
2209 }
2210 Some(mounts)
2211 },
2212 ..Default::default()
2213 }],
2214 volumes: {
2215 let mut vols = vec![
2216 Volume {
2217 name: DATA_VOLUME.to_string(),
2218 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2219 claim_name: primary_pvc_name(target)?,
2220 read_only: Some(false),
2221 }),
2222 ..Default::default()
2223 },
2224 Volume {
2225 name: TLS_VOLUME.to_string(),
2226 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2227 secret_name: Some(target.effective_tls_secret_name()),
2228 ..Default::default()
2229 }),
2230 ..Default::default()
2231 },
2232 Volume {
2233 name: SHARED_VOLUME.to_string(),
2234 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2235 size_limit: Some(crate::controller::backup_job_volume_size()),
2236 ..Default::default()
2237 }),
2238 ..Default::default()
2239 },
2240 Volume {
2241 name: "operation".to_string(),
2242 config_map: Some(ConfigMapVolumeSource {
2243 name: operation_cm_name,
2244 ..Default::default()
2245 }),
2246 ..Default::default()
2247 },
2248 Volume {
2249 name: "result".to_string(),
2250 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2251 size_limit: Some(Quantity("16Mi".to_string())),
2252 ..Default::default()
2253 }),
2254 ..Default::default()
2255 },
2256 ];
2257 vols.extend(build_auth_volumes(auth_method));
2258 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2259 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2260 }
2261 Some(vols)
2262 },
2263 ..Default::default()
2264 }),
2265 },
2266 ..Default::default()
2267 }),
2268 ..Default::default()
2269 };
2270 api.create(&PostParams::default(), &job)
2271 .await
2272 .map(|_| ())
2273 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2274}
2275
2276async fn ensure_source_prep_job(
2277 restore: &KanidmRestore,
2278 target: &Kanidm,
2279 ctx: &RestoreContext,
2280 name: &str,
2281) -> Result<()> {
2282 let ns = restore.namespace().unwrap();
2283 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2284 if api
2285 .get_opt(name)
2286 .await
2287 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2288 .is_some()
2289 {
2290 return Ok(());
2291 }
2292 let backup_name = restore
2293 .spec
2294 .source
2295 .backup_ref
2296 .as_ref()
2297 .map(|b| b.name.clone())
2298 .unwrap_or_default();
2299 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2300 .get(&backup_name)
2301 .await
2302 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2303 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2304 .get(&backup.spec.repository_ref.name)
2305 .await
2306 .map_err(|e| {
2307 Error::kube_error(
2308 "get",
2309 "KanidmBackupRepository",
2310 &ns,
2311 &backup.spec.repository_ref.name,
2312 e,
2313 )
2314 })?;
2315 let endpoint = &repo.spec.s3.endpoint;
2316 let region = &repo.spec.s3.region;
2317 let operation_doc = build_download_operation_doc(
2318 restore,
2319 target,
2320 &backup.spec.manifest_key,
2321 &backup.spec.backup_id,
2322 &repo.spec.s3.bucket,
2323 &repo.spec.s3.prefix,
2324 endpoint,
2325 region,
2326 repo.spec.s3.force_path_style,
2327 repo.spec.s3.insecure,
2328 repo.spec
2329 .s3
2330 .ca_bundle_ref
2331 .as_deref()
2332 .map(|_| ca_bundle_path())
2333 .as_deref(),
2334 repo.spec.encryption.as_ref(),
2335 );
2336 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2337 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2338 let auth_method = &repo.spec.authentication.reader;
2339 let data_mover_image = data_mover_image();
2340 let download_script = format!(
2341 r#"set -eu
2342/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2343RESULT_FILE="/run/kaniop-result/result.json"
2344TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2345if [ ! -f "$RESULT_FILE" ]; then
2346 echo "ERROR: result document not found at $RESULT_FILE"
2347 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2348 exit 1
2349fi
2350if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2351 echo "ERROR: source preparation download reported failure"
2352 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2353 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2354 exit 1
2355fi
2356dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2357echo "source preparation download completed successfully"
2358"#
2359 );
2360 let job = Job {
2361 metadata: ObjectMeta {
2362 name: Some(name.to_string()),
2363 namespace: Some(ns.clone()),
2364 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2365 ..Default::default()
2366 },
2367 spec: Some(JobSpec {
2368 backoff_limit: Some(0),
2369 template: PodTemplateSpec {
2370 metadata: Some(ObjectMeta::default()),
2371 spec: Some(PodSpec {
2372 automount_service_account_token: Some(false),
2373 restart_policy: Some("Never".to_string()),
2374 security_context: Some(source_job_pod_security_context(target)),
2375 containers: vec![Container {
2376 name: "source-prep".to_string(),
2377 image: Some(data_mover_image),
2378 command: Some(vec!["/bin/sh".to_string()]),
2379 args: Some(vec!["-c".to_string(), download_script]),
2380 security_context: Some(hardened_security_context()),
2381 resources: Some(job_resource_requirements()),
2382 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2383 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2384 env: {
2385 let mut env_vars = build_auth_env_vars(
2386 auth_method,
2387 &backup.spec.repository_ref.name,
2388 AuthRole::Reader,
2389 );
2390 env_vars.push(EnvVar {
2391 name: "RUST_LOG".to_string(),
2392 value: Some("info".to_string()),
2393 ..Default::default()
2394 });
2395 if repo.spec.s3.ca_bundle_ref.is_some() {
2396 env_vars.push(ca_bundle_env_var());
2397 }
2398 env_vars.extend(build_encryption_env_vars(
2399 repo.spec
2400 .encryption
2401 .as_ref()
2402 .and_then(|e| e.key_ref.as_ref()),
2403 ));
2404 Some(env_vars)
2405 },
2406 volume_mounts: {
2407 let mut mounts = vec![
2408 VolumeMount {
2409 name: DATA_VOLUME.to_string(),
2410 mount_path: DATA_PATH.to_string(),
2411 ..Default::default()
2412 },
2413 VolumeMount {
2414 name: "operation".to_string(),
2415 mount_path: "/run/kaniop".to_string(),
2416 read_only: Some(true),
2417 ..Default::default()
2418 },
2419 VolumeMount {
2420 name: "result".to_string(),
2421 mount_path: "/run/kaniop-result".to_string(),
2422 ..Default::default()
2423 },
2424 ];
2425 mounts.extend(build_auth_volume_mounts(auth_method));
2426 if repo.spec.s3.ca_bundle_ref.is_some() {
2427 mounts.push(build_ca_bundle_volume_mount());
2428 }
2429 Some(mounts)
2430 },
2431 ..Default::default()
2432 }],
2433 volumes: {
2434 let mut vols = vec![
2435 Volume {
2436 name: DATA_VOLUME.to_string(),
2437 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2438 claim_name: primary_pvc_name(target)?,
2439 read_only: Some(false),
2440 }),
2441 ..Default::default()
2442 },
2443 Volume {
2444 name: "operation".to_string(),
2445 config_map: Some(ConfigMapVolumeSource {
2446 name: operation_cm_name,
2447 ..Default::default()
2448 }),
2449 ..Default::default()
2450 },
2451 Volume {
2452 name: "result".to_string(),
2453 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2454 size_limit: Some(Quantity("16Mi".to_string())),
2455 ..Default::default()
2456 }),
2457 ..Default::default()
2458 },
2459 ];
2460 vols.extend(build_auth_volumes(auth_method));
2461 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2462 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2463 }
2464 Some(vols)
2465 },
2466 ..Default::default()
2467 }),
2468 },
2469 ..Default::default()
2470 }),
2471 ..Default::default()
2472 };
2473 api.create(&PostParams::default(), &job)
2474 .await
2475 .map(|_| ())
2476 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2477}
2478
2479#[allow(clippy::too_many_arguments)]
2480fn build_safety_upload_operation_doc(
2481 restore: &KanidmRestore,
2482 target: &Kanidm,
2483 backup_id: &str,
2484 manifest_key: &str,
2485 bucket: &str,
2486 prefix: &str,
2487 endpoint: &str,
2488 region: &str,
2489 force_path_style: bool,
2490 insecure: bool,
2491 ca_bundle_path: Option<&str>,
2492 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2493) -> Result<String> {
2494 let enc_mode =
2495 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2496 let enc_key_id = encryption
2497 .and_then(|e| e.key_id.as_ref())
2498 .map(|s| serde_json::Value::String(s.clone()));
2499 let op = UploadOperation {
2500 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2501 bucket: bucket.to_string(),
2502 prefix: prefix.to_string(),
2503 endpoint: endpoint.to_string(),
2504 region: region.to_string(),
2505 force_path_style,
2506 insecure,
2507 ca_bundle_path: ca_bundle_path.map(str::to_string),
2508 backup_id: backup_id.to_string(),
2509 namespace_uid: restore.namespace().unwrap_or_default(),
2510 kanidm_uid: restore.spec.target_ref.uid.clone(),
2511 kanidm_name: restore.spec.target_ref.name.clone(),
2512 domain: target.spec.domain.clone(),
2513 kanidm_version: target
2514 .status
2515 .as_ref()
2516 .and_then(|s| s.version.as_ref())
2517 .map(|v| v.image_tag.clone())
2518 .unwrap_or_default(),
2519 image_digest: None,
2520 consistency: "kanidm-offline".to_string(),
2521 reason: "restore-safety".to_string(),
2522 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2523 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2524 result_path: "/run/kaniop-result/result.json".to_string(),
2525 max_concurrent_parts: 4,
2526 max_retries: 3,
2527 };
2528 let _ = manifest_key;
2529 let doc = OperationDocument {
2530 api_version: OPERATION_DOC_VERSION.to_string(),
2531 kind: "OperationDocument".to_string(),
2532 spec: OperationSpec::Upload(op),
2533 };
2534 serde_json::to_string(&doc)
2535 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2536}
2537
2538async fn validate_backup_ref(
2539 restore: &KanidmRestore,
2540 target: &Kanidm,
2541 ctx: &RestoreContext,
2542) -> Result<()> {
2543 let backup_ref = restore
2544 .spec
2545 .source
2546 .backup_ref
2547 .as_ref()
2548 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2549 let ns = restore.namespace().unwrap();
2550 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2551 .get(&backup_ref.name)
2552 .await
2553 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2554 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2555 return Err(Error::MissingData(format!(
2556 "KanidmBackup '{}' is not Ready (phase: {:?})",
2557 backup_ref.name,
2558 backup.status.as_ref().map(|s| &s.phase)
2559 )));
2560 }
2561 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2562 return Err(Error::MissingData(format!(
2563 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2564 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2565 )));
2566 }
2567 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2568 return Err(Error::MissingData(format!(
2569 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2570 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2571 )));
2572 }
2573 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2574 .get(&backup.spec.repository_ref.name)
2575 .await
2576 .map_err(|e| {
2577 Error::kube_error(
2578 "get",
2579 "KanidmBackupRepository",
2580 &ns,
2581 &backup.spec.repository_ref.name,
2582 e,
2583 )
2584 })?;
2585 if !has_accepted_condition(
2586 &repo
2587 .status
2588 .as_ref()
2589 .map(|s| &s.conditions)
2590 .cloned()
2591 .unwrap_or_default(),
2592 ) {
2593 return Err(Error::MissingData(format!(
2594 "KanidmBackupRepository '{}' configuration has not been accepted",
2595 backup.spec.repository_ref.name
2596 )));
2597 }
2598 validate_backup_compatibility(restore, target, &backup)?;
2599 Ok(())
2600}
2601
2602fn has_accepted_condition(conditions: &[Condition]) -> bool {
2603 conditions
2604 .iter()
2605 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2606}
2607
2608fn validate_backup_compatibility(
2609 restore: &KanidmRestore,
2610 target: &Kanidm,
2611 backup: &KanidmBackup,
2612) -> Result<()> {
2613 let status = backup.status.as_ref();
2614 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2615 let target_version = target
2616 .status
2617 .as_ref()
2618 .and_then(|s| s.version.as_ref())
2619 .map(|v| v.image_tag.as_str());
2620 if let Some(tv) = target_version {
2621 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2622 return Err(Error::MissingData(format!(
2623 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2624 )));
2625 }
2626 }
2627 }
2628 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2629 if !backup_digest.is_empty()
2630 && restore.spec.restore_image.contains('@')
2631 && !restore.spec.restore_image.contains(backup_digest)
2632 {
2633 return Err(Error::MissingData(format!(
2634 "restore image digest does not match backup image digest '{backup_digest}'"
2635 )));
2636 }
2637 }
2638 Ok(())
2639}
2640
2641#[allow(clippy::too_many_arguments)]
2642fn build_download_operation_doc(
2643 restore: &KanidmRestore,
2644 target: &Kanidm,
2645 manifest_key: &str,
2646 expected_backup_id: &str,
2647 bucket: &str,
2648 prefix: &str,
2649 endpoint: &str,
2650 region: &str,
2651 force_path_style: bool,
2652 insecure: bool,
2653 ca_bundle_path: Option<&str>,
2654 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2655) -> String {
2656 let enc_mode =
2657 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2658 let enc_key_id = encryption
2659 .and_then(|e| e.key_id.as_ref())
2660 .map(|s| serde_json::Value::String(s.clone()));
2661 serde_json::json!({
2662 "apiVersion": "backup.kaniop.rs/v1alpha1",
2663 "kind": "OperationDocument",
2664 "operation": "download",
2665 "manifestKey": manifest_key,
2666 "bucket": bucket,
2667 "prefix": prefix,
2668 "endpoint": endpoint,
2669 "region": region,
2670 "forcePathStyle": force_path_style,
2671 "insecure": insecure,
2672 "caBundlePath": ca_bundle_path,
2673 "expectedBackupId": expected_backup_id,
2674 "expectedKanidmUid": restore.spec.target_ref.uid,
2675 "expectedDomain": target.spec.domain,
2676 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2677 "resultPath": "/run/kaniop-result/result.json",
2678 "maxRetries": 3,
2679 "encryptionMode": enc_mode,
2680 "encryptionKeyId": enc_key_id,
2681 })
2682 .to_string()
2683}
2684
2685async fn ensure_operation_configmap(
2686 restore: &KanidmRestore,
2687 name: &str,
2688 data: &str,
2689 ns: &str,
2690 ctx: &RestoreContext,
2691) -> Result<()> {
2692 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2693 if api
2694 .get_opt(name)
2695 .await
2696 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2697 .is_some()
2698 {
2699 return Ok(());
2700 }
2701 let cm = ConfigMap {
2702 metadata: ObjectMeta {
2703 name: Some(name.to_string()),
2704 namespace: Some(ns.to_string()),
2705 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2706 ..Default::default()
2707 },
2708 data: Some(std::collections::BTreeMap::from([(
2709 "operation.json".to_string(),
2710 data.to_string(),
2711 )])),
2712 ..Default::default()
2713 };
2714 api.create(&PostParams::default(), &cm)
2715 .await
2716 .map(|_| ())
2717 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2718}
2719
2720#[cfg(test)]
2721mod tests {
2722 use super::{
2723 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2724 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2725 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2726 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2727 hardened_security_context, has_accepted_condition, is_remote_source,
2728 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2729 validate_safety_backup_config, validate_source,
2730 };
2731 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2732 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2733 use kube::api::ObjectMeta as ApiObjectMeta;
2734 use std::collections::BTreeMap;
2735
2736 fn make_restore(
2737 source: KanidmRestoreSource,
2738 safety: Option<SafetyBackupConfig>,
2739 ) -> KanidmRestore {
2740 KanidmRestore {
2741 metadata: ApiObjectMeta {
2742 name: Some("test-restore".to_string()),
2743 namespace: Some("default".to_string()),
2744 ..Default::default()
2745 },
2746 spec: KanidmRestoreSpec {
2747 target_ref: KanidmRestoreTargetRef {
2748 name: "corp-idm".to_string(),
2749 uid: "test-uid-123".to_string(),
2750 },
2751 source,
2752 restore_image: "kanidm/server@sha256:abc".to_string(),
2753 safety_backup: safety,
2754 },
2755 status: None,
2756 }
2757 }
2758
2759 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2760 super::super::crd::Kanidm {
2761 status: Some(super::super::crd::KanidmStatus {
2762 available_replicas: 1,
2763 replicas: 1,
2764 unavailable_replicas: 0,
2765 updated_replicas: 1,
2766 replica_statuses: vec![],
2767 replica_column: "1/1".to_string(),
2768 secret_name: None,
2769 version: Some(super::super::crd::KanidmVersionStatus {
2770 image_tag: image_tag.to_string(),
2771 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2772 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2773 }),
2774 domain_appearance_image: None,
2775 mail_sender: None,
2776 conditions: None,
2777 }),
2778 ..Default::default()
2779 }
2780 }
2781
2782 #[test]
2783 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2784 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2785 }
2786
2787 #[test]
2788 fn completed_restore_sets_ready_condition() {
2789 let mut status = KanidmRestoreStatus {
2790 phase: KanidmRestorePhase::Completed,
2791 ..Default::default()
2792 };
2793 super::update_restore_conditions(&mut status, Some(7));
2794 assert!(status.conditions.iter().any(|condition| {
2795 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2796 }));
2797 assert!(!status.conditions.iter().any(|condition| {
2798 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2799 }));
2800 }
2801
2802 #[test]
2803 fn failed_restore_sets_failed_condition() {
2804 let mut status = KanidmRestoreStatus {
2805 phase: KanidmRestorePhase::Failed,
2806 message: Some("verification failed".to_string()),
2807 ..Default::default()
2808 };
2809 super::update_restore_conditions(&mut status, Some(8));
2810 assert!(status.conditions.iter().any(|condition| {
2811 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2812 }));
2813 }
2814
2815 #[test]
2816 fn restore_filename_is_confined_to_backup_directory() {
2817 assert!(safe_basename("backup.json.gz"));
2818 assert!(!safe_basename("../backup.json.gz"));
2819 assert!(!safe_basename("a/b"));
2820 assert!(!safe_basename(""));
2821 }
2822
2823 #[test]
2824 fn restore_image_rejects_latest_and_unpinned_names() {
2825 assert!(mutable_image("kanidm/server:latest"));
2826 assert!(mutable_image("kanidm/server"));
2827 assert!(!mutable_image("kanidm/server:1.10.0"));
2828 assert!(!mutable_image("kanidm/server@sha256:abc"));
2829 }
2830
2831 #[test]
2832 fn source_validation_rejects_both_local_and_backup_ref() {
2833 let source = KanidmRestoreSource {
2834 local: Some(super::KanidmRestoreLocalSource {
2835 file_name: "backup.json".to_string(),
2836 }),
2837 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2838 name: "some-backup".to_string(),
2839 }),
2840 };
2841 let restore = make_restore(source, None);
2842 assert!(validate_source(&restore).is_err());
2843 }
2844
2845 #[test]
2846 fn source_validation_rejects_empty_source() {
2847 let source = KanidmRestoreSource {
2848 local: None,
2849 backup_ref: None,
2850 };
2851 let restore = make_restore(source, None);
2852 assert!(validate_source(&restore).is_err());
2853 }
2854
2855 #[test]
2856 fn source_validation_accepts_local_source() {
2857 let source = KanidmRestoreSource {
2858 local: Some(super::KanidmRestoreLocalSource {
2859 file_name: "backup.json".to_string(),
2860 }),
2861 backup_ref: None,
2862 };
2863 let restore = make_restore(source, None);
2864 assert!(validate_source(&restore).is_ok());
2865 }
2866
2867 #[test]
2868 fn source_validation_accepts_backup_ref_source() {
2869 let source = KanidmRestoreSource {
2870 local: None,
2871 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2872 name: "corp-idm-019c7c76".to_string(),
2873 }),
2874 };
2875 let restore = make_restore(source, None);
2876 assert!(validate_source(&restore).is_ok());
2877 }
2878
2879 #[test]
2880 fn source_validation_rejects_empty_backup_ref_name() {
2881 let source = KanidmRestoreSource {
2882 local: None,
2883 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2884 name: "".to_string(),
2885 }),
2886 };
2887 let restore = make_restore(source, None);
2888 assert!(validate_source(&restore).is_err());
2889 }
2890
2891 #[test]
2892 fn is_remote_source_true_for_backup_ref() {
2893 let source = KanidmRestoreSource {
2894 local: None,
2895 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2896 name: "backup-1".to_string(),
2897 }),
2898 };
2899 let restore = make_restore(source, None);
2900 assert!(is_remote_source(&restore));
2901 }
2902
2903 #[test]
2904 fn is_remote_source_false_for_local() {
2905 let source = KanidmRestoreSource {
2906 local: Some(super::KanidmRestoreLocalSource {
2907 file_name: "backup.json".to_string(),
2908 }),
2909 backup_ref: None,
2910 };
2911 let restore = make_restore(source, None);
2912 assert!(!is_remote_source(&restore));
2913 }
2914
2915 #[test]
2916 fn requires_safety_backup_default_true() {
2917 let source = KanidmRestoreSource {
2918 local: None,
2919 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2920 name: "backup-1".to_string(),
2921 }),
2922 };
2923 let restore = make_restore(source, None);
2924 assert!(requires_safety_backup(&restore));
2925 }
2926
2927 #[test]
2928 fn requires_safety_backup_false_when_skip() {
2929 let source = KanidmRestoreSource {
2930 local: None,
2931 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2932 name: "backup-1".to_string(),
2933 }),
2934 };
2935 let safety = SafetyBackupConfig {
2936 repository_ref: Some(SafetyBackupRepositoryRef {
2937 name: "offsite".to_string(),
2938 }),
2939 skip: true,
2940 };
2941 let restore = make_restore(source, Some(safety));
2942 assert!(!requires_safety_backup(&restore));
2943 }
2944
2945 #[test]
2946 fn break_glass_validation_rejects_skip_without_reason() {
2947 let source = KanidmRestoreSource {
2948 local: None,
2949 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2950 name: "backup-1".to_string(),
2951 }),
2952 };
2953 let safety = SafetyBackupConfig {
2954 repository_ref: Some(SafetyBackupRepositoryRef {
2955 name: "offsite".to_string(),
2956 }),
2957 skip: true,
2958 };
2959 let mut restore = make_restore(source, Some(safety));
2960 restore.metadata.annotations = Some(BTreeMap::from([(
2961 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
2962 "commander@example.com".to_string(),
2963 )]));
2964 assert!(validate_safety_backup_config(&restore).is_err());
2965 }
2966
2967 #[test]
2968 fn break_glass_validation_rejects_skip_without_approver() {
2969 let source = KanidmRestoreSource {
2970 local: None,
2971 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2972 name: "backup-1".to_string(),
2973 }),
2974 };
2975 let safety = SafetyBackupConfig {
2976 repository_ref: Some(SafetyBackupRepositoryRef {
2977 name: "offsite".to_string(),
2978 }),
2979 skip: true,
2980 };
2981 let mut restore = make_restore(source, Some(safety));
2982 restore.metadata.annotations = Some(BTreeMap::from([(
2983 BREAK_GLASS_REASON_ANNOTATION.to_string(),
2984 "PVC is unreadable".to_string(),
2985 )]));
2986 assert!(validate_safety_backup_config(&restore).is_err());
2987 }
2988
2989 #[test]
2990 fn break_glass_validation_accepts_valid_annotations() {
2991 let source = KanidmRestoreSource {
2992 local: None,
2993 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2994 name: "backup-1".to_string(),
2995 }),
2996 };
2997 let safety = SafetyBackupConfig {
2998 repository_ref: Some(SafetyBackupRepositoryRef {
2999 name: "offsite".to_string(),
3000 }),
3001 skip: true,
3002 };
3003 let mut restore = make_restore(source, Some(safety));
3004 restore.metadata.annotations = Some(BTreeMap::from([
3005 (
3006 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3007 "PVC is unreadable".to_string(),
3008 ),
3009 (
3010 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3011 "commander@example.com".to_string(),
3012 ),
3013 ]));
3014 assert!(validate_safety_backup_config(&restore).is_ok());
3015 }
3016
3017 #[test]
3018 fn remote_source_requires_repository_ref_when_not_skipping() {
3019 let source = KanidmRestoreSource {
3020 local: None,
3021 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3022 name: "backup-1".to_string(),
3023 }),
3024 };
3025 let safety = SafetyBackupConfig {
3026 repository_ref: None,
3027 skip: false,
3028 };
3029 let restore = make_restore(source, Some(safety));
3030 assert!(validate_safety_backup_config(&restore).is_err());
3031 }
3032
3033 #[test]
3034 fn remote_source_accepts_repository_ref() {
3035 let source = KanidmRestoreSource {
3036 local: None,
3037 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3038 name: "backup-1".to_string(),
3039 }),
3040 };
3041 let safety = SafetyBackupConfig {
3042 repository_ref: Some(SafetyBackupRepositoryRef {
3043 name: "offsite".to_string(),
3044 }),
3045 skip: false,
3046 };
3047 let restore = make_restore(source, Some(safety));
3048 assert!(validate_safety_backup_config(&restore).is_ok());
3049 }
3050
3051 #[test]
3052 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3053 let phases = vec![
3054 KanidmRestorePhase::Pending,
3055 KanidmRestorePhase::Validating,
3056 KanidmRestorePhase::Quiescing,
3057 KanidmRestorePhase::SafetyBackup,
3058 KanidmRestorePhase::PreparingSource,
3059 KanidmRestorePhase::RestoringPrimary,
3060 KanidmRestorePhase::Verifying,
3061 KanidmRestorePhase::RebuildingReplicas,
3062 KanidmRestorePhase::Resuming,
3063 KanidmRestorePhase::Completed,
3064 ];
3065 assert_eq!(phases.len(), 10);
3066 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3067 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3068 }
3069
3070 #[test]
3071 fn status_default_has_empty_phase_timestamps() {
3072 let status = KanidmRestoreStatus::default();
3073 assert!(status.phase_timestamps.is_empty());
3074 }
3075
3076 #[test]
3077 fn status_default_has_empty_original_replicas() {
3078 let status = KanidmRestoreStatus::default();
3079 assert!(status.original_replicas.is_empty());
3080 }
3081
3082 #[test]
3083 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3084 let mut target = super::super::crd::Kanidm::default();
3085 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3086 run_as_non_root: Some(false),
3087 run_as_user: Some(389),
3088 run_as_group: Some(389),
3089 fs_group: Some(389),
3090 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3091 supplemental_groups: Some(vec![390, 391]),
3092 ..Default::default()
3093 });
3094
3095 let ctx = super::source_job_pod_security_context(&target);
3096
3097 assert_eq!(ctx.run_as_non_root, Some(false));
3098 assert_eq!(ctx.run_as_user, Some(389));
3099 assert_eq!(ctx.run_as_group, Some(389));
3100 assert_eq!(ctx.fs_group, Some(389));
3101 assert_eq!(
3102 ctx.fs_group_change_policy.as_deref(),
3103 Some("OnRootMismatch")
3104 );
3105 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3106 assert_eq!(
3107 ctx.seccomp_profile
3108 .as_ref()
3109 .map(|profile| profile.type_.as_str()),
3110 Some("RuntimeDefault")
3111 );
3112 }
3113
3114 #[test]
3115 fn hardened_security_context_drops_all_capabilities() {
3116 let ctx = hardened_security_context();
3117 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3118 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3119 assert_eq!(ctx.run_as_non_root, Some(true));
3120 let caps = ctx.capabilities.unwrap();
3121 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3122 assert_eq!(
3123 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3124 Some("RuntimeDefault")
3125 );
3126 }
3127
3128 #[test]
3129 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3130 let ctx = kanidm_job_security_context();
3131 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3132 assert!(ctx.run_as_non_root.is_none());
3133 assert!(ctx.read_only_root_filesystem.is_none());
3134 let caps = ctx.capabilities.unwrap();
3135 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3136 assert_eq!(
3137 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3138 Some("RuntimeDefault")
3139 );
3140 }
3141
3142 #[test]
3143 fn safety_backup_job_name_is_deterministic() {
3144 let source = KanidmRestoreSource {
3145 local: Some(super::KanidmRestoreLocalSource {
3146 file_name: "backup.json".to_string(),
3147 }),
3148 backup_ref: None,
3149 };
3150 let restore = make_restore(source, None);
3151 let name1 = super::safety_backup_job_name(&restore);
3152 let name2 = super::safety_backup_job_name(&restore);
3153 assert_eq!(name1, name2);
3154 assert_eq!(name1, "test-restore-safety-backup");
3155 }
3156
3157 #[test]
3158 fn source_prep_job_name_is_deterministic() {
3159 let source = KanidmRestoreSource {
3160 local: None,
3161 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3162 name: "backup-1".to_string(),
3163 }),
3164 };
3165 let restore = make_restore(source, None);
3166 let name = super::source_prep_job_name(&restore);
3167 assert_eq!(name, "test-restore-source-prep");
3168 }
3169
3170 #[test]
3171 fn break_glass_condition_is_recorded_in_status() {
3172 let mut status = KanidmRestoreStatus::default();
3173 let condition = super::restore_condition(
3174 &status.conditions,
3175 CONDITION_BREAK_GLASS,
3176 CONDITION_TRUE,
3177 "BreakGlassOverride",
3178 "Safety backup skipped",
3179 Some(1),
3180 );
3181 status.conditions.push(condition);
3182 assert!(status.conditions.iter().any(|c| {
3183 c.type_ == CONDITION_BREAK_GLASS
3184 && c.status == CONDITION_TRUE
3185 && c.reason == "BreakGlassOverride"
3186 }));
3187 }
3188
3189 #[test]
3190 fn replica_count_entry_serialization() {
3191 let entry = ReplicaCountEntry {
3192 group: "primary".to_string(),
3193 replicas: 3,
3194 };
3195 let json = serde_json::to_string(&entry).unwrap();
3196 assert!(json.contains("\"group\":\"primary\""));
3197 assert!(json.contains("\"replicas\":3"));
3198 }
3199
3200 #[test]
3201 fn local_source_does_not_require_repository_ref() {
3202 let source = KanidmRestoreSource {
3203 local: Some(super::KanidmRestoreLocalSource {
3204 file_name: "backup.json".to_string(),
3205 }),
3206 backup_ref: None,
3207 };
3208 let safety = SafetyBackupConfig {
3209 repository_ref: None,
3210 skip: false,
3211 };
3212 let restore = make_restore(source, Some(safety));
3213 assert!(validate_safety_backup_config(&restore).is_ok());
3214 }
3215
3216 #[test]
3217 fn skip_without_break_glass_annotations_is_rejected() {
3218 let source = KanidmRestoreSource {
3219 local: Some(super::KanidmRestoreLocalSource {
3220 file_name: "backup.json".to_string(),
3221 }),
3222 backup_ref: None,
3223 };
3224 let safety = SafetyBackupConfig {
3225 repository_ref: None,
3226 skip: true,
3227 };
3228 let restore = make_restore(source, Some(safety));
3229 assert!(validate_safety_backup_config(&restore).is_err());
3230 }
3231
3232 #[test]
3233 fn has_accepted_condition_true() {
3234 let conditions = vec![Condition {
3235 type_: "Ready".to_string(),
3236 status: "True".to_string(),
3237 reason: "Accepted".to_string(),
3238 message: String::new(),
3239 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3240 k8s_openapi::jiff::Timestamp::now(),
3241 ),
3242 observed_generation: None,
3243 }];
3244 assert!(has_accepted_condition(&conditions));
3245 }
3246
3247 #[test]
3248 fn has_accepted_condition_false_when_not_accepted_reason() {
3249 let conditions = vec![Condition {
3250 type_: "Ready".to_string(),
3251 status: "True".to_string(),
3252 reason: "SomeOtherReason".to_string(),
3253 message: String::new(),
3254 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3255 k8s_openapi::jiff::Timestamp::now(),
3256 ),
3257 observed_generation: None,
3258 }];
3259 assert!(!has_accepted_condition(&conditions));
3260 }
3261
3262 #[test]
3263 fn has_accepted_condition_false_when_status_false() {
3264 let conditions = vec![Condition {
3265 type_: "Ready".to_string(),
3266 status: "False".to_string(),
3267 reason: "Accepted".to_string(),
3268 message: String::new(),
3269 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3270 k8s_openapi::jiff::Timestamp::now(),
3271 ),
3272 observed_generation: None,
3273 }];
3274 assert!(!has_accepted_condition(&conditions));
3275 }
3276
3277 #[test]
3278 fn has_accepted_condition_empty() {
3279 assert!(!has_accepted_condition(&[]));
3280 }
3281
3282 #[test]
3283 fn safety_backup_upload_operation_doc_is_valid() {
3284 let source = KanidmRestoreSource {
3285 local: None,
3286 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3287 name: "backup-1".to_string(),
3288 }),
3289 };
3290 let mut restore = make_restore(source, None);
3291 restore.metadata.namespace = Some("test-ns".to_string());
3292 let target = super::super::crd::Kanidm::default();
3293 let doc_str = super::build_safety_upload_operation_doc(
3294 &restore,
3295 &target,
3296 "test-backup-id",
3297 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3298 "my-bucket",
3299 "prod",
3300 "https://s3.example.com",
3301 "us-east-1",
3302 false,
3303 false,
3304 None,
3305 None,
3306 )
3307 .unwrap();
3308 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3309 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3310 assert_eq!(parsed["kind"], "OperationDocument");
3311 assert_eq!(parsed["operation"], "upload");
3312 assert_eq!(parsed["bucket"], "my-bucket");
3313 assert_eq!(parsed["prefix"], "prod");
3314 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3315 assert_eq!(parsed["region"], "us-east-1");
3316 assert_eq!(parsed["backupId"], "test-backup-id");
3317 assert_eq!(parsed["consistency"], "kanidm-offline");
3318 assert_eq!(parsed["reason"], "restore-safety");
3319 assert_eq!(
3320 parsed["payloadPath"],
3321 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3322 );
3323 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3324 }
3325
3326 #[test]
3327 fn download_operation_doc_uses_real_repository_data() {
3328 let source = KanidmRestoreSource {
3329 local: None,
3330 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3331 name: "backup-1".to_string(),
3332 }),
3333 };
3334 let restore = make_restore(source, None);
3335 let target = super::super::crd::Kanidm::default();
3336 let doc_str = super::build_download_operation_doc(
3337 &restore,
3338 &target,
3339 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3340 "019c7c76-f423-7a12-8f41-2bea7588a303",
3341 "real-bucket",
3342 "real-prefix",
3343 "https://real-endpoint.com",
3344 "eu-west-1",
3345 true,
3346 false,
3347 None,
3348 None,
3349 );
3350 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3351 assert_eq!(parsed["bucket"], "real-bucket");
3352 assert_eq!(parsed["prefix"], "real-prefix");
3353 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3354 assert_eq!(parsed["region"], "eu-west-1");
3355 assert_eq!(parsed["forcePathStyle"], true);
3356 assert_eq!(
3357 parsed["manifestKey"],
3358 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3359 );
3360 assert_eq!(
3361 parsed["expectedBackupId"],
3362 "019c7c76-f423-7a12-8f41-2bea7588a303"
3363 );
3364 }
3365
3366 #[test]
3367 fn safety_backup_ref_is_set_after_verification() {
3368 let status = KanidmRestoreStatus {
3369 phase: KanidmRestorePhase::SafetyBackup,
3370 safety_backup_job_name: Some("test-safety-backup".to_string()),
3371 safety_backup_ref: Some("safety-test-restore".to_string()),
3372 safety_backup_expected_backup_id: Some(
3373 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3374 ),
3375 safety_backup_manifest_key: Some(
3376 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3377 ),
3378 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3379 ..Default::default()
3380 };
3381 assert!(status.safety_backup_ref.is_some());
3382 assert!(
3383 status
3384 .safety_backup_ref
3385 .as_ref()
3386 .unwrap()
3387 .starts_with("safety-")
3388 );
3389 assert!(status.safety_backup_expected_backup_id.is_some());
3390 assert!(status.safety_backup_manifest_key.is_some());
3391 assert!(status.safety_backup_payload_sha256.is_some());
3392 }
3393
3394 #[test]
3395 fn database_mutation_not_started_without_safety_backup_ref() {
3396 let status = KanidmRestoreStatus {
3397 phase: KanidmRestorePhase::PreparingSource,
3398 safety_backup_ref: None,
3399 database_mutation_started: false,
3400 ..Default::default()
3401 };
3402 assert!(!status.database_mutation_started);
3403 assert!(status.safety_backup_ref.is_none());
3404 }
3405
3406 #[test]
3407 fn pre_boundary_failure_does_not_set_mutation_started() {
3408 let status = KanidmRestoreStatus {
3409 phase: KanidmRestorePhase::Failed,
3410 message: Some(
3411 "safety backup job failed; target restored to original state".to_string(),
3412 ),
3413 database_mutation_started: false,
3414 ..Default::default()
3415 };
3416 assert!(!status.database_mutation_started);
3417 }
3418
3419 #[test]
3420 fn shared_volume_constants_are_distinct() {
3421 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3422 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3423 }
3424
3425 #[test]
3426 fn validate_backup_compatibility_matching_versions_passes() {
3427 let source = KanidmRestoreSource {
3428 local: None,
3429 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3430 name: "backup-1".to_string(),
3431 }),
3432 };
3433 let restore = make_restore(source, None);
3434 let target = kanidm_with_version("1.10.4");
3435 let backup = kaniop_backup_core::crd::KanidmBackup {
3436 metadata: kube::api::ObjectMeta::default(),
3437 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3438 backup_id: "id".to_string(),
3439 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3440 name: "k".to_string(),
3441 uid: "uid".to_string(),
3442 },
3443 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3444 name: "repo".to_string(),
3445 },
3446 manifest_key: "key".to_string(),
3447 },
3448 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3449 kanidm_version: Some("1.10.4".to_string()),
3450 ..Default::default()
3451 }),
3452 };
3453 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3454 }
3455
3456 #[test]
3457 fn validate_backup_compatibility_version_mismatch_fails() {
3458 let source = KanidmRestoreSource {
3459 local: None,
3460 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3461 name: "backup-1".to_string(),
3462 }),
3463 };
3464 let restore = make_restore(source, None);
3465 let target = kanidm_with_version("1.11.0");
3466 let backup = kaniop_backup_core::crd::KanidmBackup {
3467 metadata: kube::api::ObjectMeta::default(),
3468 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3469 backup_id: "id".to_string(),
3470 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3471 name: "k".to_string(),
3472 uid: "uid".to_string(),
3473 },
3474 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3475 name: "repo".to_string(),
3476 },
3477 manifest_key: "key".to_string(),
3478 },
3479 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3480 kanidm_version: Some("1.10.4".to_string()),
3481 ..Default::default()
3482 }),
3483 };
3484 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3485 }
3486
3487 #[test]
3488 fn validate_backup_compatibility_no_backup_version_passes() {
3489 let source = KanidmRestoreSource {
3490 local: None,
3491 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3492 name: "backup-1".to_string(),
3493 }),
3494 };
3495 let restore = make_restore(source, None);
3496 let target = super::super::crd::Kanidm::default();
3497 let backup = kaniop_backup_core::crd::KanidmBackup {
3498 metadata: kube::api::ObjectMeta::default(),
3499 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3500 backup_id: "id".to_string(),
3501 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3502 name: "k".to_string(),
3503 uid: "uid".to_string(),
3504 },
3505 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3506 name: "repo".to_string(),
3507 },
3508 manifest_key: "key".to_string(),
3509 },
3510 status: None,
3511 };
3512 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3513 }
3514
3515 #[test]
3516 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3517 let source = KanidmRestoreSource {
3518 local: None,
3519 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3520 name: "backup-1".to_string(),
3521 }),
3522 };
3523 let mut restore = make_restore(source, None);
3524 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3525 let target = super::super::crd::Kanidm::default();
3526 let backup = kaniop_backup_core::crd::KanidmBackup {
3527 metadata: kube::api::ObjectMeta::default(),
3528 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3529 backup_id: "id".to_string(),
3530 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3531 name: "k".to_string(),
3532 uid: "uid".to_string(),
3533 },
3534 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3535 name: "repo".to_string(),
3536 },
3537 manifest_key: "key".to_string(),
3538 },
3539 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3540 image_digest: Some("sha256:originaldigest".to_string()),
3541 ..Default::default()
3542 }),
3543 };
3544 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3545 }
3546
3547 #[test]
3548 fn validate_backup_compatibility_image_digest_match_passes() {
3549 let source = KanidmRestoreSource {
3550 local: None,
3551 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3552 name: "backup-1".to_string(),
3553 }),
3554 };
3555 let mut restore = make_restore(source, None);
3556 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3557 let target = super::super::crd::Kanidm::default();
3558 let backup = kaniop_backup_core::crd::KanidmBackup {
3559 metadata: kube::api::ObjectMeta::default(),
3560 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3561 backup_id: "id".to_string(),
3562 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3563 name: "k".to_string(),
3564 uid: "uid".to_string(),
3565 },
3566 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3567 name: "repo".to_string(),
3568 },
3569 manifest_key: "key".to_string(),
3570 },
3571 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3572 image_digest: Some("sha256:abc123".to_string()),
3573 ..Default::default()
3574 }),
3575 };
3576 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3577 }
3578
3579 #[test]
3580 fn safety_backup_id_is_deterministic() {
3581 let source = KanidmRestoreSource {
3582 local: None,
3583 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3584 name: "backup-1".to_string(),
3585 }),
3586 };
3587 let restore = make_restore(source, None);
3588 let id1 = super::compute_safety_backup_id(&restore);
3589 let id2 = super::compute_safety_backup_id(&restore);
3590 assert_eq!(id1, id2);
3591 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3592 }
3593
3594 #[test]
3595 fn safety_backup_id_differs_for_different_restores() {
3596 let source1 = KanidmRestoreSource {
3597 local: None,
3598 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3599 name: "backup-1".to_string(),
3600 }),
3601 };
3602 let restore1 = make_restore(source1, None);
3603 let mut restore2 = make_restore(
3604 KanidmRestoreSource {
3605 local: None,
3606 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3607 name: "backup-2".to_string(),
3608 }),
3609 },
3610 None,
3611 );
3612 restore2.metadata.name = Some("different-restore".to_string());
3613 let id1 = super::compute_safety_backup_id(&restore1);
3614 let id2 = super::compute_safety_backup_id(&restore2);
3615 assert_ne!(id1, id2);
3616 }
3617
3618 #[test]
3619 fn safety_manifest_key_is_deterministic() {
3620 let source = KanidmRestoreSource {
3621 local: None,
3622 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3623 name: "backup-1".to_string(),
3624 }),
3625 };
3626 let restore = make_restore(source, None);
3627 let key1 = super::compute_safety_manifest_key(&restore);
3628 let key2 = super::compute_safety_manifest_key(&restore);
3629 assert_eq!(key1, key2);
3630 assert!(key1.contains("manifest.json"));
3631 assert!(key1.starts_with("v1/tenants/"));
3632 }
3633
3634 #[test]
3635 fn safety_backup_id_survives_restart() {
3636 let source = KanidmRestoreSource {
3637 local: None,
3638 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3639 name: "backup-1".to_string(),
3640 }),
3641 };
3642 let mut restore = make_restore(source, None);
3643 let original_id = super::compute_safety_backup_id(&restore);
3644 restore.status = Some(KanidmRestoreStatus {
3645 phase: KanidmRestorePhase::SafetyBackup,
3646 safety_backup_expected_backup_id: Some(original_id.clone()),
3647 ..Default::default()
3648 });
3649 let recomputed_id = super::compute_safety_backup_id(&restore);
3650 assert_eq!(original_id, recomputed_id);
3651 }
3652
3653 #[test]
3654 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3655 let source = KanidmRestoreSource {
3656 local: None,
3657 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3658 name: "backup-1".to_string(),
3659 }),
3660 };
3661 let restore = make_restore(source, None);
3662 assert!(super::requires_safety_backup(&restore));
3663 let status_without_ref = KanidmRestoreStatus {
3664 phase: KanidmRestorePhase::RestoringPrimary,
3665 safety_backup_ref: None,
3666 database_mutation_started: false,
3667 ..Default::default()
3668 };
3669 assert!(status_without_ref.safety_backup_ref.is_none());
3670 assert!(!status_without_ref.database_mutation_started);
3671 }
3672
3673 #[test]
3674 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3675 let status_with_ref = KanidmRestoreStatus {
3676 phase: KanidmRestorePhase::RestoringPrimary,
3677 safety_backup_ref: Some("safety-test-restore".to_string()),
3678 safety_backup_manifest_key: Some(
3679 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3680 ),
3681 safety_backup_payload_sha256: Some("abc123".to_string()),
3682 database_mutation_started: false,
3683 ..Default::default()
3684 };
3685 assert!(status_with_ref.safety_backup_ref.is_some());
3686 assert!(!status_with_ref.database_mutation_started);
3687 }
3688
3689 #[test]
3690 fn result_document_verification_rejects_missing_backup_id() {
3691 let result_json = r#"{
3692 "apiVersion": "backup.kaniop.rs/v1alpha1",
3693 "kind": "ResultDocument",
3694 "operation": "upload",
3695 "success": true,
3696 "exitCode": "success",
3697 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3698 "payloadSha256": "abc123"
3699 }"#;
3700 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3701 assert!(result.backup_id.is_none());
3702 }
3703
3704 #[test]
3705 fn result_document_verification_rejects_missing_manifest_key() {
3706 let result_json = r#"{
3707 "apiVersion": "backup.kaniop.rs/v1alpha1",
3708 "kind": "ResultDocument",
3709 "operation": "upload",
3710 "success": true,
3711 "exitCode": "success",
3712 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3713 "payloadSha256": "abc123"
3714 }"#;
3715 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3716 assert!(result.manifest_key.is_none());
3717 }
3718
3719 #[test]
3720 fn result_document_verification_rejects_missing_payload_sha256() {
3721 let result_json = r#"{
3722 "apiVersion": "backup.kaniop.rs/v1alpha1",
3723 "kind": "ResultDocument",
3724 "operation": "upload",
3725 "success": true,
3726 "exitCode": "success",
3727 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3728 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3729 }"#;
3730 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3731 assert!(result.payload_sha256.is_none());
3732 }
3733
3734 #[test]
3735 fn result_document_verification_rejects_failure_exit_code() {
3736 let result_json = r#"{
3737 "apiVersion": "backup.kaniop.rs/v1alpha1",
3738 "kind": "ResultDocument",
3739 "operation": "upload",
3740 "success": false,
3741 "exitCode": "integrity",
3742 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3743 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3744 "payloadSha256": "abc123"
3745 }"#;
3746 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3747 assert!(!result.success);
3748 assert_eq!(
3749 result.exit_code,
3750 kaniop_backup_core::result::ExitCode::Integrity
3751 );
3752 }
3753
3754 #[test]
3755 fn result_document_verification_rejects_wrong_operation() {
3756 let result_json = r#"{
3757 "apiVersion": "backup.kaniop.rs/v1alpha1",
3758 "kind": "ResultDocument",
3759 "operation": "download",
3760 "success": true,
3761 "exitCode": "success",
3762 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3763 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3764 "payloadSha256": "abc123"
3765 }"#;
3766 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3767 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3768 }
3769
3770 #[test]
3771 fn result_document_verification_accepts_valid_document() {
3772 let result_json = r#"{
3773 "apiVersion": "backup.kaniop.rs/v1alpha1",
3774 "kind": "ResultDocument",
3775 "operation": "upload",
3776 "success": true,
3777 "exitCode": "success",
3778 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3779 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3780 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3781 }"#;
3782 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3783 assert!(result.success);
3784 assert_eq!(
3785 result.exit_code,
3786 kaniop_backup_core::result::ExitCode::Success
3787 );
3788 assert_eq!(result.operation, "upload");
3789 assert!(result.backup_id.is_some());
3790 assert!(result.manifest_key.is_some());
3791 assert!(result.payload_sha256.is_some());
3792 }
3793
3794 #[test]
3795 fn termination_message_path_constant_is_valid() {
3796 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3797 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3798 }
3799
3800 #[test]
3801 fn status_tracks_safety_backup_evidence() {
3802 let status = KanidmRestoreStatus {
3803 phase: KanidmRestorePhase::PreparingSource,
3804 safety_backup_ref: Some("safety-test-restore".to_string()),
3805 safety_backup_expected_backup_id: Some(
3806 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3807 ),
3808 safety_backup_manifest_key: Some(
3809 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3810 ),
3811 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3812 ..Default::default()
3813 };
3814 assert!(status.safety_backup_ref.is_some());
3815 assert!(status.safety_backup_expected_backup_id.is_some());
3816 assert!(status.safety_backup_manifest_key.is_some());
3817 assert!(status.safety_backup_payload_sha256.is_some());
3818 }
3819
3820 #[test]
3821 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3822 let invalid_json = "not valid json";
3823 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3824 assert!(result.is_err());
3825 }
3826
3827 #[test]
3828 fn fail_closed_semantics_preserved_on_oversized_result() {
3829 let oversized_json = format!(
3830 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3831 kaniop_backup_core::result::RESULT_DOC_VERSION,
3832 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3833 );
3834 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3835 assert!(result.is_err());
3836 }
3837
3838 #[test]
3839 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3840 let source = KanidmRestoreSource {
3841 local: None,
3842 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3843 name: "backup-1".to_string(),
3844 }),
3845 };
3846 let mut restore = make_restore(source, None);
3847 restore.metadata.namespace = Some("test-ns".to_string());
3848 let target = super::super::crd::Kanidm::default();
3849 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3850 let doc_str = super::build_safety_upload_operation_doc(
3851 &restore,
3852 &target,
3853 "test-backup-id",
3854 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3855 "my-bucket",
3856 "prod",
3857 "https://s3.example.com",
3858 "us-east-1",
3859 false,
3860 false,
3861 Some(&ca_path),
3862 None,
3863 )
3864 .unwrap();
3865 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3866 assert_eq!(parsed["caBundlePath"], ca_path);
3867 }
3868
3869 #[test]
3870 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3871 let source = KanidmRestoreSource {
3872 local: None,
3873 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3874 name: "backup-1".to_string(),
3875 }),
3876 };
3877 let mut restore = make_restore(source, None);
3878 restore.metadata.namespace = Some("test-ns".to_string());
3879 let target = super::super::crd::Kanidm::default();
3880 let doc_str = super::build_safety_upload_operation_doc(
3881 &restore,
3882 &target,
3883 "test-backup-id",
3884 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3885 "my-bucket",
3886 "prod",
3887 "https://s3.example.com",
3888 "us-east-1",
3889 false,
3890 false,
3891 None,
3892 None,
3893 )
3894 .unwrap();
3895 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3896 assert!(parsed["caBundlePath"].is_null());
3897 }
3898
3899 #[test]
3900 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3901 let source = KanidmRestoreSource {
3902 local: None,
3903 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3904 name: "backup-1".to_string(),
3905 }),
3906 };
3907 let restore = make_restore(source, None);
3908 let target = super::super::crd::Kanidm::default();
3909 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3910 let doc_str = super::build_download_operation_doc(
3911 &restore,
3912 &target,
3913 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3914 "019c7c76-f423-7a12-8f41-2bea7588a303",
3915 "real-bucket",
3916 "real-prefix",
3917 "https://real-endpoint.com",
3918 "eu-west-1",
3919 true,
3920 false,
3921 Some(&ca_path),
3922 None,
3923 );
3924 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3925 assert_eq!(parsed["caBundlePath"], ca_path);
3926 }
3927
3928 #[test]
3929 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3930 let source = KanidmRestoreSource {
3931 local: None,
3932 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3933 name: "backup-1".to_string(),
3934 }),
3935 };
3936 let restore = make_restore(source, None);
3937 let target = super::super::crd::Kanidm::default();
3938 let doc_str = super::build_download_operation_doc(
3939 &restore,
3940 &target,
3941 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3942 "019c7c76-f423-7a12-8f41-2bea7588a303",
3943 "real-bucket",
3944 "real-prefix",
3945 "https://real-endpoint.com",
3946 "eu-west-1",
3947 true,
3948 false,
3949 None,
3950 None,
3951 );
3952 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3953 assert!(parsed["caBundlePath"].is_null());
3954 }
3955
3956 #[test]
3957 fn source_prep_result_operation_is_download() {
3958 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3959 assert_ne!(
3960 super::SOURCE_PREP_RESULT_OPERATION,
3961 super::SAFETY_BACKUP_RESULT_OPERATION
3962 );
3963 }
3964
3965 #[test]
3966 fn download_result_document_verification_accepts_valid_document() {
3967 let result_json = r#"{
3968 "apiVersion": "backup.kaniop.rs/v1alpha1",
3969 "kind": "ResultDocument",
3970 "operation": "download",
3971 "success": true,
3972 "exitCode": "success",
3973 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3974 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3975 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3976 }"#;
3977 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3978 assert!(result.success);
3979 assert_eq!(
3980 result.exit_code,
3981 kaniop_backup_core::result::ExitCode::Success
3982 );
3983 assert_eq!(result.operation, "download");
3984 assert_eq!(
3985 result.backup_id.as_deref(),
3986 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
3987 );
3988 assert!(result.manifest_key.is_some());
3989 assert!(result.payload_sha256.is_some());
3990 }
3991
3992 #[test]
3993 fn download_result_document_rejects_wrong_operation() {
3994 let result_json = r#"{
3995 "apiVersion": "backup.kaniop.rs/v1alpha1",
3996 "kind": "ResultDocument",
3997 "operation": "upload",
3998 "success": true,
3999 "exitCode": "success",
4000 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4001 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4002 "payloadSha256": "abc123"
4003 }"#;
4004 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4005 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4006 }
4007
4008 #[test]
4009 fn download_result_document_rejects_failure_success_field() {
4010 let result_json = r#"{
4011 "apiVersion": "backup.kaniop.rs/v1alpha1",
4012 "kind": "ResultDocument",
4013 "operation": "download",
4014 "success": false,
4015 "exitCode": "retryable",
4016 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018 "payloadSha256": "abc123"
4019 }"#;
4020 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021 assert!(!result.success);
4022 assert_ne!(
4023 result.exit_code,
4024 kaniop_backup_core::result::ExitCode::Success
4025 );
4026 }
4027
4028 #[test]
4029 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4030 let status = KanidmRestoreStatus {
4031 phase: KanidmRestorePhase::PreparingSource,
4032 database_mutation_started: false,
4033 ..Default::default()
4034 };
4035 assert!(!status.database_mutation_started);
4036 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4037 }
4038
4039 #[test]
4040 fn backup_job_volume_size_default_is_10gi() {
4041 let qty = crate::controller::backup_job_volume_size();
4042 assert_eq!(qty, Quantity("10Gi".to_string()));
4043 }
4044
4045 #[test]
4046 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4047 use kaniop_backup_core::auth::build_encryption_env_vars;
4048 use kaniop_backup_core::crd::SecretRef;
4049
4050 let key_ref = SecretRef {
4051 name: "kek-secret".to_string(),
4052 };
4053 let env_vars = build_encryption_env_vars(Some(&key_ref));
4054 assert_eq!(env_vars.len(), 1);
4055 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4056 let secret_ref = env_vars[0]
4057 .value_from
4058 .as_ref()
4059 .unwrap()
4060 .secret_key_ref
4061 .as_ref()
4062 .unwrap();
4063 assert_eq!(secret_ref.name, "kek-secret");
4064 assert_eq!(secret_ref.key, "encryption-key");
4065 }
4066
4067 #[test]
4068 fn encryption_env_vars_absent_when_no_key_ref() {
4069 use kaniop_backup_core::auth::build_encryption_env_vars;
4070
4071 let env_vars = build_encryption_env_vars(None);
4072 assert!(
4073 env_vars.is_empty(),
4074 "encryption env vars must be empty when key_ref is None"
4075 );
4076 }
4077
4078 #[test]
4079 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4080 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4081
4082 let source = KanidmRestoreSource {
4083 local: None,
4084 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4085 name: "backup-1".to_string(),
4086 }),
4087 };
4088 let mut restore = make_restore(source, None);
4089 restore.metadata.namespace = Some("test-ns".to_string());
4090 let target = super::super::crd::Kanidm::default();
4091 let encryption = RepositoryEncryption {
4092 mode: EncryptionMode::ClientSide,
4093 key_id: None,
4094 key_ref: Some(SecretRef {
4095 name: "kek-secret".to_string(),
4096 }),
4097 };
4098 let doc_str = super::build_safety_upload_operation_doc(
4099 &restore,
4100 &target,
4101 "test-backup-id",
4102 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4103 "my-bucket",
4104 "prod",
4105 "https://s3.example.com",
4106 "us-east-1",
4107 false,
4108 false,
4109 None,
4110 Some(&encryption),
4111 )
4112 .unwrap();
4113 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4114 assert_eq!(parsed["encryptionMode"], "clientSide");
4115 assert!(parsed["encryptionKeyId"].is_null());
4116 }
4117
4118 #[test]
4119 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4120 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4121
4122 let source = KanidmRestoreSource {
4123 local: None,
4124 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4125 name: "backup-1".to_string(),
4126 }),
4127 };
4128 let restore = make_restore(source, None);
4129 let target = super::super::crd::Kanidm::default();
4130 let encryption = RepositoryEncryption {
4131 mode: EncryptionMode::ProviderKms,
4132 key_id: Some("alias/kaniop-backups".to_string()),
4133 key_ref: None,
4134 };
4135 let doc_str = super::build_download_operation_doc(
4136 &restore,
4137 &target,
4138 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4139 "019c7c76-f423-7a12-8f41-2bea7588a303",
4140 "real-bucket",
4141 "real-prefix",
4142 "https://real-endpoint.com",
4143 "eu-west-1",
4144 true,
4145 false,
4146 None,
4147 Some(&encryption),
4148 );
4149 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4150 assert_eq!(parsed["encryptionMode"], "providerKms");
4151 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4152 }
4153}