1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76 feature = "schemars",
77 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80 category = "kaniop",
81 group = "kaniop.rs",
82 version = "v1beta1",
83 kind = "KanidmRestore",
84 plural = "kanidmrestores",
85 singular = "kanidmrestore",
86 shortname = "idmrestore",
87 namespaced,
88 status = "KanidmRestoreStatus",
89 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95 pub target_ref: KanidmRestoreTargetRef,
96 pub source: KanidmRestoreSource,
97 pub restore_image: String,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106 pub name: String,
107 pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113 feature = "schemars",
114 schemars(extend("x-kubernetes-validations" = [
115 {
116 "message": "exactly one of local or backupRef must be set",
117 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118 }
119 ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123 #[serde(skip_serializing_if = "Option::is_none")]
125 pub local: Option<KanidmRestoreLocalSource>,
126 #[serde(skip_serializing_if = "Option::is_none")]
128 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135 pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143 pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151 #[serde(skip_serializing_if = "Option::is_none")]
153 pub repository_ref: Option<SafetyBackupRepositoryRef>,
154 #[serde(default)]
156 pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163 pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169 #[default]
170 Pending,
171 Validating,
172 Quiescing,
173 SafetyBackup,
174 PreparingSource,
175 RestoringPrimary,
176 Verifying,
177 RebuildingReplicas,
178 Resuming,
179 Completed,
180 Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187 #[serde(skip_serializing_if = "Option::is_none")]
188 pub observed_generation: Option<i64>,
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_target_uid: Option<String>,
191 #[serde(default)]
192 pub phase: KanidmRestorePhase,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 pub restore_job_name: Option<String>,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub verify_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub safety_backup_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub source_prep_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub safety_backup_ref: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_expected_backup_id: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_manifest_key: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_payload_sha256: Option<String>,
209 #[serde(default)]
210 pub replicas_cleared: bool,
211 #[serde(default)]
212 pub certificates_cleared: bool,
213 #[serde(default)]
214 pub database_mutation_started: bool,
215 #[serde(default, skip_serializing_if = "Vec::is_empty")]
216 pub original_replicas: Vec<ReplicaCountEntry>,
217 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218 pub phase_timestamps: std::collections::BTreeMap<String, String>,
219 #[serde(default, skip_serializing_if = "Vec::is_empty")]
220 #[cfg_attr(
221 feature = "schemars",
222 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223 )]
224 pub conditions: Vec<Condition>,
225 #[serde(skip_serializing_if = "Option::is_none")]
226 pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233 pub group: String,
234 pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239 attempts: Counter<u64>,
240 outcomes: Counter<u64>,
241 duration_seconds: Histogram<f64>,
242 break_glass_total: Counter<u64>,
243 safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247 fn new() -> Self {
248 let meter = global::meter("kaniop");
249 Self {
250 attempts: meter
251 .u64_counter("kaniop_restore_attempts_total")
252 .with_description("Number of Kanidm restore attempts started")
253 .build(),
254 outcomes: meter
255 .u64_counter("kaniop_restore_outcomes_total")
256 .with_description("Number of terminal Kanidm restore outcomes")
257 .build(),
258 duration_seconds: meter
259 .f64_histogram("kaniop_restore_duration_seconds")
260 .with_description("Kanidm restore duration from object creation to terminal phase")
261 .with_unit("s")
262 .build(),
263 break_glass_total: meter
264 .u64_counter("kaniop_restore_break_glass_total")
265 .with_description("Number of break-glass safety backup overrides")
266 .build(),
267 safety_backup_duration_seconds: meter
268 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269 .with_description("Duration of pre-restore safety backup creation")
270 .with_unit("s")
271 .build(),
272 }
273 }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278 client: Client,
279 recorder: Recorder,
280 metrics: RestoreMetrics,
281}
282
283pub async fn run(client: Client) {
284 let api = Api::<KanidmRestore>::all(client.clone());
285 let recorder = Recorder::new(
286 client.clone(),
287 Reporter {
288 controller: CONTROLLER_ID.into(),
289 instance: None,
290 },
291 );
292 let ctx = Arc::new(RestoreContext {
293 client,
294 recorder,
295 metrics: RestoreMetrics::new(),
296 });
297 info!("starting {CONTROLLER_ID} controller");
298 Controller::new(api, watcher::Config::default().any_semantic())
299 .shutdown_on_signal()
300 .run(reconcile_restore, error_policy, ctx)
301 .for_each(|result| async move {
302 if let Err(error) = result {
303 error!(%error, "KanidmRestore reconciliation failed");
304 }
305 })
306 .await;
307}
308
309fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
310 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
311 Action::requeue(Duration::from_secs(5))
312}
313
314async fn reconcile_restore(
315 restore: Arc<KanidmRestore>,
316 ctx: Arc<RestoreContext>,
317) -> Result<Action> {
318 let namespace = restore
319 .namespace()
320 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
321 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
322 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
323 let ctx = ctx.clone();
324 async move {
325 match event {
326 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
327 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
328 }
329 }
330 })
331 .await
332 .map_err(|error| {
333 Error::FinalizerError(
334 "failed on KanidmRestore finalizer".to_string(),
335 Box::new(error),
336 )
337 })
338}
339
340async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
341 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
342 match phase {
343 KanidmRestorePhase::Pending => {
344 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
345 Ok(Action::requeue(REQUEUE))
346 }
347 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
348 Ok(target) => {
349 mark_restoring(&restore, &target, &ctx).await?;
350 let mut status = restore.status.clone().unwrap_or_default();
351 status.observed_target_uid = target.uid();
352 status.original_replicas = capture_original_replicas(&target);
353 status.phase = KanidmRestorePhase::Quiescing;
354 status.message = None;
355 patch_status(&restore, &ctx, status).await?;
356 Ok(Action::requeue(REQUEUE))
357 }
358 Err(error) => {
359 set_phase(
360 &restore,
361 &ctx,
362 KanidmRestorePhase::Failed,
363 Some(error.to_string()),
364 )
365 .await?;
366 Ok(Action::requeue(Duration::from_secs(300)))
367 }
368 },
369 KanidmRestorePhase::Quiescing => {
370 let target = get_target(&restore, &ctx).await?;
371 scale_all(&target, &ctx, 0).await?;
372 if target_pods_stopped(&target, &ctx).await?
373 && target_volumes_detached(&target, &ctx).await?
374 {
375 let next_phase = if requires_safety_backup(&restore) {
376 KanidmRestorePhase::SafetyBackup
377 } else {
378 record_break_glass(&restore, &ctx).await?;
379 KanidmRestorePhase::PreparingSource
380 };
381 if next_phase == KanidmRestorePhase::PreparingSource {
382 let refreshed = Api::<KanidmRestore>::namespaced(
383 ctx.client.clone(),
384 restore.namespace().as_deref().unwrap_or_default(),
385 )
386 .get(&restore.name_any())
387 .await
388 .map_err(|error| {
389 Error::kube_error(
390 "get",
391 "KanidmRestore",
392 restore.namespace().as_deref().unwrap_or_default(),
393 restore.name_any(),
394 error,
395 )
396 })?;
397 set_phase(&refreshed, &ctx, next_phase, None).await?;
398 } else {
399 set_phase(&restore, &ctx, next_phase, None).await?;
400 }
401 }
402 Ok(Action::requeue(REQUEUE))
403 }
404 KanidmRestorePhase::SafetyBackup => {
405 let target = get_target(&restore, &ctx).await?;
406 let name = safety_backup_job_name(&restore);
407 let expected_backup_id = compute_safety_backup_id(&restore);
408 let expected_manifest_key = compute_safety_manifest_key(&restore);
409 ensure_safety_backup_job(
410 &restore,
411 &target,
412 &ctx,
413 &name,
414 &expected_backup_id,
415 &expected_manifest_key,
416 )
417 .await?;
418 let mut status = restore.status.clone().unwrap_or_default();
419 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
420 match job_state(&restore, &ctx, &name).await? {
421 JobState::Complete => {
422 let result =
423 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
424 match result {
425 Ok(verified) => {
426 status.safety_backup_job_name = Some(name);
427 status.safety_backup_ref =
428 Some(format!("safety-{}", restore.name_any()));
429 status.safety_backup_manifest_key = Some(verified.manifest_key);
430 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
431 record_safety_backup_duration(&restore, &ctx).await;
432 status.phase = KanidmRestorePhase::PreparingSource;
433 status.message = None;
434 patch_status(&restore, &ctx, status).await?;
435 }
436 Err(error) => {
437 resume_before_mutation(&restore, &ctx).await?;
438 set_phase(
439 &restore,
440 &ctx,
441 KanidmRestorePhase::Failed,
442 Some(format!("safety backup result verification failed: {error}")),
443 )
444 .await?;
445 }
446 }
447 }
448 JobState::Failed => {
449 resume_before_mutation(&restore, &ctx).await?;
450 set_phase(
451 &restore,
452 &ctx,
453 KanidmRestorePhase::Failed,
454 Some(
455 "safety backup job failed; target restored to original state"
456 .to_string(),
457 ),
458 )
459 .await?;
460 }
461 JobState::Running => {
462 patch_status(&restore, &ctx, status).await?;
463 }
464 }
465 Ok(Action::requeue(REQUEUE))
466 }
467 KanidmRestorePhase::PreparingSource => {
468 if is_remote_source(&restore) {
469 let target = get_target(&restore, &ctx).await?;
470 let name = source_prep_job_name(&restore);
471 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
472 match job_state(&restore, &ctx, &name).await? {
473 JobState::Complete => {
474 let backup_id = restore
475 .spec
476 .source
477 .backup_ref
478 .as_ref()
479 .map(|b| b.name.clone())
480 .unwrap_or_default();
481 let backup = {
482 let ns = restore.namespace().unwrap();
483 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
484 .get(&backup_id)
485 .await
486 .map_err(|e| {
487 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
488 })?
489 };
490 let expected_backup_id = &backup.spec.backup_id;
491 let result =
492 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
493 .await;
494 match result {
495 Ok(verified) => {
496 if let Some(expected_sha256) = backup
497 .status
498 .as_ref()
499 .and_then(|s| s.payload_sha256.as_deref())
500 {
501 if verified.payload_sha256 != expected_sha256 {
502 resume_before_mutation(&restore, &ctx).await?;
503 set_phase(
504 &restore,
505 &ctx,
506 KanidmRestorePhase::Failed,
507 Some(format!(
508 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
509 verified.payload_sha256
510 )),
511 )
512 .await?;
513 return Ok(Action::requeue(REQUEUE));
514 }
515 }
516 let mut status = restore.status.clone().unwrap_or_default();
517 status.source_prep_job_name = Some(name);
518 status.phase = KanidmRestorePhase::RestoringPrimary;
519 status.message = None;
520 patch_status(&restore, &ctx, status).await?;
521 }
522 Err(error) => {
523 resume_before_mutation(&restore, &ctx).await?;
524 set_phase(
525 &restore,
526 &ctx,
527 KanidmRestorePhase::Failed,
528 Some(format!(
529 "source preparation result verification failed: {error}"
530 )),
531 )
532 .await?;
533 }
534 }
535 }
536 JobState::Failed => {
537 resume_before_mutation(&restore, &ctx).await?;
538 set_phase(
539 &restore,
540 &ctx,
541 KanidmRestorePhase::Failed,
542 Some(
543 "source preparation job failed; target restored to original state"
544 .to_string(),
545 ),
546 )
547 .await?;
548 }
549 JobState::Running => {}
550 }
551 } else {
552 let target = get_target(&restore, &ctx).await?;
553 let name = source_check_job_name(&restore);
554 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
555 match job_state(&restore, &ctx, &name).await? {
556 JobState::Complete => {
557 let mut status = restore.status.clone().unwrap_or_default();
558 status.source_prep_job_name = Some(name);
559 status.phase = KanidmRestorePhase::RestoringPrimary;
560 status.message = None;
561 patch_status(&restore, &ctx, status).await?;
562 }
563 JobState::Failed => {
564 resume_before_mutation(&restore, &ctx).await?;
565 set_phase(
566 &restore,
567 &ctx,
568 KanidmRestorePhase::Failed,
569 Some(
570 "local backup file check failed; target restored to original state"
571 .to_string(),
572 ),
573 )
574 .await?;
575 }
576 JobState::Running => {}
577 }
578 }
579 Ok(Action::requeue(REQUEUE))
580 }
581 KanidmRestorePhase::RestoringPrimary => {
582 let target = get_target(&restore, &ctx).await?;
583 let status = restore.status.as_ref().cloned().unwrap_or_default();
584 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
585 return Err(Error::MissingData(
586 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
587 ));
588 }
589 if !status.database_mutation_started {
590 let mut status = status;
591 status.database_mutation_started = true;
592 status.message = Some(
593 "database mutation boundary persisted; starting primary restore".to_string(),
594 );
595 patch_status(&restore, &ctx, status).await?;
596 return Ok(Action::requeue(REQUEUE));
597 }
598 let name = restore_job_name(&restore);
599 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
600 match job_state(&restore, &ctx, &name).await? {
601 JobState::Complete => {
602 let mut status = restore.status.clone().unwrap_or_default();
603 status.restore_job_name = Some(name);
604 status.phase = KanidmRestorePhase::Verifying;
605 status.message = None;
606 patch_status(&restore, &ctx, status).await?;
607 }
608 JobState::Failed => {
609 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
610 }
611 JobState::Running => {}
612 }
613 Ok(Action::requeue(REQUEUE))
614 }
615 KanidmRestorePhase::Verifying => {
616 let target = get_target(&restore, &ctx).await?;
617 let name = verify_job_name(&restore);
618 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
619 match job_state(&restore, &ctx, &name).await? {
620 JobState::Complete => {
621 let mut status = restore.status.clone().unwrap_or_default();
622 status.verify_job_name = Some(name);
623 status.phase = KanidmRestorePhase::RebuildingReplicas;
624 status.message = None;
625 patch_status(&restore, &ctx, status).await?;
626 }
627 JobState::Failed => {
628 fail_after_mutation(&restore, &ctx, "database verification failed").await?
629 }
630 JobState::Running => {}
631 }
632 Ok(Action::requeue(REQUEUE))
633 }
634 KanidmRestorePhase::RebuildingReplicas => {
635 let target = get_target(&restore, &ctx).await?;
636 let mut status = restore.status.clone().unwrap_or_default();
637 if !status.replicas_cleared {
638 if delete_secondary_pvcs(&target, &ctx).await? {
639 status.replicas_cleared = true;
640 status.message = Some("secondary database state cleared".to_string());
641 patch_status(&restore, &ctx, status).await?;
642 }
643 return Ok(Action::requeue(REQUEUE));
644 }
645
646 scale_primary(&target, &ctx, 1).await?;
647 if !primary_ready(&target, &ctx).await? {
648 return Ok(Action::requeue(REQUEUE));
649 }
650 if !status.certificates_cleared {
651 delete_replica_cert_secrets(&target, &ctx).await?;
652 delete_admin_secret(&target, &ctx).await?;
653 status.certificates_cleared = true;
654 status.message = Some(
655 "replica certificates and admin secret cleared for regeneration".to_string(),
656 );
657 patch_status(&restore, &ctx, status).await?;
658 return Ok(Action::requeue(REQUEUE));
659 }
660 status.phase = KanidmRestorePhase::Resuming;
661 status.message = None;
662 patch_status(&restore, &ctx, status).await?;
663 Ok(Action::requeue(REQUEUE))
664 }
665 KanidmRestorePhase::Resuming => {
666 let target = get_target(&restore, &ctx).await?;
667 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
668 clear_restoring(&restore, &target, &ctx).await?;
669 Ok(Action::requeue(Duration::from_secs(3600)))
670 }
671 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
672 Ok(Action::requeue(Duration::from_secs(3600)))
673 }
674 }
675}
676
677async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
678 let status = restore.status.clone().unwrap_or_default();
679 if status.database_mutation_started
680 && status.phase != KanidmRestorePhase::Completed
681 && status.phase != KanidmRestorePhase::Failed
682 {
683 reconcile_apply(restore.clone(), ctx.clone()).await?;
684 return Ok(Action::requeue(REQUEUE));
685 }
686 if let Ok(target) = get_target(&restore, &ctx).await {
687 let owns_maintenance =
688 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
689 if owns_maintenance && !status.database_mutation_started {
690 scale_desired(&target, &ctx).await?;
691 }
692 clear_restoring(&restore, &target, &ctx).await?;
693 }
694 Ok(Action::await_change())
695}
696
697async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
698 let target = get_target(restore, ctx).await?;
699 let actual_uid = target
700 .uid()
701 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
702 if actual_uid != restore.spec.target_ref.uid {
703 return Err(Error::MissingData(format!(
704 "target UID mismatch: expected {}, got {}",
705 restore.spec.target_ref.uid, actual_uid
706 )));
707 }
708 validate_source(restore)?;
709 if is_remote_source(restore) {
710 validate_backup_ref(restore, &target, ctx).await?;
711 }
712 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
713 {
714 return Err(Error::MissingData(format!(
715 "restoreImage must be the target's pinned Kanidm image (target image is {})",
716 target.spec.image
717 )));
718 }
719 let storage = target
720 .spec
721 .storage
722 .as_ref()
723 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
724 if storage.empty_dir.is_some()
725 || storage.ephemeral.is_some()
726 || storage.volume_claim_template.is_none()
727 {
728 return Err(Error::MissingData(
729 "restore requires PVC-backed Kanidm storage".to_string(),
730 ));
731 }
732 let primaries = target
733 .spec
734 .replica_groups
735 .iter()
736 .filter(|rg| rg.primary_node)
737 .count();
738 if primaries != 1 {
739 return Err(Error::MissingData(
740 "backup/restore requires exactly one primary replica group".to_string(),
741 ));
742 }
743 let ns = restore.namespace().unwrap();
744 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
745 .list(&ListParams::default())
746 .await
747 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
748 if restores.items.iter().any(|other| {
749 other.name_any() != restore.name_any()
750 && other.spec.target_ref.name == restore.spec.target_ref.name
751 && !matches!(
752 other.status.as_ref().map(|s| s.phase),
753 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
754 )
755 }) {
756 return Err(Error::MissingData(
757 "another active restore targets this Kanidm".to_string(),
758 ));
759 }
760 validate_safety_backup_config(restore)?;
761 Ok(target)
762}
763
764fn safe_basename(name: &str) -> bool {
765 !name.is_empty()
766 && name != "."
767 && name != ".."
768 && !name.contains('/')
769 && !name.contains('\\')
770 && !name.contains('\0')
771}
772
773fn mutable_image(image: &str) -> bool {
774 image == "kanidm/server:latest"
775 || image.ends_with(":latest")
776 || (!image.contains('@')
777 && !image
778 .rsplit('/')
779 .next()
780 .is_some_and(|part| part.contains(':')))
781}
782
783async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
784 let ns = restore
785 .namespace()
786 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
787 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
788 .get(&restore.spec.target_ref.name)
789 .await
790 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
791}
792
793async fn mark_restoring(
794 restore: &KanidmRestore,
795 target: &Kanidm,
796 ctx: &RestoreContext,
797) -> Result<()> {
798 let ns = target.namespace().unwrap();
799 let uid = restore
800 .uid()
801 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
802 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
803 .patch(
804 &target.name_any(),
805 &PatchParams::default(),
806 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
807 )
808 .await
809 .map(|_| ())
810 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
811}
812
813async fn clear_restoring(
814 restore: &KanidmRestore,
815 target: &Kanidm,
816 ctx: &RestoreContext,
817) -> Result<()> {
818 let ns = target.namespace().unwrap();
819 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
820 return Ok(());
821 }
822 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
823 .patch(
824 &target.name_any(),
825 &PatchParams::default(),
826 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
827 )
828 .await
829 .map(|_| ())
830 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
831}
832
833async fn patch_status(
834 restore: &KanidmRestore,
835 ctx: &RestoreContext,
836 mut status: KanidmRestoreStatus,
837) -> Result<()> {
838 let ns = restore.namespace().unwrap();
839 let previous_phase = restore.status.as_ref().map(|current| current.phase);
840 let next_phase = status.phase;
841 status.observed_generation = restore.metadata.generation;
842 update_restore_conditions(&mut status, restore.metadata.generation);
843
844 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
845 .patch_status(
846 &restore.name_any(),
847 &PatchParams::default(),
848 &Patch::Merge(json!({"status": &status})),
849 )
850 .await
851 .map_err(|e| {
852 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
853 })?;
854
855 if previous_phase != Some(next_phase) {
856 record_restore_transition(
857 restore,
858 ctx,
859 previous_phase,
860 next_phase,
861 status.message.as_deref(),
862 )
863 .await;
864 }
865 Ok(())
866}
867
868fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
869 let previous = status.conditions.clone();
870 let phase = status.phase;
871 let phase_reason = format!("{phase:?}");
872 let message = status
873 .message
874 .clone()
875 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
876 let terminal_success = phase == KanidmRestorePhase::Completed;
877 let terminal_failure = phase == KanidmRestorePhase::Failed;
878 let progressing = !terminal_success && !terminal_failure;
879
880 let break_glass = previous
881 .iter()
882 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
883 .cloned();
884 status.conditions = vec![
885 restore_condition(
886 &previous,
887 CONDITION_PROGRESSING,
888 if progressing {
889 CONDITION_TRUE
890 } else {
891 CONDITION_FALSE
892 },
893 &phase_reason,
894 &message,
895 generation,
896 ),
897 restore_condition(
898 &previous,
899 CONDITION_READY,
900 if terminal_success {
901 CONDITION_TRUE
902 } else {
903 CONDITION_FALSE
904 },
905 if terminal_success {
906 "RestoreCompleted"
907 } else {
908 "RestoreNotCompleted"
909 },
910 if terminal_success {
911 "Kanidm restore completed successfully."
912 } else {
913 "Kanidm restore has not completed successfully."
914 },
915 generation,
916 ),
917 restore_condition(
918 &previous,
919 CONDITION_FAILED,
920 if terminal_failure {
921 CONDITION_TRUE
922 } else {
923 CONDITION_FALSE
924 },
925 if terminal_failure {
926 "RestoreFailed"
927 } else {
928 "NoRestoreFailure"
929 },
930 if terminal_failure {
931 status
932 .message
933 .as_deref()
934 .unwrap_or("Kanidm restore failed.")
935 } else {
936 "No terminal restore failure has been recorded."
937 },
938 generation,
939 ),
940 ];
941 if let Some(condition) = break_glass {
942 status.conditions.push(condition);
943 }
944}
945
946fn restore_condition(
947 previous: &[Condition],
948 condition_type: &str,
949 condition_status: &str,
950 reason: &str,
951 message: &str,
952 generation: Option<i64>,
953) -> Condition {
954 let last_transition_time = previous
955 .iter()
956 .find(|condition| {
957 condition.type_ == condition_type
958 && condition.status == condition_status
959 && condition.reason == reason
960 })
961 .map(|condition| condition.last_transition_time.clone())
962 .unwrap_or_else(|| Time(Timestamp::now()));
963 Condition {
964 type_: condition_type.to_string(),
965 status: condition_status.to_string(),
966 reason: reason.to_string(),
967 message: message.to_string(),
968 last_transition_time,
969 observed_generation: generation,
970 }
971}
972
973async fn record_restore_transition(
974 restore: &KanidmRestore,
975 ctx: &RestoreContext,
976 previous_phase: Option<KanidmRestorePhase>,
977 phase: KanidmRestorePhase,
978 message: Option<&str>,
979) {
980 if phase == KanidmRestorePhase::Validating {
981 ctx.metrics.attempts.add(1, &[]);
982 }
983
984 let result = match phase {
985 KanidmRestorePhase::Completed => Some("success"),
986 KanidmRestorePhase::Failed => Some("failure"),
987 _ => None,
988 };
989 if let Some(result) = result {
990 let attributes = [KeyValue::new("result", result)];
991 ctx.metrics.outcomes.add(1, &attributes);
992 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
993 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
994 ctx.metrics.duration_seconds.record(elapsed, &attributes);
995 }
996 }
997
998 let reason = if phase == KanidmRestorePhase::Failed {
999 "RestoreFailed"
1000 } else {
1001 "RestorePhaseChanged"
1002 };
1003 let note = message.map(str::to_string).or_else(|| {
1004 Some(format!(
1005 "Kanidm restore phase changed from {} to {phase:?}.",
1006 previous_phase
1007 .map(|previous| format!("{previous:?}"))
1008 .unwrap_or_else(|| "None".to_string())
1009 ))
1010 });
1011 if let Err(error) = ctx
1012 .recorder
1013 .publish(
1014 &Event {
1015 type_: if phase == KanidmRestorePhase::Failed {
1016 EventType::Warning
1017 } else {
1018 EventType::Normal
1019 },
1020 reason: reason.to_string(),
1021 note,
1022 action: "Restore".to_string(),
1023 secondary: None,
1024 },
1025 &restore.object_ref(&()),
1026 )
1027 .await
1028 {
1029 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1030 }
1031}
1032
1033async fn set_phase(
1034 restore: &KanidmRestore,
1035 ctx: &RestoreContext,
1036 phase: KanidmRestorePhase,
1037 message: Option<String>,
1038) -> Result<()> {
1039 let mut status = restore.status.clone().unwrap_or_default();
1040 status.phase = phase;
1041 status.message = message;
1042 patch_status(restore, ctx, status).await
1043}
1044
1045async fn fail_after_mutation(
1046 restore: &KanidmRestore,
1047 ctx: &RestoreContext,
1048 message: &str,
1049) -> Result<()> {
1050 set_phase(
1051 restore,
1052 ctx,
1053 KanidmRestorePhase::Failed,
1054 Some(message.to_string()),
1055 )
1056 .await
1057}
1058
1059fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1060 target
1061 .spec
1062 .replica_groups
1063 .iter()
1064 .find(|rg| rg.primary_node)
1065 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1066}
1067
1068async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1069 let ns = target.namespace().unwrap();
1070 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1071 for rg in &target.spec.replica_groups {
1072 let name = target.statefulset_name(&rg.name);
1073 api.patch(
1074 &name,
1075 &PatchParams::default(),
1076 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1077 )
1078 .await
1079 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1080 }
1081 Ok(())
1082}
1083
1084async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1085 let ns = target.namespace().unwrap();
1086 let rg = primary_group(target)?;
1087 let name = target.statefulset_name(&rg.name);
1088 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1089 .patch(
1090 &name,
1091 &PatchParams::default(),
1092 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1093 )
1094 .await
1095 .map(|_| ())
1096 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1097}
1098
1099async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1100 let ns = target.namespace().unwrap();
1101 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1102 for rg in &target.spec.replica_groups {
1103 let name = target.statefulset_name(&rg.name);
1104 api.patch(
1105 &name,
1106 &PatchParams::default(),
1107 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1108 )
1109 .await
1110 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1111 }
1112 Ok(())
1113}
1114
1115async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1116 let ns = target.namespace().unwrap();
1117 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1118 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1119 .await
1120 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1121 Ok(pods.items.is_empty())
1122}
1123
1124async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1125 let ns = target.namespace().unwrap();
1126 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1127 let mut pv_names = BTreeSet::new();
1128 let primary_name = primary_pvc_name(target)?;
1129 let primary_pvc = pvc_api
1130 .get(&primary_name)
1131 .await
1132 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1133 let primary_pv = primary_pvc
1134 .spec
1135 .as_ref()
1136 .and_then(|spec| spec.volume_name.clone())
1137 .ok_or_else(|| {
1138 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1139 })?;
1140 pv_names.insert(primary_pv);
1141
1142 for rg in &target.spec.replica_groups {
1143 let sts = target.statefulset_name(&rg.name);
1144 for ordinal in 0..rg.replicas {
1145 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1146 if name == primary_name {
1147 continue;
1148 }
1149 if let Some(pvc) = pvc_api
1150 .get_opt(&name)
1151 .await
1152 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1153 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1154 {
1155 pv_names.insert(pv_name);
1156 }
1157 }
1158 }
1159
1160 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1161 .list(&ListParams::default())
1162 .await
1163 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1164 Ok(!attachments.items.iter().any(|attachment| {
1165 attachment
1166 .spec
1167 .source
1168 .persistent_volume_name
1169 .as_ref()
1170 .is_some_and(|pv_name| pv_names.contains(pv_name))
1171 }))
1172}
1173
1174async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1175 let ns = target.namespace().unwrap();
1176 let name = target.statefulset_name(&primary_group(target)?.name);
1177 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1178 .get(&name)
1179 .await
1180 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1181 Ok(sts
1182 .status
1183 .as_ref()
1184 .and_then(|s| s.ready_replicas)
1185 .unwrap_or(0)
1186 >= 1)
1187}
1188
1189fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1190 let rg = primary_group(target)?;
1191 Ok(format!(
1192 "{DATA_VOLUME}-{}-0",
1193 target.statefulset_name(&rg.name)
1194 ))
1195}
1196
1197async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1198 let ns = target.namespace().unwrap();
1199 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1200 let mut all_absent = true;
1201 for rg in &target.spec.replica_groups {
1202 let sts = target.statefulset_name(&rg.name);
1203 for ordinal in 0..rg.replicas {
1204 if rg.primary_node && ordinal == 0 {
1205 continue;
1206 }
1207 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1208 match api
1209 .get_opt(&name)
1210 .await
1211 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1212 {
1213 None => {}
1214 Some(pvc) => {
1215 all_absent = false;
1216 if pvc.metadata.deletion_timestamp.is_none() {
1217 match api.delete(&name, &DeleteParams::default()).await {
1218 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1219 Err(kube::Error::Api(status)) if status.code == 404 => {}
1220 Err(error) => {
1221 return Err(Error::kube_error(
1222 "delete",
1223 "PersistentVolumeClaim",
1224 &ns,
1225 &name,
1226 error,
1227 ));
1228 }
1229 }
1230 }
1231 }
1232 }
1233 }
1234 }
1235 Ok(all_absent)
1236}
1237
1238async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1239 let ns = target.namespace().unwrap();
1240 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1241 let label_selector = format!(
1242 "{}={},{}={}",
1243 CLUSTER_LABEL,
1244 target.name_any(),
1245 SECRET_TYPE_LABEL,
1246 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1247 );
1248 let secrets = api
1249 .list(&ListParams::default().labels(&label_selector))
1250 .await
1251 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1252 let mut all_absent = secrets.items.is_empty();
1253 for secret in secrets.items {
1254 let name = secret.name_any();
1255 if secret.metadata.deletion_timestamp.is_none() {
1256 match api.delete(&name, &DeleteParams::default()).await {
1257 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1258 Err(kube::Error::Api(status)) if status.code == 404 => {}
1259 Err(error) => {
1260 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1261 }
1262 }
1263 all_absent = false;
1264 }
1265 }
1266 Ok(all_absent)
1267}
1268
1269async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1270 let ns = target.namespace().unwrap();
1271 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1272 let name = format!("{}-admin-passwords", target.name_any());
1273 match api
1274 .get_opt(&name)
1275 .await
1276 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1277 {
1278 None => Ok(true),
1279 Some(secret) => {
1280 if secret.metadata.deletion_timestamp.is_some() {
1281 return Ok(false);
1282 }
1283 match api.delete(&name, &DeleteParams::default()).await {
1284 Ok(_) => {
1285 debug!(secret = %name, "deleting stale admin secret");
1286 Ok(false)
1287 }
1288 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1289 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1290 }
1291 }
1292 }
1293}
1294
1295fn restore_job_name(restore: &KanidmRestore) -> String {
1296 format!("{}-restore", restore.name_any())
1297}
1298fn verify_job_name(restore: &KanidmRestore) -> String {
1299 format!("{}-verify", restore.name_any())
1300}
1301
1302fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1303 let origin = target
1304 .spec
1305 .origin
1306 .clone()
1307 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1308 vec![
1309 EnvVar {
1310 name: "KANIDM_DB_PATH".to_string(),
1311 value: Some("/data/kanidm.db".to_string()),
1312 ..Default::default()
1313 },
1314 EnvVar {
1315 name: "KANIDM_DOMAIN".to_string(),
1316 value: Some(target.spec.domain.clone()),
1317 ..Default::default()
1318 },
1319 EnvVar {
1320 name: "KANIDM_ORIGIN".to_string(),
1321 value: Some(origin),
1322 ..Default::default()
1323 },
1324 EnvVar {
1325 name: "KANIDM_TLS_CHAIN".to_string(),
1326 value: Some(format!("{TLS_PATH}/tls.crt")),
1327 ..Default::default()
1328 },
1329 EnvVar {
1330 name: "KANIDM_TLS_KEY".to_string(),
1331 value: Some(format!("{TLS_PATH}/tls.key")),
1332 ..Default::default()
1333 },
1334 ]
1335}
1336
1337async fn ensure_database_job(
1338 restore: &KanidmRestore,
1339 target: &Kanidm,
1340 ctx: &RestoreContext,
1341 name: &str,
1342 verify: bool,
1343) -> Result<()> {
1344 let ns = restore.namespace().unwrap();
1345 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1346 if api
1347 .get_opt(name)
1348 .await
1349 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1350 .is_some()
1351 {
1352 return Ok(());
1353 }
1354 let mut command = vec![
1355 "kanidmd".to_string(),
1356 "database".to_string(),
1357 if verify { "backup" } else { "restore" }.to_string(),
1358 ];
1359 if verify {
1360 command.push("/verify/verification.json.gz".to_string());
1361 } else {
1362 if is_remote_source(restore) {
1363 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1364 } else {
1365 let file_name = restore
1366 .spec
1367 .source
1368 .local
1369 .as_ref()
1370 .map(|l| l.file_name.clone())
1371 .unwrap_or_default();
1372 command.push(format!("{BACKUP_PATH}/{file_name}"));
1373 }
1374 }
1375 let job = Job {
1376 metadata: ObjectMeta {
1377 name: Some(name.to_string()),
1378 namespace: Some(ns.clone()),
1379 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1380 ..Default::default()
1381 },
1382 spec: Some(JobSpec {
1383 backoff_limit: Some(0),
1384 template: PodTemplateSpec {
1385 metadata: Some(ObjectMeta::default()),
1386 spec: Some(PodSpec {
1387 automount_service_account_token: Some(false),
1388 restart_policy: Some("Never".to_string()),
1389 security_context: Some(source_job_pod_security_context(target)),
1390 containers: vec![Container {
1391 name: if verify { "verify" } else { "restore" }.to_string(),
1392 image: Some(restore.spec.restore_image.clone()),
1393 command: Some(command),
1394 env: Some(build_kanidm_db_env_vars(target)),
1395 security_context: Some(kanidm_job_security_context()),
1396 resources: Some(job_resource_requirements()),
1397 volume_mounts: Some(vec![
1398 VolumeMount {
1399 name: DATA_VOLUME.to_string(),
1400 mount_path: DATA_PATH.to_string(),
1401 ..Default::default()
1402 },
1403 VolumeMount {
1404 name: TLS_VOLUME.to_string(),
1405 mount_path: TLS_PATH.to_string(),
1406 read_only: Some(true),
1407 ..Default::default()
1408 },
1409 VolumeMount {
1410 name: "verification-output".to_string(),
1411 mount_path: "/verify".to_string(),
1412 ..Default::default()
1413 },
1414 ]),
1415 ..Default::default()
1416 }],
1417 volumes: Some(vec![
1418 Volume {
1419 name: DATA_VOLUME.to_string(),
1420 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1421 claim_name: primary_pvc_name(target)?,
1422 read_only: Some(false),
1423 }),
1424 ..Default::default()
1425 },
1426 Volume {
1427 name: TLS_VOLUME.to_string(),
1428 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1429 secret_name: Some(target.effective_tls_secret_name()),
1430 ..Default::default()
1431 }),
1432 ..Default::default()
1433 },
1434 Volume {
1435 name: "verification-output".to_string(),
1436 empty_dir: Some(Default::default()),
1437 ..Default::default()
1438 },
1439 ]),
1440 ..Default::default()
1441 }),
1442 },
1443 ..Default::default()
1444 }),
1445 ..Default::default()
1446 };
1447 api.create(&PostParams::default(), &job)
1448 .await
1449 .map(|_| ())
1450 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1451}
1452
1453enum JobState {
1454 Running,
1455 Complete,
1456 Failed,
1457}
1458
1459async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1460 let ns = restore.namespace().unwrap();
1461 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1462 .get(name)
1463 .await
1464 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1465 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1466 return Ok(JobState::Failed);
1467 }
1468 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1469 return Ok(JobState::Complete);
1470 }
1471 Ok(JobState::Running)
1472}
1473
1474fn is_remote_source(restore: &KanidmRestore) -> bool {
1475 restore.spec.source.backup_ref.is_some()
1476}
1477
1478fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1479 let safety = restore.spec.safety_backup.as_ref();
1480 let skip = safety.is_some_and(|s| s.skip);
1481 !skip
1482}
1483
1484fn validate_source(restore: &KanidmRestore) -> Result<()> {
1485 let has_local = restore.spec.source.local.is_some();
1486 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1487 if has_local && has_backup_ref {
1488 return Err(Error::MissingData(
1489 "source.local and source.backupRef are mutually exclusive".to_string(),
1490 ));
1491 }
1492 if !has_local && !has_backup_ref {
1493 return Err(Error::MissingData(
1494 "source must specify either local or backupRef".to_string(),
1495 ));
1496 }
1497 if let Some(local) = &restore.spec.source.local {
1498 if !safe_basename(&local.file_name) {
1499 return Err(Error::MissingData(
1500 "restore source fileName must be a safe basename".to_string(),
1501 ));
1502 }
1503 }
1504 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1505 if backup_ref.name.is_empty() {
1506 return Err(Error::MissingData(
1507 "source.backupRef.name must not be empty".to_string(),
1508 ));
1509 }
1510 }
1511 Ok(())
1512}
1513
1514fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1515 let safety = restore.spec.safety_backup.as_ref();
1516 let skip = safety.is_some_and(|s| s.skip);
1517 if skip {
1518 let annotations = restore.metadata.annotations.as_ref();
1519 let reason = annotations
1520 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1521 .map(|s| s.trim())
1522 .unwrap_or("");
1523 let approver = annotations
1524 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1525 .map(|s| s.trim())
1526 .unwrap_or("");
1527 if reason.is_empty() {
1528 return Err(Error::MissingData(format!(
1529 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1530 )));
1531 }
1532 if approver.is_empty() {
1533 return Err(Error::MissingData(format!(
1534 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1535 )));
1536 }
1537 }
1538 if is_remote_source(restore) && !skip {
1539 let has_repo = safety
1540 .and_then(|s| s.repository_ref.as_ref())
1541 .is_some_and(|r| !r.name.is_empty());
1542 if !has_repo {
1543 return Err(Error::MissingData(
1544 "remote restore requires safetyBackup.repositoryRef".to_string(),
1545 ));
1546 }
1547 }
1548 Ok(())
1549}
1550
1551fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1552 target
1553 .spec
1554 .replica_groups
1555 .iter()
1556 .map(|rg| ReplicaCountEntry {
1557 group: rg.name.clone(),
1558 replicas: rg.replicas,
1559 })
1560 .collect()
1561}
1562
1563async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564 if let Ok(target) = get_target(restore, ctx).await {
1565 let owns_maintenance =
1566 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1567 if owns_maintenance {
1568 scale_desired(&target, ctx).await?;
1569 clear_restoring(restore, &target, ctx).await?;
1570 }
1571 }
1572 Ok(())
1573}
1574
1575async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1576 ctx.metrics.break_glass_total.add(1, &[]);
1577 let reason = restore
1578 .metadata
1579 .annotations
1580 .as_ref()
1581 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1582 .cloned()
1583 .unwrap_or_default();
1584 let approver = restore
1585 .metadata
1586 .annotations
1587 .as_ref()
1588 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1589 .cloned()
1590 .unwrap_or_default();
1591 let mut status = restore.status.clone().unwrap_or_default();
1592 let break_glass_condition = restore_condition(
1593 &status.conditions,
1594 CONDITION_BREAK_GLASS,
1595 CONDITION_TRUE,
1596 "BreakGlassOverride",
1597 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1598 restore.metadata.generation,
1599 );
1600 status
1601 .conditions
1602 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1603 status.conditions.push(break_glass_condition);
1604 patch_status(restore, ctx, status).await?;
1605 if let Err(error) = ctx
1606 .recorder
1607 .publish(
1608 &Event {
1609 type_: EventType::Warning,
1610 reason: "SafetyBackupSkipped".to_string(),
1611 note: Some(format!(
1612 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1613 )),
1614 action: "Restore".to_string(),
1615 secondary: None,
1616 },
1617 &restore.object_ref(&()),
1618 )
1619 .await
1620 {
1621 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1622 }
1623 warn!(
1624 restore = %restore.name_any(),
1625 reason = %reason,
1626 approver = %approver,
1627 "break-glass safety backup override activated"
1628 );
1629 Ok(())
1630}
1631
1632async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1633 let status = restore.status.as_ref().cloned().unwrap_or_default();
1634 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1635 if let Ok(start_ts) = start.parse::<Timestamp>() {
1636 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1637 ctx.metrics
1638 .safety_backup_duration_seconds
1639 .record(elapsed, &[]);
1640 }
1641 }
1642}
1643
1644fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1645 format!("{}-safety-backup", restore.name_any())
1646}
1647
1648fn source_prep_job_name(restore: &KanidmRestore) -> String {
1649 format!("{}-source-prep", restore.name_any())
1650}
1651
1652fn source_check_job_name(restore: &KanidmRestore) -> String {
1653 format!("{}-source-check", restore.name_any())
1654}
1655
1656async fn ensure_source_check_job(
1657 restore: &KanidmRestore,
1658 target: &Kanidm,
1659 ctx: &RestoreContext,
1660 name: &str,
1661) -> Result<()> {
1662 let ns = restore.namespace().unwrap();
1663 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1664 if api
1665 .get_opt(name)
1666 .await
1667 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1668 .is_some()
1669 {
1670 return Ok(());
1671 }
1672 let file_name = restore
1673 .spec
1674 .source
1675 .local
1676 .as_ref()
1677 .map(|l| l.file_name.clone())
1678 .unwrap_or_default();
1679 let operation_doc = serde_json::json!({
1680 "apiVersion": "backup.kaniop.rs/v1alpha1",
1681 "kind": "OperationDocument",
1682 "operation": "check",
1683 "path": format!("{BACKUP_PATH}/{file_name}"),
1684 "resultPath": "/run/kaniop-result/result.json",
1685 "format": "kanidmJsonGzip",
1686 })
1687 .to_string();
1688 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1689 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1690 let data_mover_image = data_mover_image();
1691 let job = Job {
1692 metadata: ObjectMeta {
1693 name: Some(name.to_string()),
1694 namespace: Some(ns.clone()),
1695 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1696 ..Default::default()
1697 },
1698 spec: Some(JobSpec {
1699 backoff_limit: Some(0),
1700 template: PodTemplateSpec {
1701 metadata: Some(ObjectMeta::default()),
1702 spec: Some(PodSpec {
1703 automount_service_account_token: Some(false),
1704 restart_policy: Some("Never".to_string()),
1705 security_context: Some(source_job_pod_security_context(target)),
1706 containers: vec![Container {
1707 name: "source-check".to_string(),
1708 image: Some(data_mover_image),
1709 command: Some(vec![
1710 "/bin/kaniop-data-mover".to_string(),
1711 "check".to_string(),
1712 "--operation-doc".to_string(),
1713 "/run/kaniop/operation.json".to_string(),
1714 ]),
1715 security_context: Some(hardened_security_context()),
1716 resources: Some(job_resource_requirements()),
1717 env: Some(vec![EnvVar {
1718 name: "RUST_LOG".to_string(),
1719 value: Some("info".to_string()),
1720 ..Default::default()
1721 }]),
1722 volume_mounts: Some(vec![
1723 VolumeMount {
1724 name: DATA_VOLUME.to_string(),
1725 mount_path: DATA_PATH.to_string(),
1726 read_only: Some(true),
1727 ..Default::default()
1728 },
1729 VolumeMount {
1730 name: "operation".to_string(),
1731 mount_path: "/run/kaniop".to_string(),
1732 read_only: Some(true),
1733 ..Default::default()
1734 },
1735 VolumeMount {
1736 name: "result".to_string(),
1737 mount_path: "/run/kaniop-result".to_string(),
1738 ..Default::default()
1739 },
1740 ]),
1741 ..Default::default()
1742 }],
1743 volumes: Some(vec![
1744 Volume {
1745 name: DATA_VOLUME.to_string(),
1746 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1747 claim_name: primary_pvc_name(target)?,
1748 read_only: Some(true),
1749 }),
1750 ..Default::default()
1751 },
1752 Volume {
1753 name: "operation".to_string(),
1754 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1755 name: operation_cm_name,
1756 ..Default::default()
1757 }),
1758 ..Default::default()
1759 },
1760 Volume {
1761 name: "result".to_string(),
1762 empty_dir: Some(Default::default()),
1763 ..Default::default()
1764 },
1765 ]),
1766 ..Default::default()
1767 }),
1768 },
1769 ..Default::default()
1770 }),
1771 ..Default::default()
1772 };
1773 api.create(&PostParams::default(), &job)
1774 .await
1775 .map(|_| ())
1776 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1777}
1778
1779const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1780
1781fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1782 let ns = restore.namespace().unwrap_or_default();
1783 let name = restore.name_any();
1784 let namespace_uuid =
1785 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1786 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1787 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1788}
1789
1790fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1791 let ns = restore.namespace().unwrap_or_default();
1792 let kanidm_uid = &restore.spec.target_ref.uid;
1793 let backup_id = compute_safety_backup_id(restore);
1794 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1795}
1796
1797struct VerifiedSafetyBackup {
1798 manifest_key: String,
1799 payload_sha256: String,
1800}
1801
1802async fn read_safety_backup_result(
1803 restore: &KanidmRestore,
1804 ctx: &RestoreContext,
1805 job_name: &str,
1806 expected_backup_id: &str,
1807) -> Result<VerifiedSafetyBackup> {
1808 let ns = restore.namespace().unwrap();
1809 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1810 let pod_list = pods
1811 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1812 .await
1813 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1814 let pod = pod_list.items.first().ok_or_else(|| {
1815 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1816 })?;
1817 let pod_name = pod.name_any();
1818 let container_status = pod
1819 .status
1820 .as_ref()
1821 .and_then(|s| s.container_statuses.as_ref())
1822 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1823 .ok_or_else(|| {
1824 Error::MissingData(format!(
1825 "safety-upload container status not found in pod {pod_name}"
1826 ))
1827 })?;
1828 let termination_message = container_status
1829 .state
1830 .as_ref()
1831 .and_then(|state| state.terminated.as_ref())
1832 .and_then(|t| t.message.as_ref())
1833 .ok_or_else(|| {
1834 Error::MissingData(format!(
1835 "termination message not found in pod {pod_name}; result transport failed"
1836 ))
1837 })?;
1838 if termination_message.is_empty() {
1839 return Err(Error::MissingData(format!(
1840 "termination message is empty in pod {pod_name}; result document was not written"
1841 )));
1842 }
1843 let result_doc = parse_result_document(termination_message).map_err(|e| {
1844 Error::ParseError(format!(
1845 "failed to parse result document from pod {pod_name}: {e}"
1846 ))
1847 })?;
1848 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1849 return Err(Error::ParseError(format!(
1850 "unsupported result document version: {}",
1851 result_doc.api_version
1852 )));
1853 }
1854 if result_doc.kind != "ResultDocument" {
1855 return Err(Error::ParseError(format!(
1856 "invalid result document kind: {}",
1857 result_doc.kind
1858 )));
1859 }
1860 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1861 return Err(Error::ParseError(format!(
1862 "result document operation mismatch: expected '{}', got '{}'",
1863 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1864 )));
1865 }
1866 if !result_doc.success {
1867 return Err(Error::ParseError(format!(
1868 "safety backup reported failure with exit code {:?}",
1869 result_doc.exit_code
1870 )));
1871 }
1872 if result_doc.exit_code != ExitCode::Success {
1873 return Err(Error::ParseError(format!(
1874 "safety backup exit code is not Success: {:?}",
1875 result_doc.exit_code
1876 )));
1877 }
1878 let doc_backup_id = result_doc
1879 .backup_id
1880 .as_deref()
1881 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1882 if doc_backup_id != expected_backup_id {
1883 return Err(Error::ParseError(format!(
1884 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1885 )));
1886 }
1887 let manifest_key = result_doc
1888 .manifest_key
1889 .clone()
1890 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1891 if manifest_key.is_empty() {
1892 return Err(Error::ParseError(
1893 "result document manifestKey is empty".to_string(),
1894 ));
1895 }
1896 let payload_sha256 = result_doc
1897 .payload_sha256
1898 .clone()
1899 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1900 if payload_sha256.is_empty() {
1901 return Err(Error::ParseError(
1902 "result document payloadSha256 is empty".to_string(),
1903 ));
1904 }
1905 info!(
1906 restore = %restore.name_any(),
1907 backup_id = %expected_backup_id,
1908 manifest_key = %manifest_key,
1909 "safety backup result verified"
1910 );
1911 Ok(VerifiedSafetyBackup {
1912 manifest_key,
1913 payload_sha256,
1914 })
1915}
1916
1917struct VerifiedSourcePrepResult {
1918 #[allow(dead_code)]
1919 manifest_key: String,
1920 payload_sha256: String,
1921}
1922
1923async fn read_source_prep_result(
1924 restore: &KanidmRestore,
1925 ctx: &RestoreContext,
1926 job_name: &str,
1927 expected_backup_id: &str,
1928) -> Result<VerifiedSourcePrepResult> {
1929 let ns = restore.namespace().unwrap();
1930 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1931 let pod_list = pods
1932 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1933 .await
1934 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1935 let pod = pod_list.items.first().ok_or_else(|| {
1936 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1937 })?;
1938 let pod_name = pod.name_any();
1939 let container_status = pod
1940 .status
1941 .as_ref()
1942 .and_then(|s| s.container_statuses.as_ref())
1943 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1944 .ok_or_else(|| {
1945 Error::MissingData(format!(
1946 "source-prep container status not found in pod {pod_name}"
1947 ))
1948 })?;
1949 let termination_message = container_status
1950 .state
1951 .as_ref()
1952 .and_then(|state| state.terminated.as_ref())
1953 .and_then(|t| t.message.as_ref())
1954 .ok_or_else(|| {
1955 Error::MissingData(format!(
1956 "termination message not found in pod {pod_name}; result transport failed"
1957 ))
1958 })?;
1959 if termination_message.is_empty() {
1960 return Err(Error::MissingData(format!(
1961 "termination message is empty in pod {pod_name}; result document was not written"
1962 )));
1963 }
1964 let result_doc = parse_result_document(termination_message).map_err(|e| {
1965 Error::ParseError(format!(
1966 "failed to parse result document from pod {pod_name}: {e}"
1967 ))
1968 })?;
1969 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1970 return Err(Error::ParseError(format!(
1971 "unsupported result document version: {}",
1972 result_doc.api_version
1973 )));
1974 }
1975 if result_doc.kind != "ResultDocument" {
1976 return Err(Error::ParseError(format!(
1977 "invalid result document kind: {}",
1978 result_doc.kind
1979 )));
1980 }
1981 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1982 return Err(Error::ParseError(format!(
1983 "result document operation mismatch: expected '{}', got '{}'",
1984 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1985 )));
1986 }
1987 if !result_doc.success {
1988 return Err(Error::ParseError(format!(
1989 "source prep reported failure with exit code {:?}",
1990 result_doc.exit_code
1991 )));
1992 }
1993 if result_doc.exit_code != ExitCode::Success {
1994 return Err(Error::ParseError(format!(
1995 "source prep exit code is not Success: {:?}",
1996 result_doc.exit_code
1997 )));
1998 }
1999 let doc_backup_id = result_doc
2000 .backup_id
2001 .as_deref()
2002 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
2003 if doc_backup_id != expected_backup_id {
2004 return Err(Error::ParseError(format!(
2005 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2006 )));
2007 }
2008 let manifest_key = result_doc
2009 .manifest_key
2010 .clone()
2011 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2012 if manifest_key.is_empty() {
2013 return Err(Error::ParseError(
2014 "result document manifestKey is empty".to_string(),
2015 ));
2016 }
2017 let payload_sha256 = result_doc
2018 .payload_sha256
2019 .clone()
2020 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2021 if payload_sha256.is_empty() {
2022 return Err(Error::ParseError(
2023 "result document payloadSha256 is empty".to_string(),
2024 ));
2025 }
2026 info!(
2027 restore = %restore.name_any(),
2028 backup_id = %expected_backup_id,
2029 manifest_key = %manifest_key,
2030 payload_sha256 = %payload_sha256,
2031 "source prep result verified"
2032 );
2033 Ok(VerifiedSourcePrepResult {
2034 manifest_key,
2035 payload_sha256,
2036 })
2037}
2038
2039fn source_job_pod_security_context(
2040 target: &Kanidm,
2041) -> k8s_openapi::api::core::v1::PodSecurityContext {
2042 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
2043 security_context.seccomp_profile = Some(SeccompProfile {
2044 type_: "RuntimeDefault".to_string(),
2045 ..Default::default()
2046 });
2047 security_context
2048}
2049
2050fn hardened_security_context() -> SecurityContext {
2051 SecurityContext {
2052 allow_privilege_escalation: Some(false),
2053 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2054 drop: Some(vec!["ALL".to_string()]),
2055 ..Default::default()
2056 }),
2057 read_only_root_filesystem: Some(true),
2058 run_as_non_root: Some(true),
2059 seccomp_profile: Some(SeccompProfile {
2060 type_: "RuntimeDefault".to_string(),
2061 ..Default::default()
2062 }),
2063 ..Default::default()
2064 }
2065}
2066
2067fn kanidm_job_security_context() -> SecurityContext {
2068 SecurityContext {
2069 allow_privilege_escalation: Some(false),
2070 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2071 drop: Some(vec!["ALL".to_string()]),
2072 ..Default::default()
2073 }),
2074 seccomp_profile: Some(SeccompProfile {
2075 type_: "RuntimeDefault".to_string(),
2076 ..Default::default()
2077 }),
2078 ..Default::default()
2079 }
2080}
2081
2082fn job_resource_requirements() -> ResourceRequirements {
2083 ResourceRequirements {
2084 requests: Some(std::collections::BTreeMap::from([
2085 ("cpu".to_string(), Quantity("100m".to_string())),
2086 ("memory".to_string(), Quantity("128Mi".to_string())),
2087 ])),
2088 limits: Some(std::collections::BTreeMap::from([
2089 ("cpu".to_string(), Quantity("2".to_string())),
2090 ("memory".to_string(), Quantity("2Gi".to_string())),
2091 ])),
2092 ..Default::default()
2093 }
2094}
2095
2096async fn ensure_safety_backup_job(
2097 restore: &KanidmRestore,
2098 target: &Kanidm,
2099 ctx: &RestoreContext,
2100 name: &str,
2101 backup_id: &str,
2102 manifest_key: &str,
2103) -> Result<()> {
2104 let ns = restore.namespace().unwrap();
2105 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2106 if api
2107 .get_opt(name)
2108 .await
2109 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2110 .is_some()
2111 {
2112 return Ok(());
2113 }
2114 let safety = restore
2115 .spec
2116 .safety_backup
2117 .as_ref()
2118 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2119 let repo_ref = safety
2120 .repository_ref
2121 .as_ref()
2122 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2123 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2124 .get(&repo_ref.name)
2125 .await
2126 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2127 let _namespace_uid = &ns;
2128 let _kanidm_uid = &restore.spec.target_ref.uid;
2129 let endpoint = &repo.spec.s3.endpoint;
2130 let region = &repo.spec.s3.region;
2131 let operation_doc = build_safety_upload_operation_doc(
2132 restore,
2133 target,
2134 backup_id,
2135 manifest_key,
2136 &repo.spec.s3.bucket,
2137 &repo.spec.s3.prefix,
2138 endpoint,
2139 region,
2140 repo.spec.s3.force_path_style,
2141 repo.spec.s3.insecure,
2142 repo.spec
2143 .s3
2144 .ca_bundle_ref
2145 .as_deref()
2146 .map(|_| ca_bundle_path())
2147 .as_deref(),
2148 repo.spec.encryption.as_ref(),
2149 )?;
2150 let operation_cm_name = format!("{name}-op");
2151 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2152 let auth_method = &repo.spec.authentication.writer;
2153 let data_mover_image = data_mover_image();
2154 let init_command = vec![
2155 "kanidmd".to_string(),
2156 "database".to_string(),
2157 "backup".to_string(),
2158 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2159 ];
2160 let upload_script = format!(
2161 r#"set -eu
2162/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2163RESULT_FILE="/run/kaniop-result/result.json"
2164TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2165if [ ! -f "$RESULT_FILE" ]; then
2166 echo "ERROR: result document not found at $RESULT_FILE"
2167 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2168 exit 1
2169fi
2170if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2171 echo "ERROR: safety backup upload reported failure"
2172 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2173 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2174 exit 1
2175fi
2176dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2177echo "safety backup upload completed successfully"
2178"#
2179 );
2180 let job = Job {
2181 metadata: ObjectMeta {
2182 name: Some(name.to_string()),
2183 namespace: Some(ns.clone()),
2184 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2185 ..Default::default()
2186 },
2187 spec: Some(JobSpec {
2188 backoff_limit: Some(0),
2189 template: PodTemplateSpec {
2190 metadata: Some(ObjectMeta::default()),
2191 spec: Some(PodSpec {
2192 automount_service_account_token: Some(false),
2193 restart_policy: Some("Never".to_string()),
2194 security_context: Some(source_job_pod_security_context(target)),
2195 init_containers: Some(vec![Container {
2196 name: "safety-backup".to_string(),
2197 image: Some(restore.spec.restore_image.clone()),
2198 command: Some(init_command),
2199 security_context: Some(SecurityContext {
2200 allow_privilege_escalation: Some(false),
2201 capabilities: Some(Capabilities {
2202 drop: Some(vec!["ALL".to_string()]),
2203 ..Default::default()
2204 }),
2205 read_only_root_filesystem: Some(true),
2206 seccomp_profile: Some(SeccompProfile {
2207 type_: "RuntimeDefault".to_string(),
2208 ..Default::default()
2209 }),
2210 ..Default::default()
2211 }),
2212 resources: Some(job_resource_requirements()),
2213 env: Some(build_kanidm_db_env_vars(target)),
2214 volume_mounts: Some(vec![
2215 VolumeMount {
2216 name: DATA_VOLUME.to_string(),
2217 mount_path: DATA_PATH.to_string(),
2218 read_only: Some(false),
2219 ..Default::default()
2220 },
2221 VolumeMount {
2222 name: TLS_VOLUME.to_string(),
2223 mount_path: TLS_PATH.to_string(),
2224 read_only: Some(true),
2225 ..Default::default()
2226 },
2227 VolumeMount {
2228 name: SHARED_VOLUME.to_string(),
2229 mount_path: SHARED_VOL_PATH.to_string(),
2230 ..Default::default()
2231 },
2232 ]),
2233 ..Default::default()
2234 }]),
2235 containers: vec![Container {
2236 name: "safety-upload".to_string(),
2237 image: Some(data_mover_image),
2238 command: Some(vec!["/bin/sh".to_string()]),
2239 args: Some(vec!["-c".to_string(), upload_script]),
2240 security_context: Some(hardened_security_context()),
2241 resources: Some(job_resource_requirements()),
2242 env: {
2243 let mut env_vars =
2244 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2245 env_vars.push(EnvVar {
2246 name: "RUST_LOG".to_string(),
2247 value: Some("info".to_string()),
2248 ..Default::default()
2249 });
2250 if repo.spec.s3.ca_bundle_ref.is_some() {
2251 env_vars.push(ca_bundle_env_var());
2252 }
2253 env_vars.extend(build_encryption_env_vars(
2254 repo.spec
2255 .encryption
2256 .as_ref()
2257 .and_then(|e| e.key_ref.as_ref()),
2258 ));
2259 Some(env_vars)
2260 },
2261 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2262 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2263 volume_mounts: {
2264 let mut mounts = vec![
2265 VolumeMount {
2266 name: SHARED_VOLUME.to_string(),
2267 mount_path: SHARED_VOL_PATH.to_string(),
2268 read_only: Some(true),
2269 ..Default::default()
2270 },
2271 VolumeMount {
2272 name: "operation".to_string(),
2273 mount_path: "/run/kaniop".to_string(),
2274 read_only: Some(true),
2275 ..Default::default()
2276 },
2277 VolumeMount {
2278 name: "result".to_string(),
2279 mount_path: "/run/kaniop-result".to_string(),
2280 ..Default::default()
2281 },
2282 ];
2283 mounts.extend(build_auth_volume_mounts(auth_method));
2284 if repo.spec.s3.ca_bundle_ref.is_some() {
2285 mounts.push(build_ca_bundle_volume_mount());
2286 }
2287 Some(mounts)
2288 },
2289 ..Default::default()
2290 }],
2291 volumes: {
2292 let mut vols = vec![
2293 Volume {
2294 name: DATA_VOLUME.to_string(),
2295 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2296 claim_name: primary_pvc_name(target)?,
2297 read_only: Some(false),
2298 }),
2299 ..Default::default()
2300 },
2301 Volume {
2302 name: TLS_VOLUME.to_string(),
2303 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2304 secret_name: Some(target.effective_tls_secret_name()),
2305 ..Default::default()
2306 }),
2307 ..Default::default()
2308 },
2309 Volume {
2310 name: SHARED_VOLUME.to_string(),
2311 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2312 size_limit: Some(crate::controller::backup_job_volume_size()),
2313 ..Default::default()
2314 }),
2315 ..Default::default()
2316 },
2317 Volume {
2318 name: "operation".to_string(),
2319 config_map: Some(ConfigMapVolumeSource {
2320 name: operation_cm_name,
2321 ..Default::default()
2322 }),
2323 ..Default::default()
2324 },
2325 Volume {
2326 name: "result".to_string(),
2327 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2328 size_limit: Some(Quantity("16Mi".to_string())),
2329 ..Default::default()
2330 }),
2331 ..Default::default()
2332 },
2333 ];
2334 vols.extend(build_auth_volumes(auth_method));
2335 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2336 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2337 }
2338 Some(vols)
2339 },
2340 ..Default::default()
2341 }),
2342 },
2343 ..Default::default()
2344 }),
2345 ..Default::default()
2346 };
2347 api.create(&PostParams::default(), &job)
2348 .await
2349 .map(|_| ())
2350 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2351}
2352
2353async fn ensure_source_prep_job(
2354 restore: &KanidmRestore,
2355 target: &Kanidm,
2356 ctx: &RestoreContext,
2357 name: &str,
2358) -> Result<()> {
2359 let ns = restore.namespace().unwrap();
2360 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2361 if api
2362 .get_opt(name)
2363 .await
2364 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2365 .is_some()
2366 {
2367 return Ok(());
2368 }
2369 let backup_name = restore
2370 .spec
2371 .source
2372 .backup_ref
2373 .as_ref()
2374 .map(|b| b.name.clone())
2375 .unwrap_or_default();
2376 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2377 .get(&backup_name)
2378 .await
2379 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2380 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2381 .get(&backup.spec.repository_ref.name)
2382 .await
2383 .map_err(|e| {
2384 Error::kube_error(
2385 "get",
2386 "KanidmBackupRepository",
2387 &ns,
2388 &backup.spec.repository_ref.name,
2389 e,
2390 )
2391 })?;
2392 let endpoint = &repo.spec.s3.endpoint;
2393 let region = &repo.spec.s3.region;
2394 let operation_doc = build_download_operation_doc(
2395 restore,
2396 target,
2397 &backup.spec.manifest_key,
2398 &backup.spec.backup_id,
2399 &repo.spec.s3.bucket,
2400 &repo.spec.s3.prefix,
2401 endpoint,
2402 region,
2403 repo.spec.s3.force_path_style,
2404 repo.spec.s3.insecure,
2405 repo.spec
2406 .s3
2407 .ca_bundle_ref
2408 .as_deref()
2409 .map(|_| ca_bundle_path())
2410 .as_deref(),
2411 repo.spec.encryption.as_ref(),
2412 );
2413 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2414 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2415 let auth_method = &repo.spec.authentication.reader;
2416 let data_mover_image = data_mover_image();
2417 let download_script = format!(
2418 r#"set -eu
2419/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2420RESULT_FILE="/run/kaniop-result/result.json"
2421TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2422if [ ! -f "$RESULT_FILE" ]; then
2423 echo "ERROR: result document not found at $RESULT_FILE"
2424 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2425 exit 1
2426fi
2427if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2428 echo "ERROR: source preparation download reported failure"
2429 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2430 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2431 exit 1
2432fi
2433dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2434echo "source preparation download completed successfully"
2435"#
2436 );
2437 let job = Job {
2438 metadata: ObjectMeta {
2439 name: Some(name.to_string()),
2440 namespace: Some(ns.clone()),
2441 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2442 ..Default::default()
2443 },
2444 spec: Some(JobSpec {
2445 backoff_limit: Some(0),
2446 template: PodTemplateSpec {
2447 metadata: Some(ObjectMeta::default()),
2448 spec: Some(PodSpec {
2449 automount_service_account_token: Some(false),
2450 restart_policy: Some("Never".to_string()),
2451 security_context: Some(source_job_pod_security_context(target)),
2452 containers: vec![Container {
2453 name: "source-prep".to_string(),
2454 image: Some(data_mover_image),
2455 command: Some(vec!["/bin/sh".to_string()]),
2456 args: Some(vec!["-c".to_string(), download_script]),
2457 security_context: Some(hardened_security_context()),
2458 resources: Some(job_resource_requirements()),
2459 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2460 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2461 env: {
2462 let mut env_vars = build_auth_env_vars(
2463 auth_method,
2464 &backup.spec.repository_ref.name,
2465 AuthRole::Reader,
2466 );
2467 env_vars.push(EnvVar {
2468 name: "RUST_LOG".to_string(),
2469 value: Some("info".to_string()),
2470 ..Default::default()
2471 });
2472 if repo.spec.s3.ca_bundle_ref.is_some() {
2473 env_vars.push(ca_bundle_env_var());
2474 }
2475 env_vars.extend(build_encryption_env_vars(
2476 repo.spec
2477 .encryption
2478 .as_ref()
2479 .and_then(|e| e.key_ref.as_ref()),
2480 ));
2481 Some(env_vars)
2482 },
2483 volume_mounts: {
2484 let mut mounts = vec![
2485 VolumeMount {
2486 name: DATA_VOLUME.to_string(),
2487 mount_path: DATA_PATH.to_string(),
2488 ..Default::default()
2489 },
2490 VolumeMount {
2491 name: "operation".to_string(),
2492 mount_path: "/run/kaniop".to_string(),
2493 read_only: Some(true),
2494 ..Default::default()
2495 },
2496 VolumeMount {
2497 name: "result".to_string(),
2498 mount_path: "/run/kaniop-result".to_string(),
2499 ..Default::default()
2500 },
2501 ];
2502 mounts.extend(build_auth_volume_mounts(auth_method));
2503 if repo.spec.s3.ca_bundle_ref.is_some() {
2504 mounts.push(build_ca_bundle_volume_mount());
2505 }
2506 Some(mounts)
2507 },
2508 ..Default::default()
2509 }],
2510 volumes: {
2511 let mut vols = vec![
2512 Volume {
2513 name: DATA_VOLUME.to_string(),
2514 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2515 claim_name: primary_pvc_name(target)?,
2516 read_only: Some(false),
2517 }),
2518 ..Default::default()
2519 },
2520 Volume {
2521 name: "operation".to_string(),
2522 config_map: Some(ConfigMapVolumeSource {
2523 name: operation_cm_name,
2524 ..Default::default()
2525 }),
2526 ..Default::default()
2527 },
2528 Volume {
2529 name: "result".to_string(),
2530 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2531 size_limit: Some(Quantity("16Mi".to_string())),
2532 ..Default::default()
2533 }),
2534 ..Default::default()
2535 },
2536 ];
2537 vols.extend(build_auth_volumes(auth_method));
2538 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2539 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2540 }
2541 Some(vols)
2542 },
2543 ..Default::default()
2544 }),
2545 },
2546 ..Default::default()
2547 }),
2548 ..Default::default()
2549 };
2550 api.create(&PostParams::default(), &job)
2551 .await
2552 .map(|_| ())
2553 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2554}
2555
2556#[allow(clippy::too_many_arguments)]
2557fn build_safety_upload_operation_doc(
2558 restore: &KanidmRestore,
2559 target: &Kanidm,
2560 backup_id: &str,
2561 manifest_key: &str,
2562 bucket: &str,
2563 prefix: &str,
2564 endpoint: &str,
2565 region: &str,
2566 force_path_style: bool,
2567 insecure: bool,
2568 ca_bundle_path: Option<&str>,
2569 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2570) -> Result<String> {
2571 let enc_mode =
2572 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2573 let enc_key_id = encryption
2574 .and_then(|e| e.key_id.as_ref())
2575 .map(|s| serde_json::Value::String(s.clone()));
2576 let op = UploadOperation {
2577 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2578 bucket: bucket.to_string(),
2579 prefix: prefix.to_string(),
2580 endpoint: endpoint.to_string(),
2581 region: region.to_string(),
2582 force_path_style,
2583 insecure,
2584 ca_bundle_path: ca_bundle_path.map(str::to_string),
2585 backup_id: backup_id.to_string(),
2586 namespace_uid: restore.namespace().unwrap_or_default(),
2587 kanidm_uid: restore.spec.target_ref.uid.clone(),
2588 kanidm_name: restore.spec.target_ref.name.clone(),
2589 domain: target.spec.domain.clone(),
2590 kanidm_version: target
2591 .status
2592 .as_ref()
2593 .and_then(|s| s.version.as_ref())
2594 .map(|v| v.image_tag.clone())
2595 .unwrap_or_default(),
2596 image_digest: None,
2597 consistency: "kanidm-offline".to_string(),
2598 reason: "restore-safety".to_string(),
2599 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2600 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2601 result_path: "/run/kaniop-result/result.json".to_string(),
2602 max_concurrent_parts: 4,
2603 max_retries: 3,
2604 };
2605 let _ = manifest_key;
2606 let doc = OperationDocument {
2607 api_version: OPERATION_DOC_VERSION.to_string(),
2608 kind: "OperationDocument".to_string(),
2609 spec: OperationSpec::Upload(op),
2610 };
2611 serde_json::to_string(&doc)
2612 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2613}
2614
2615async fn validate_backup_ref(
2616 restore: &KanidmRestore,
2617 target: &Kanidm,
2618 ctx: &RestoreContext,
2619) -> Result<()> {
2620 let backup_ref = restore
2621 .spec
2622 .source
2623 .backup_ref
2624 .as_ref()
2625 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2626 let ns = restore.namespace().unwrap();
2627 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2628 .get(&backup_ref.name)
2629 .await
2630 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2631 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2632 return Err(Error::MissingData(format!(
2633 "KanidmBackup '{}' is not Ready (phase: {:?})",
2634 backup_ref.name,
2635 backup.status.as_ref().map(|s| &s.phase)
2636 )));
2637 }
2638 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2639 return Err(Error::MissingData(format!(
2640 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2641 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2642 )));
2643 }
2644 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2645 return Err(Error::MissingData(format!(
2646 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2647 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2648 )));
2649 }
2650 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2651 .get(&backup.spec.repository_ref.name)
2652 .await
2653 .map_err(|e| {
2654 Error::kube_error(
2655 "get",
2656 "KanidmBackupRepository",
2657 &ns,
2658 &backup.spec.repository_ref.name,
2659 e,
2660 )
2661 })?;
2662 if !has_accepted_condition(
2663 &repo
2664 .status
2665 .as_ref()
2666 .map(|s| &s.conditions)
2667 .cloned()
2668 .unwrap_or_default(),
2669 ) {
2670 return Err(Error::MissingData(format!(
2671 "KanidmBackupRepository '{}' configuration has not been accepted",
2672 backup.spec.repository_ref.name
2673 )));
2674 }
2675 validate_backup_compatibility(restore, target, &backup)?;
2676 Ok(())
2677}
2678
2679fn has_accepted_condition(conditions: &[Condition]) -> bool {
2680 conditions
2681 .iter()
2682 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2683}
2684
2685fn validate_backup_compatibility(
2686 restore: &KanidmRestore,
2687 target: &Kanidm,
2688 backup: &KanidmBackup,
2689) -> Result<()> {
2690 let status = backup.status.as_ref();
2691 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2692 let target_version = target
2693 .status
2694 .as_ref()
2695 .and_then(|s| s.version.as_ref())
2696 .map(|v| v.image_tag.as_str());
2697 if let Some(tv) = target_version {
2698 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2699 return Err(Error::MissingData(format!(
2700 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2701 )));
2702 }
2703 }
2704 }
2705 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2706 if !backup_digest.is_empty()
2707 && restore.spec.restore_image.contains('@')
2708 && !restore.spec.restore_image.contains(backup_digest)
2709 {
2710 return Err(Error::MissingData(format!(
2711 "restore image digest does not match backup image digest '{backup_digest}'"
2712 )));
2713 }
2714 }
2715 Ok(())
2716}
2717
2718#[allow(clippy::too_many_arguments)]
2719fn build_download_operation_doc(
2720 restore: &KanidmRestore,
2721 target: &Kanidm,
2722 manifest_key: &str,
2723 expected_backup_id: &str,
2724 bucket: &str,
2725 prefix: &str,
2726 endpoint: &str,
2727 region: &str,
2728 force_path_style: bool,
2729 insecure: bool,
2730 ca_bundle_path: Option<&str>,
2731 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2732) -> String {
2733 let enc_mode =
2734 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2735 let enc_key_id = encryption
2736 .and_then(|e| e.key_id.as_ref())
2737 .map(|s| serde_json::Value::String(s.clone()));
2738 serde_json::json!({
2739 "apiVersion": "backup.kaniop.rs/v1alpha1",
2740 "kind": "OperationDocument",
2741 "operation": "download",
2742 "manifestKey": manifest_key,
2743 "bucket": bucket,
2744 "prefix": prefix,
2745 "endpoint": endpoint,
2746 "region": region,
2747 "forcePathStyle": force_path_style,
2748 "insecure": insecure,
2749 "caBundlePath": ca_bundle_path,
2750 "expectedBackupId": expected_backup_id,
2751 "expectedKanidmUid": restore.spec.target_ref.uid,
2752 "expectedDomain": target.spec.domain,
2753 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2754 "resultPath": "/run/kaniop-result/result.json",
2755 "maxRetries": 3,
2756 "encryptionMode": enc_mode,
2757 "encryptionKeyId": enc_key_id,
2758 })
2759 .to_string()
2760}
2761
2762async fn ensure_operation_configmap(
2763 restore: &KanidmRestore,
2764 name: &str,
2765 data: &str,
2766 ns: &str,
2767 ctx: &RestoreContext,
2768) -> Result<()> {
2769 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2770 if api
2771 .get_opt(name)
2772 .await
2773 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2774 .is_some()
2775 {
2776 return Ok(());
2777 }
2778 let cm = ConfigMap {
2779 metadata: ObjectMeta {
2780 name: Some(name.to_string()),
2781 namespace: Some(ns.to_string()),
2782 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2783 ..Default::default()
2784 },
2785 data: Some(std::collections::BTreeMap::from([(
2786 "operation.json".to_string(),
2787 data.to_string(),
2788 )])),
2789 ..Default::default()
2790 };
2791 api.create(&PostParams::default(), &cm)
2792 .await
2793 .map(|_| ())
2794 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2795}
2796
2797#[cfg(test)]
2798mod tests {
2799 use super::{
2800 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2801 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2802 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2803 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2804 hardened_security_context, has_accepted_condition, is_remote_source,
2805 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2806 validate_safety_backup_config, validate_source,
2807 };
2808 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2809 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2810 use kube::api::ObjectMeta as ApiObjectMeta;
2811 use std::collections::BTreeMap;
2812
2813 fn make_restore(
2814 source: KanidmRestoreSource,
2815 safety: Option<SafetyBackupConfig>,
2816 ) -> KanidmRestore {
2817 KanidmRestore {
2818 metadata: ApiObjectMeta {
2819 name: Some("test-restore".to_string()),
2820 namespace: Some("default".to_string()),
2821 ..Default::default()
2822 },
2823 spec: KanidmRestoreSpec {
2824 target_ref: KanidmRestoreTargetRef {
2825 name: "corp-idm".to_string(),
2826 uid: "test-uid-123".to_string(),
2827 },
2828 source,
2829 restore_image: "kanidm/server@sha256:abc".to_string(),
2830 safety_backup: safety,
2831 },
2832 status: None,
2833 }
2834 }
2835
2836 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2837 super::super::crd::Kanidm {
2838 status: Some(super::super::crd::KanidmStatus {
2839 available_replicas: 1,
2840 replicas: 1,
2841 unavailable_replicas: 0,
2842 updated_replicas: 1,
2843 replica_statuses: vec![],
2844 replica_column: "1/1".to_string(),
2845 secret_name: None,
2846 version: Some(super::super::crd::KanidmVersionStatus {
2847 image_tag: image_tag.to_string(),
2848 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2849 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2850 }),
2851 domain_appearance_image: None,
2852 mail_sender: None,
2853 conditions: None,
2854 }),
2855 ..Default::default()
2856 }
2857 }
2858
2859 #[test]
2860 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2861 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2862 }
2863
2864 #[test]
2865 fn completed_restore_sets_ready_condition() {
2866 let mut status = KanidmRestoreStatus {
2867 phase: KanidmRestorePhase::Completed,
2868 ..Default::default()
2869 };
2870 super::update_restore_conditions(&mut status, Some(7));
2871 assert!(status.conditions.iter().any(|condition| {
2872 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2873 }));
2874 assert!(!status.conditions.iter().any(|condition| {
2875 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2876 }));
2877 }
2878
2879 #[test]
2880 fn failed_restore_sets_failed_condition() {
2881 let mut status = KanidmRestoreStatus {
2882 phase: KanidmRestorePhase::Failed,
2883 message: Some("verification failed".to_string()),
2884 ..Default::default()
2885 };
2886 super::update_restore_conditions(&mut status, Some(8));
2887 assert!(status.conditions.iter().any(|condition| {
2888 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2889 }));
2890 }
2891
2892 #[test]
2893 fn restore_filename_is_confined_to_backup_directory() {
2894 assert!(safe_basename("backup.json.gz"));
2895 assert!(!safe_basename("../backup.json.gz"));
2896 assert!(!safe_basename("a/b"));
2897 assert!(!safe_basename(""));
2898 }
2899
2900 #[test]
2901 fn restore_image_rejects_latest_and_unpinned_names() {
2902 assert!(mutable_image("kanidm/server:latest"));
2903 assert!(mutable_image("kanidm/server"));
2904 assert!(!mutable_image("kanidm/server:1.10.0"));
2905 assert!(!mutable_image("kanidm/server@sha256:abc"));
2906 }
2907
2908 #[test]
2909 fn source_validation_rejects_both_local_and_backup_ref() {
2910 let source = KanidmRestoreSource {
2911 local: Some(super::KanidmRestoreLocalSource {
2912 file_name: "backup.json".to_string(),
2913 }),
2914 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2915 name: "some-backup".to_string(),
2916 }),
2917 };
2918 let restore = make_restore(source, None);
2919 assert!(validate_source(&restore).is_err());
2920 }
2921
2922 #[test]
2923 fn source_validation_rejects_empty_source() {
2924 let source = KanidmRestoreSource {
2925 local: None,
2926 backup_ref: None,
2927 };
2928 let restore = make_restore(source, None);
2929 assert!(validate_source(&restore).is_err());
2930 }
2931
2932 #[test]
2933 fn source_validation_accepts_local_source() {
2934 let source = KanidmRestoreSource {
2935 local: Some(super::KanidmRestoreLocalSource {
2936 file_name: "backup.json".to_string(),
2937 }),
2938 backup_ref: None,
2939 };
2940 let restore = make_restore(source, None);
2941 assert!(validate_source(&restore).is_ok());
2942 }
2943
2944 #[test]
2945 fn source_validation_accepts_backup_ref_source() {
2946 let source = KanidmRestoreSource {
2947 local: None,
2948 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2949 name: "corp-idm-019c7c76".to_string(),
2950 }),
2951 };
2952 let restore = make_restore(source, None);
2953 assert!(validate_source(&restore).is_ok());
2954 }
2955
2956 #[test]
2957 fn source_validation_rejects_empty_backup_ref_name() {
2958 let source = KanidmRestoreSource {
2959 local: None,
2960 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2961 name: "".to_string(),
2962 }),
2963 };
2964 let restore = make_restore(source, None);
2965 assert!(validate_source(&restore).is_err());
2966 }
2967
2968 #[test]
2969 fn is_remote_source_true_for_backup_ref() {
2970 let source = KanidmRestoreSource {
2971 local: None,
2972 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2973 name: "backup-1".to_string(),
2974 }),
2975 };
2976 let restore = make_restore(source, None);
2977 assert!(is_remote_source(&restore));
2978 }
2979
2980 #[test]
2981 fn is_remote_source_false_for_local() {
2982 let source = KanidmRestoreSource {
2983 local: Some(super::KanidmRestoreLocalSource {
2984 file_name: "backup.json".to_string(),
2985 }),
2986 backup_ref: None,
2987 };
2988 let restore = make_restore(source, None);
2989 assert!(!is_remote_source(&restore));
2990 }
2991
2992 #[test]
2993 fn requires_safety_backup_default_true() {
2994 let source = KanidmRestoreSource {
2995 local: None,
2996 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2997 name: "backup-1".to_string(),
2998 }),
2999 };
3000 let restore = make_restore(source, None);
3001 assert!(requires_safety_backup(&restore));
3002 }
3003
3004 #[test]
3005 fn requires_safety_backup_false_when_skip() {
3006 let source = KanidmRestoreSource {
3007 local: None,
3008 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3009 name: "backup-1".to_string(),
3010 }),
3011 };
3012 let safety = SafetyBackupConfig {
3013 repository_ref: Some(SafetyBackupRepositoryRef {
3014 name: "offsite".to_string(),
3015 }),
3016 skip: true,
3017 };
3018 let restore = make_restore(source, Some(safety));
3019 assert!(!requires_safety_backup(&restore));
3020 }
3021
3022 #[test]
3023 fn break_glass_validation_rejects_skip_without_reason() {
3024 let source = KanidmRestoreSource {
3025 local: None,
3026 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3027 name: "backup-1".to_string(),
3028 }),
3029 };
3030 let safety = SafetyBackupConfig {
3031 repository_ref: Some(SafetyBackupRepositoryRef {
3032 name: "offsite".to_string(),
3033 }),
3034 skip: true,
3035 };
3036 let mut restore = make_restore(source, Some(safety));
3037 restore.metadata.annotations = Some(BTreeMap::from([(
3038 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3039 "commander@example.com".to_string(),
3040 )]));
3041 assert!(validate_safety_backup_config(&restore).is_err());
3042 }
3043
3044 #[test]
3045 fn break_glass_validation_rejects_skip_without_approver() {
3046 let source = KanidmRestoreSource {
3047 local: None,
3048 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3049 name: "backup-1".to_string(),
3050 }),
3051 };
3052 let safety = SafetyBackupConfig {
3053 repository_ref: Some(SafetyBackupRepositoryRef {
3054 name: "offsite".to_string(),
3055 }),
3056 skip: true,
3057 };
3058 let mut restore = make_restore(source, Some(safety));
3059 restore.metadata.annotations = Some(BTreeMap::from([(
3060 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3061 "PVC is unreadable".to_string(),
3062 )]));
3063 assert!(validate_safety_backup_config(&restore).is_err());
3064 }
3065
3066 #[test]
3067 fn break_glass_validation_accepts_valid_annotations() {
3068 let source = KanidmRestoreSource {
3069 local: None,
3070 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3071 name: "backup-1".to_string(),
3072 }),
3073 };
3074 let safety = SafetyBackupConfig {
3075 repository_ref: Some(SafetyBackupRepositoryRef {
3076 name: "offsite".to_string(),
3077 }),
3078 skip: true,
3079 };
3080 let mut restore = make_restore(source, Some(safety));
3081 restore.metadata.annotations = Some(BTreeMap::from([
3082 (
3083 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3084 "PVC is unreadable".to_string(),
3085 ),
3086 (
3087 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3088 "commander@example.com".to_string(),
3089 ),
3090 ]));
3091 assert!(validate_safety_backup_config(&restore).is_ok());
3092 }
3093
3094 #[test]
3095 fn remote_source_requires_repository_ref_when_not_skipping() {
3096 let source = KanidmRestoreSource {
3097 local: None,
3098 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3099 name: "backup-1".to_string(),
3100 }),
3101 };
3102 let safety = SafetyBackupConfig {
3103 repository_ref: None,
3104 skip: false,
3105 };
3106 let restore = make_restore(source, Some(safety));
3107 assert!(validate_safety_backup_config(&restore).is_err());
3108 }
3109
3110 #[test]
3111 fn remote_source_accepts_repository_ref() {
3112 let source = KanidmRestoreSource {
3113 local: None,
3114 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3115 name: "backup-1".to_string(),
3116 }),
3117 };
3118 let safety = SafetyBackupConfig {
3119 repository_ref: Some(SafetyBackupRepositoryRef {
3120 name: "offsite".to_string(),
3121 }),
3122 skip: false,
3123 };
3124 let restore = make_restore(source, Some(safety));
3125 assert!(validate_safety_backup_config(&restore).is_ok());
3126 }
3127
3128 #[test]
3129 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3130 let phases = vec![
3131 KanidmRestorePhase::Pending,
3132 KanidmRestorePhase::Validating,
3133 KanidmRestorePhase::Quiescing,
3134 KanidmRestorePhase::SafetyBackup,
3135 KanidmRestorePhase::PreparingSource,
3136 KanidmRestorePhase::RestoringPrimary,
3137 KanidmRestorePhase::Verifying,
3138 KanidmRestorePhase::RebuildingReplicas,
3139 KanidmRestorePhase::Resuming,
3140 KanidmRestorePhase::Completed,
3141 ];
3142 assert_eq!(phases.len(), 10);
3143 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3144 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3145 }
3146
3147 #[test]
3148 fn status_default_has_empty_phase_timestamps() {
3149 let status = KanidmRestoreStatus::default();
3150 assert!(status.phase_timestamps.is_empty());
3151 }
3152
3153 #[test]
3154 fn status_default_has_empty_original_replicas() {
3155 let status = KanidmRestoreStatus::default();
3156 assert!(status.original_replicas.is_empty());
3157 }
3158
3159 #[test]
3160 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3161 let mut target = super::super::crd::Kanidm::default();
3162 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3163 run_as_non_root: Some(false),
3164 run_as_user: Some(389),
3165 run_as_group: Some(389),
3166 fs_group: Some(389),
3167 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3168 supplemental_groups: Some(vec![390, 391]),
3169 ..Default::default()
3170 });
3171
3172 let ctx = super::source_job_pod_security_context(&target);
3173
3174 assert_eq!(ctx.run_as_non_root, Some(false));
3175 assert_eq!(ctx.run_as_user, Some(389));
3176 assert_eq!(ctx.run_as_group, Some(389));
3177 assert_eq!(ctx.fs_group, Some(389));
3178 assert_eq!(
3179 ctx.fs_group_change_policy.as_deref(),
3180 Some("OnRootMismatch")
3181 );
3182 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3183 assert_eq!(
3184 ctx.seccomp_profile
3185 .as_ref()
3186 .map(|profile| profile.type_.as_str()),
3187 Some("RuntimeDefault")
3188 );
3189 }
3190
3191 #[test]
3192 fn hardened_security_context_drops_all_capabilities() {
3193 let ctx = hardened_security_context();
3194 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3195 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3196 assert_eq!(ctx.run_as_non_root, Some(true));
3197 let caps = ctx.capabilities.unwrap();
3198 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3199 assert_eq!(
3200 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3201 Some("RuntimeDefault")
3202 );
3203 }
3204
3205 #[test]
3206 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3207 let ctx = kanidm_job_security_context();
3208 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3209 assert!(ctx.run_as_non_root.is_none());
3210 assert!(ctx.read_only_root_filesystem.is_none());
3211 let caps = ctx.capabilities.unwrap();
3212 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3213 assert_eq!(
3214 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3215 Some("RuntimeDefault")
3216 );
3217 }
3218
3219 #[test]
3220 fn safety_backup_job_name_is_deterministic() {
3221 let source = KanidmRestoreSource {
3222 local: Some(super::KanidmRestoreLocalSource {
3223 file_name: "backup.json".to_string(),
3224 }),
3225 backup_ref: None,
3226 };
3227 let restore = make_restore(source, None);
3228 let name1 = super::safety_backup_job_name(&restore);
3229 let name2 = super::safety_backup_job_name(&restore);
3230 assert_eq!(name1, name2);
3231 assert_eq!(name1, "test-restore-safety-backup");
3232 }
3233
3234 #[test]
3235 fn source_prep_job_name_is_deterministic() {
3236 let source = KanidmRestoreSource {
3237 local: None,
3238 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3239 name: "backup-1".to_string(),
3240 }),
3241 };
3242 let restore = make_restore(source, None);
3243 let name = super::source_prep_job_name(&restore);
3244 assert_eq!(name, "test-restore-source-prep");
3245 }
3246
3247 #[test]
3248 fn break_glass_condition_is_recorded_in_status() {
3249 let mut status = KanidmRestoreStatus::default();
3250 let condition = super::restore_condition(
3251 &status.conditions,
3252 CONDITION_BREAK_GLASS,
3253 CONDITION_TRUE,
3254 "BreakGlassOverride",
3255 "Safety backup skipped",
3256 Some(1),
3257 );
3258 status.conditions.push(condition);
3259 assert!(status.conditions.iter().any(|c| {
3260 c.type_ == CONDITION_BREAK_GLASS
3261 && c.status == CONDITION_TRUE
3262 && c.reason == "BreakGlassOverride"
3263 }));
3264 }
3265
3266 #[test]
3267 fn replica_count_entry_serialization() {
3268 let entry = ReplicaCountEntry {
3269 group: "primary".to_string(),
3270 replicas: 3,
3271 };
3272 let json = serde_json::to_string(&entry).unwrap();
3273 assert!(json.contains("\"group\":\"primary\""));
3274 assert!(json.contains("\"replicas\":3"));
3275 }
3276
3277 #[test]
3278 fn local_source_does_not_require_repository_ref() {
3279 let source = KanidmRestoreSource {
3280 local: Some(super::KanidmRestoreLocalSource {
3281 file_name: "backup.json".to_string(),
3282 }),
3283 backup_ref: None,
3284 };
3285 let safety = SafetyBackupConfig {
3286 repository_ref: None,
3287 skip: false,
3288 };
3289 let restore = make_restore(source, Some(safety));
3290 assert!(validate_safety_backup_config(&restore).is_ok());
3291 }
3292
3293 #[test]
3294 fn skip_without_break_glass_annotations_is_rejected() {
3295 let source = KanidmRestoreSource {
3296 local: Some(super::KanidmRestoreLocalSource {
3297 file_name: "backup.json".to_string(),
3298 }),
3299 backup_ref: None,
3300 };
3301 let safety = SafetyBackupConfig {
3302 repository_ref: None,
3303 skip: true,
3304 };
3305 let restore = make_restore(source, Some(safety));
3306 assert!(validate_safety_backup_config(&restore).is_err());
3307 }
3308
3309 #[test]
3310 fn has_accepted_condition_true() {
3311 let conditions = vec![Condition {
3312 type_: "Ready".to_string(),
3313 status: "True".to_string(),
3314 reason: "Accepted".to_string(),
3315 message: String::new(),
3316 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3317 k8s_openapi::jiff::Timestamp::now(),
3318 ),
3319 observed_generation: None,
3320 }];
3321 assert!(has_accepted_condition(&conditions));
3322 }
3323
3324 #[test]
3325 fn has_accepted_condition_false_when_not_accepted_reason() {
3326 let conditions = vec![Condition {
3327 type_: "Ready".to_string(),
3328 status: "True".to_string(),
3329 reason: "SomeOtherReason".to_string(),
3330 message: String::new(),
3331 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3332 k8s_openapi::jiff::Timestamp::now(),
3333 ),
3334 observed_generation: None,
3335 }];
3336 assert!(!has_accepted_condition(&conditions));
3337 }
3338
3339 #[test]
3340 fn has_accepted_condition_false_when_status_false() {
3341 let conditions = vec![Condition {
3342 type_: "Ready".to_string(),
3343 status: "False".to_string(),
3344 reason: "Accepted".to_string(),
3345 message: String::new(),
3346 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3347 k8s_openapi::jiff::Timestamp::now(),
3348 ),
3349 observed_generation: None,
3350 }];
3351 assert!(!has_accepted_condition(&conditions));
3352 }
3353
3354 #[test]
3355 fn has_accepted_condition_empty() {
3356 assert!(!has_accepted_condition(&[]));
3357 }
3358
3359 #[test]
3360 fn safety_backup_upload_operation_doc_is_valid() {
3361 let source = KanidmRestoreSource {
3362 local: None,
3363 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3364 name: "backup-1".to_string(),
3365 }),
3366 };
3367 let mut restore = make_restore(source, None);
3368 restore.metadata.namespace = Some("test-ns".to_string());
3369 let target = super::super::crd::Kanidm::default();
3370 let doc_str = super::build_safety_upload_operation_doc(
3371 &restore,
3372 &target,
3373 "test-backup-id",
3374 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3375 "my-bucket",
3376 "prod",
3377 "https://s3.example.com",
3378 "us-east-1",
3379 false,
3380 false,
3381 None,
3382 None,
3383 )
3384 .unwrap();
3385 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3386 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3387 assert_eq!(parsed["kind"], "OperationDocument");
3388 assert_eq!(parsed["operation"], "upload");
3389 assert_eq!(parsed["bucket"], "my-bucket");
3390 assert_eq!(parsed["prefix"], "prod");
3391 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3392 assert_eq!(parsed["region"], "us-east-1");
3393 assert_eq!(parsed["backupId"], "test-backup-id");
3394 assert_eq!(parsed["consistency"], "kanidm-offline");
3395 assert_eq!(parsed["reason"], "restore-safety");
3396 assert_eq!(
3397 parsed["payloadPath"],
3398 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3399 );
3400 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3401 }
3402
3403 #[test]
3404 fn download_operation_doc_uses_real_repository_data() {
3405 let source = KanidmRestoreSource {
3406 local: None,
3407 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3408 name: "backup-1".to_string(),
3409 }),
3410 };
3411 let restore = make_restore(source, None);
3412 let target = super::super::crd::Kanidm::default();
3413 let doc_str = super::build_download_operation_doc(
3414 &restore,
3415 &target,
3416 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3417 "019c7c76-f423-7a12-8f41-2bea7588a303",
3418 "real-bucket",
3419 "real-prefix",
3420 "https://real-endpoint.com",
3421 "eu-west-1",
3422 true,
3423 false,
3424 None,
3425 None,
3426 );
3427 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3428 assert_eq!(parsed["bucket"], "real-bucket");
3429 assert_eq!(parsed["prefix"], "real-prefix");
3430 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3431 assert_eq!(parsed["region"], "eu-west-1");
3432 assert_eq!(parsed["forcePathStyle"], true);
3433 assert_eq!(
3434 parsed["manifestKey"],
3435 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3436 );
3437 assert_eq!(
3438 parsed["expectedBackupId"],
3439 "019c7c76-f423-7a12-8f41-2bea7588a303"
3440 );
3441 }
3442
3443 #[test]
3444 fn safety_backup_ref_is_set_after_verification() {
3445 let status = KanidmRestoreStatus {
3446 phase: KanidmRestorePhase::SafetyBackup,
3447 safety_backup_job_name: Some("test-safety-backup".to_string()),
3448 safety_backup_ref: Some("safety-test-restore".to_string()),
3449 safety_backup_expected_backup_id: Some(
3450 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3451 ),
3452 safety_backup_manifest_key: Some(
3453 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3454 ),
3455 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3456 ..Default::default()
3457 };
3458 assert!(status.safety_backup_ref.is_some());
3459 assert!(
3460 status
3461 .safety_backup_ref
3462 .as_ref()
3463 .unwrap()
3464 .starts_with("safety-")
3465 );
3466 assert!(status.safety_backup_expected_backup_id.is_some());
3467 assert!(status.safety_backup_manifest_key.is_some());
3468 assert!(status.safety_backup_payload_sha256.is_some());
3469 }
3470
3471 #[test]
3472 fn database_mutation_not_started_without_safety_backup_ref() {
3473 let status = KanidmRestoreStatus {
3474 phase: KanidmRestorePhase::PreparingSource,
3475 safety_backup_ref: None,
3476 database_mutation_started: false,
3477 ..Default::default()
3478 };
3479 assert!(!status.database_mutation_started);
3480 assert!(status.safety_backup_ref.is_none());
3481 }
3482
3483 #[test]
3484 fn pre_boundary_failure_does_not_set_mutation_started() {
3485 let status = KanidmRestoreStatus {
3486 phase: KanidmRestorePhase::Failed,
3487 message: Some(
3488 "safety backup job failed; target restored to original state".to_string(),
3489 ),
3490 database_mutation_started: false,
3491 ..Default::default()
3492 };
3493 assert!(!status.database_mutation_started);
3494 }
3495
3496 #[test]
3497 fn shared_volume_constants_are_distinct() {
3498 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3499 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3500 }
3501
3502 #[test]
3503 fn validate_backup_compatibility_matching_versions_passes() {
3504 let source = KanidmRestoreSource {
3505 local: None,
3506 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3507 name: "backup-1".to_string(),
3508 }),
3509 };
3510 let restore = make_restore(source, None);
3511 let target = kanidm_with_version("1.10.4");
3512 let backup = kaniop_backup_core::crd::KanidmBackup {
3513 metadata: kube::api::ObjectMeta::default(),
3514 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3515 backup_id: "id".to_string(),
3516 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3517 name: "k".to_string(),
3518 uid: "uid".to_string(),
3519 },
3520 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3521 name: "repo".to_string(),
3522 },
3523 manifest_key: "key".to_string(),
3524 },
3525 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3526 kanidm_version: Some("1.10.4".to_string()),
3527 ..Default::default()
3528 }),
3529 };
3530 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3531 }
3532
3533 #[test]
3534 fn validate_backup_compatibility_version_mismatch_fails() {
3535 let source = KanidmRestoreSource {
3536 local: None,
3537 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3538 name: "backup-1".to_string(),
3539 }),
3540 };
3541 let restore = make_restore(source, None);
3542 let target = kanidm_with_version("1.11.0");
3543 let backup = kaniop_backup_core::crd::KanidmBackup {
3544 metadata: kube::api::ObjectMeta::default(),
3545 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3546 backup_id: "id".to_string(),
3547 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3548 name: "k".to_string(),
3549 uid: "uid".to_string(),
3550 },
3551 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3552 name: "repo".to_string(),
3553 },
3554 manifest_key: "key".to_string(),
3555 },
3556 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3557 kanidm_version: Some("1.10.4".to_string()),
3558 ..Default::default()
3559 }),
3560 };
3561 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3562 }
3563
3564 #[test]
3565 fn validate_backup_compatibility_no_backup_version_passes() {
3566 let source = KanidmRestoreSource {
3567 local: None,
3568 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3569 name: "backup-1".to_string(),
3570 }),
3571 };
3572 let restore = make_restore(source, None);
3573 let target = super::super::crd::Kanidm::default();
3574 let backup = kaniop_backup_core::crd::KanidmBackup {
3575 metadata: kube::api::ObjectMeta::default(),
3576 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3577 backup_id: "id".to_string(),
3578 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3579 name: "k".to_string(),
3580 uid: "uid".to_string(),
3581 },
3582 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3583 name: "repo".to_string(),
3584 },
3585 manifest_key: "key".to_string(),
3586 },
3587 status: None,
3588 };
3589 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3590 }
3591
3592 #[test]
3593 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3594 let source = KanidmRestoreSource {
3595 local: None,
3596 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3597 name: "backup-1".to_string(),
3598 }),
3599 };
3600 let mut restore = make_restore(source, None);
3601 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3602 let target = super::super::crd::Kanidm::default();
3603 let backup = kaniop_backup_core::crd::KanidmBackup {
3604 metadata: kube::api::ObjectMeta::default(),
3605 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3606 backup_id: "id".to_string(),
3607 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3608 name: "k".to_string(),
3609 uid: "uid".to_string(),
3610 },
3611 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3612 name: "repo".to_string(),
3613 },
3614 manifest_key: "key".to_string(),
3615 },
3616 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3617 image_digest: Some("sha256:originaldigest".to_string()),
3618 ..Default::default()
3619 }),
3620 };
3621 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3622 }
3623
3624 #[test]
3625 fn validate_backup_compatibility_image_digest_match_passes() {
3626 let source = KanidmRestoreSource {
3627 local: None,
3628 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3629 name: "backup-1".to_string(),
3630 }),
3631 };
3632 let mut restore = make_restore(source, None);
3633 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3634 let target = super::super::crd::Kanidm::default();
3635 let backup = kaniop_backup_core::crd::KanidmBackup {
3636 metadata: kube::api::ObjectMeta::default(),
3637 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3638 backup_id: "id".to_string(),
3639 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3640 name: "k".to_string(),
3641 uid: "uid".to_string(),
3642 },
3643 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3644 name: "repo".to_string(),
3645 },
3646 manifest_key: "key".to_string(),
3647 },
3648 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3649 image_digest: Some("sha256:abc123".to_string()),
3650 ..Default::default()
3651 }),
3652 };
3653 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3654 }
3655
3656 #[test]
3657 fn safety_backup_id_is_deterministic() {
3658 let source = KanidmRestoreSource {
3659 local: None,
3660 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3661 name: "backup-1".to_string(),
3662 }),
3663 };
3664 let restore = make_restore(source, None);
3665 let id1 = super::compute_safety_backup_id(&restore);
3666 let id2 = super::compute_safety_backup_id(&restore);
3667 assert_eq!(id1, id2);
3668 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3669 }
3670
3671 #[test]
3672 fn safety_backup_id_differs_for_different_restores() {
3673 let source1 = KanidmRestoreSource {
3674 local: None,
3675 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3676 name: "backup-1".to_string(),
3677 }),
3678 };
3679 let restore1 = make_restore(source1, None);
3680 let mut restore2 = make_restore(
3681 KanidmRestoreSource {
3682 local: None,
3683 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3684 name: "backup-2".to_string(),
3685 }),
3686 },
3687 None,
3688 );
3689 restore2.metadata.name = Some("different-restore".to_string());
3690 let id1 = super::compute_safety_backup_id(&restore1);
3691 let id2 = super::compute_safety_backup_id(&restore2);
3692 assert_ne!(id1, id2);
3693 }
3694
3695 #[test]
3696 fn safety_manifest_key_is_deterministic() {
3697 let source = KanidmRestoreSource {
3698 local: None,
3699 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3700 name: "backup-1".to_string(),
3701 }),
3702 };
3703 let restore = make_restore(source, None);
3704 let key1 = super::compute_safety_manifest_key(&restore);
3705 let key2 = super::compute_safety_manifest_key(&restore);
3706 assert_eq!(key1, key2);
3707 assert!(key1.contains("manifest.json"));
3708 assert!(key1.starts_with("v1/tenants/"));
3709 }
3710
3711 #[test]
3712 fn safety_backup_id_survives_restart() {
3713 let source = KanidmRestoreSource {
3714 local: None,
3715 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3716 name: "backup-1".to_string(),
3717 }),
3718 };
3719 let mut restore = make_restore(source, None);
3720 let original_id = super::compute_safety_backup_id(&restore);
3721 restore.status = Some(KanidmRestoreStatus {
3722 phase: KanidmRestorePhase::SafetyBackup,
3723 safety_backup_expected_backup_id: Some(original_id.clone()),
3724 ..Default::default()
3725 });
3726 let recomputed_id = super::compute_safety_backup_id(&restore);
3727 assert_eq!(original_id, recomputed_id);
3728 }
3729
3730 #[test]
3731 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3732 let source = KanidmRestoreSource {
3733 local: None,
3734 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3735 name: "backup-1".to_string(),
3736 }),
3737 };
3738 let restore = make_restore(source, None);
3739 assert!(super::requires_safety_backup(&restore));
3740 let status_without_ref = KanidmRestoreStatus {
3741 phase: KanidmRestorePhase::RestoringPrimary,
3742 safety_backup_ref: None,
3743 database_mutation_started: false,
3744 ..Default::default()
3745 };
3746 assert!(status_without_ref.safety_backup_ref.is_none());
3747 assert!(!status_without_ref.database_mutation_started);
3748 }
3749
3750 #[test]
3751 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3752 let status_with_ref = KanidmRestoreStatus {
3753 phase: KanidmRestorePhase::RestoringPrimary,
3754 safety_backup_ref: Some("safety-test-restore".to_string()),
3755 safety_backup_manifest_key: Some(
3756 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3757 ),
3758 safety_backup_payload_sha256: Some("abc123".to_string()),
3759 database_mutation_started: false,
3760 ..Default::default()
3761 };
3762 assert!(status_with_ref.safety_backup_ref.is_some());
3763 assert!(!status_with_ref.database_mutation_started);
3764 }
3765
3766 #[test]
3767 fn result_document_verification_rejects_missing_backup_id() {
3768 let result_json = r#"{
3769 "apiVersion": "backup.kaniop.rs/v1alpha1",
3770 "kind": "ResultDocument",
3771 "operation": "upload",
3772 "success": true,
3773 "exitCode": "success",
3774 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3775 "payloadSha256": "abc123"
3776 }"#;
3777 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3778 assert!(result.backup_id.is_none());
3779 }
3780
3781 #[test]
3782 fn result_document_verification_rejects_missing_manifest_key() {
3783 let result_json = r#"{
3784 "apiVersion": "backup.kaniop.rs/v1alpha1",
3785 "kind": "ResultDocument",
3786 "operation": "upload",
3787 "success": true,
3788 "exitCode": "success",
3789 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3790 "payloadSha256": "abc123"
3791 }"#;
3792 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3793 assert!(result.manifest_key.is_none());
3794 }
3795
3796 #[test]
3797 fn result_document_verification_rejects_missing_payload_sha256() {
3798 let result_json = r#"{
3799 "apiVersion": "backup.kaniop.rs/v1alpha1",
3800 "kind": "ResultDocument",
3801 "operation": "upload",
3802 "success": true,
3803 "exitCode": "success",
3804 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3805 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3806 }"#;
3807 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3808 assert!(result.payload_sha256.is_none());
3809 }
3810
3811 #[test]
3812 fn result_document_verification_rejects_failure_exit_code() {
3813 let result_json = r#"{
3814 "apiVersion": "backup.kaniop.rs/v1alpha1",
3815 "kind": "ResultDocument",
3816 "operation": "upload",
3817 "success": false,
3818 "exitCode": "integrity",
3819 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3820 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3821 "payloadSha256": "abc123"
3822 }"#;
3823 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3824 assert!(!result.success);
3825 assert_eq!(
3826 result.exit_code,
3827 kaniop_backup_core::result::ExitCode::Integrity
3828 );
3829 }
3830
3831 #[test]
3832 fn result_document_verification_rejects_wrong_operation() {
3833 let result_json = r#"{
3834 "apiVersion": "backup.kaniop.rs/v1alpha1",
3835 "kind": "ResultDocument",
3836 "operation": "download",
3837 "success": true,
3838 "exitCode": "success",
3839 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3840 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3841 "payloadSha256": "abc123"
3842 }"#;
3843 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3844 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3845 }
3846
3847 #[test]
3848 fn result_document_verification_accepts_valid_document() {
3849 let result_json = r#"{
3850 "apiVersion": "backup.kaniop.rs/v1alpha1",
3851 "kind": "ResultDocument",
3852 "operation": "upload",
3853 "success": true,
3854 "exitCode": "success",
3855 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3856 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3857 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3858 }"#;
3859 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3860 assert!(result.success);
3861 assert_eq!(
3862 result.exit_code,
3863 kaniop_backup_core::result::ExitCode::Success
3864 );
3865 assert_eq!(result.operation, "upload");
3866 assert!(result.backup_id.is_some());
3867 assert!(result.manifest_key.is_some());
3868 assert!(result.payload_sha256.is_some());
3869 }
3870
3871 #[test]
3872 fn termination_message_path_constant_is_valid() {
3873 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3874 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3875 }
3876
3877 #[test]
3878 fn status_tracks_safety_backup_evidence() {
3879 let status = KanidmRestoreStatus {
3880 phase: KanidmRestorePhase::PreparingSource,
3881 safety_backup_ref: Some("safety-test-restore".to_string()),
3882 safety_backup_expected_backup_id: Some(
3883 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3884 ),
3885 safety_backup_manifest_key: Some(
3886 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3887 ),
3888 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3889 ..Default::default()
3890 };
3891 assert!(status.safety_backup_ref.is_some());
3892 assert!(status.safety_backup_expected_backup_id.is_some());
3893 assert!(status.safety_backup_manifest_key.is_some());
3894 assert!(status.safety_backup_payload_sha256.is_some());
3895 }
3896
3897 #[test]
3898 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3899 let invalid_json = "not valid json";
3900 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3901 assert!(result.is_err());
3902 }
3903
3904 #[test]
3905 fn fail_closed_semantics_preserved_on_oversized_result() {
3906 let oversized_json = format!(
3907 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3908 kaniop_backup_core::result::RESULT_DOC_VERSION,
3909 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3910 );
3911 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3912 assert!(result.is_err());
3913 }
3914
3915 #[test]
3916 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3917 let source = KanidmRestoreSource {
3918 local: None,
3919 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3920 name: "backup-1".to_string(),
3921 }),
3922 };
3923 let mut restore = make_restore(source, None);
3924 restore.metadata.namespace = Some("test-ns".to_string());
3925 let target = super::super::crd::Kanidm::default();
3926 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3927 let doc_str = super::build_safety_upload_operation_doc(
3928 &restore,
3929 &target,
3930 "test-backup-id",
3931 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3932 "my-bucket",
3933 "prod",
3934 "https://s3.example.com",
3935 "us-east-1",
3936 false,
3937 false,
3938 Some(&ca_path),
3939 None,
3940 )
3941 .unwrap();
3942 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3943 assert_eq!(parsed["caBundlePath"], ca_path);
3944 }
3945
3946 #[test]
3947 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3948 let source = KanidmRestoreSource {
3949 local: None,
3950 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3951 name: "backup-1".to_string(),
3952 }),
3953 };
3954 let mut restore = make_restore(source, None);
3955 restore.metadata.namespace = Some("test-ns".to_string());
3956 let target = super::super::crd::Kanidm::default();
3957 let doc_str = super::build_safety_upload_operation_doc(
3958 &restore,
3959 &target,
3960 "test-backup-id",
3961 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3962 "my-bucket",
3963 "prod",
3964 "https://s3.example.com",
3965 "us-east-1",
3966 false,
3967 false,
3968 None,
3969 None,
3970 )
3971 .unwrap();
3972 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3973 assert!(parsed["caBundlePath"].is_null());
3974 }
3975
3976 #[test]
3977 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3978 let source = KanidmRestoreSource {
3979 local: None,
3980 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3981 name: "backup-1".to_string(),
3982 }),
3983 };
3984 let restore = make_restore(source, None);
3985 let target = super::super::crd::Kanidm::default();
3986 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3987 let doc_str = super::build_download_operation_doc(
3988 &restore,
3989 &target,
3990 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3991 "019c7c76-f423-7a12-8f41-2bea7588a303",
3992 "real-bucket",
3993 "real-prefix",
3994 "https://real-endpoint.com",
3995 "eu-west-1",
3996 true,
3997 false,
3998 Some(&ca_path),
3999 None,
4000 );
4001 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4002 assert_eq!(parsed["caBundlePath"], ca_path);
4003 }
4004
4005 #[test]
4006 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4007 let source = KanidmRestoreSource {
4008 local: None,
4009 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4010 name: "backup-1".to_string(),
4011 }),
4012 };
4013 let restore = make_restore(source, None);
4014 let target = super::super::crd::Kanidm::default();
4015 let doc_str = super::build_download_operation_doc(
4016 &restore,
4017 &target,
4018 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4019 "019c7c76-f423-7a12-8f41-2bea7588a303",
4020 "real-bucket",
4021 "real-prefix",
4022 "https://real-endpoint.com",
4023 "eu-west-1",
4024 true,
4025 false,
4026 None,
4027 None,
4028 );
4029 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4030 assert!(parsed["caBundlePath"].is_null());
4031 }
4032
4033 #[test]
4034 fn source_prep_result_operation_is_download() {
4035 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4036 assert_ne!(
4037 super::SOURCE_PREP_RESULT_OPERATION,
4038 super::SAFETY_BACKUP_RESULT_OPERATION
4039 );
4040 }
4041
4042 #[test]
4043 fn download_result_document_verification_accepts_valid_document() {
4044 let result_json = r#"{
4045 "apiVersion": "backup.kaniop.rs/v1alpha1",
4046 "kind": "ResultDocument",
4047 "operation": "download",
4048 "success": true,
4049 "exitCode": "success",
4050 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4051 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4052 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4053 }"#;
4054 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4055 assert!(result.success);
4056 assert_eq!(
4057 result.exit_code,
4058 kaniop_backup_core::result::ExitCode::Success
4059 );
4060 assert_eq!(result.operation, "download");
4061 assert_eq!(
4062 result.backup_id.as_deref(),
4063 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4064 );
4065 assert!(result.manifest_key.is_some());
4066 assert!(result.payload_sha256.is_some());
4067 }
4068
4069 #[test]
4070 fn download_result_document_rejects_wrong_operation() {
4071 let result_json = r#"{
4072 "apiVersion": "backup.kaniop.rs/v1alpha1",
4073 "kind": "ResultDocument",
4074 "operation": "upload",
4075 "success": true,
4076 "exitCode": "success",
4077 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4078 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4079 "payloadSha256": "abc123"
4080 }"#;
4081 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4082 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4083 }
4084
4085 #[test]
4086 fn download_result_document_rejects_failure_success_field() {
4087 let result_json = r#"{
4088 "apiVersion": "backup.kaniop.rs/v1alpha1",
4089 "kind": "ResultDocument",
4090 "operation": "download",
4091 "success": false,
4092 "exitCode": "retryable",
4093 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4094 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4095 "payloadSha256": "abc123"
4096 }"#;
4097 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4098 assert!(!result.success);
4099 assert_ne!(
4100 result.exit_code,
4101 kaniop_backup_core::result::ExitCode::Success
4102 );
4103 }
4104
4105 #[test]
4106 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4107 let status = KanidmRestoreStatus {
4108 phase: KanidmRestorePhase::PreparingSource,
4109 database_mutation_started: false,
4110 ..Default::default()
4111 };
4112 assert!(!status.database_mutation_started);
4113 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4114 }
4115
4116 #[test]
4117 fn backup_job_volume_size_default_is_10gi() {
4118 let qty = crate::controller::backup_job_volume_size();
4119 assert_eq!(qty, Quantity("10Gi".to_string()));
4120 }
4121
4122 #[test]
4123 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4124 use kaniop_backup_core::auth::build_encryption_env_vars;
4125 use kaniop_backup_core::crd::SecretRef;
4126
4127 let key_ref = SecretRef {
4128 name: "kek-secret".to_string(),
4129 };
4130 let env_vars = build_encryption_env_vars(Some(&key_ref));
4131 assert_eq!(env_vars.len(), 1);
4132 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4133 let secret_ref = env_vars[0]
4134 .value_from
4135 .as_ref()
4136 .unwrap()
4137 .secret_key_ref
4138 .as_ref()
4139 .unwrap();
4140 assert_eq!(secret_ref.name, "kek-secret");
4141 assert_eq!(secret_ref.key, "encryption-key");
4142 }
4143
4144 #[test]
4145 fn encryption_env_vars_absent_when_no_key_ref() {
4146 use kaniop_backup_core::auth::build_encryption_env_vars;
4147
4148 let env_vars = build_encryption_env_vars(None);
4149 assert!(
4150 env_vars.is_empty(),
4151 "encryption env vars must be empty when key_ref is None"
4152 );
4153 }
4154
4155 #[test]
4156 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4157 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4158
4159 let source = KanidmRestoreSource {
4160 local: None,
4161 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4162 name: "backup-1".to_string(),
4163 }),
4164 };
4165 let mut restore = make_restore(source, None);
4166 restore.metadata.namespace = Some("test-ns".to_string());
4167 let target = super::super::crd::Kanidm::default();
4168 let encryption = RepositoryEncryption {
4169 mode: EncryptionMode::ClientSide,
4170 key_id: None,
4171 key_ref: Some(SecretRef {
4172 name: "kek-secret".to_string(),
4173 }),
4174 };
4175 let doc_str = super::build_safety_upload_operation_doc(
4176 &restore,
4177 &target,
4178 "test-backup-id",
4179 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4180 "my-bucket",
4181 "prod",
4182 "https://s3.example.com",
4183 "us-east-1",
4184 false,
4185 false,
4186 None,
4187 Some(&encryption),
4188 )
4189 .unwrap();
4190 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4191 assert_eq!(parsed["encryptionMode"], "clientSide");
4192 assert!(parsed["encryptionKeyId"].is_null());
4193 }
4194
4195 #[test]
4196 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4197 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4198
4199 let source = KanidmRestoreSource {
4200 local: None,
4201 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4202 name: "backup-1".to_string(),
4203 }),
4204 };
4205 let restore = make_restore(source, None);
4206 let target = super::super::crd::Kanidm::default();
4207 let encryption = RepositoryEncryption {
4208 mode: EncryptionMode::ProviderKms,
4209 key_id: Some("alias/kaniop-backups".to_string()),
4210 key_ref: None,
4211 };
4212 let doc_str = super::build_download_operation_doc(
4213 &restore,
4214 &target,
4215 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4216 "019c7c76-f423-7a12-8f41-2bea7588a303",
4217 "real-bucket",
4218 "real-prefix",
4219 "https://real-endpoint.com",
4220 "eu-west-1",
4221 true,
4222 false,
4223 None,
4224 Some(&encryption),
4225 );
4226 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4227 assert_eq!(parsed["encryptionMode"], "providerKms");
4228 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4229 }
4230}