1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const STAGING_VOLUME: &str = "restore-staging";
54const SHARED_VOLUME: &str = "safety-backup-shared";
55const DATA_PATH: &str = "/data";
56const TLS_VOLUME: &str = "kanidm-certs";
57const TLS_PATH: &str = "/etc/kanidm/tls";
58const BACKUP_PATH: &str = "/data";
59const STAGING_PATH: &str = "/staging";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78 feature = "schemars",
79 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82 category = "kaniop",
83 group = "kaniop.rs",
84 version = "v1beta1",
85 kind = "KanidmRestore",
86 plural = "kanidmrestores",
87 singular = "kanidmrestore",
88 shortname = "idmrestore",
89 namespaced,
90 status = "KanidmRestoreStatus",
91 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97 pub target_ref: KanidmRestoreTargetRef,
98 pub source: KanidmRestoreSource,
99 pub restore_image: String,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108 pub name: String,
109 pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115 feature = "schemars",
116 schemars(extend("x-kubernetes-validations" = [
117 {
118 "message": "exactly one of local or backupRef must be set",
119 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120 }
121 ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125 #[serde(skip_serializing_if = "Option::is_none")]
127 pub local: Option<KanidmRestoreLocalSource>,
128 #[serde(skip_serializing_if = "Option::is_none")]
130 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137 pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145 pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153 #[serde(skip_serializing_if = "Option::is_none")]
155 pub repository_ref: Option<SafetyBackupRepositoryRef>,
156 #[serde(default)]
158 pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165 pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171 #[default]
172 Pending,
173 Validating,
174 Quiescing,
175 SafetyBackup,
176 PreparingSource,
177 RestoringPrimary,
178 Verifying,
179 RebuildingReplicas,
180 Resuming,
181 Completed,
182 Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_generation: Option<i64>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_target_uid: Option<String>,
193 #[serde(default)]
194 pub phase: KanidmRestorePhase,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub restore_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub verify_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub safety_backup_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub source_prep_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_ref: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_expected_backup_id: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_manifest_key: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_payload_sha256: Option<String>,
211 #[serde(default)]
212 pub replicas_cleared: bool,
213 #[serde(default)]
214 pub certificates_cleared: bool,
215 #[serde(default)]
216 pub database_mutation_started: bool,
217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
218 pub original_replicas: Vec<ReplicaCountEntry>,
219 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220 pub phase_timestamps: std::collections::BTreeMap<String, String>,
221 #[serde(default, skip_serializing_if = "Vec::is_empty")]
222 #[cfg_attr(
223 feature = "schemars",
224 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225 )]
226 pub conditions: Vec<Condition>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235 pub group: String,
236 pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241 attempts: Counter<u64>,
242 outcomes: Counter<u64>,
243 duration_seconds: Histogram<f64>,
244 break_glass_total: Counter<u64>,
245 safety_backup_duration_seconds: Histogram<f64>,
246}
247
248impl RestoreMetrics {
249 fn new() -> Self {
250 let meter = global::meter("kaniop");
251 Self {
252 attempts: meter
253 .u64_counter("kaniop_restore_attempts_total")
254 .with_description("Number of Kanidm restore attempts started")
255 .build(),
256 outcomes: meter
257 .u64_counter("kaniop_restore_outcomes_total")
258 .with_description("Number of terminal Kanidm restore outcomes")
259 .build(),
260 duration_seconds: meter
261 .f64_histogram("kaniop_restore_duration_seconds")
262 .with_description("Kanidm restore duration from object creation to terminal phase")
263 .with_unit("s")
264 .build(),
265 break_glass_total: meter
266 .u64_counter("kaniop_restore_break_glass_total")
267 .with_description("Number of break-glass safety backup overrides")
268 .build(),
269 safety_backup_duration_seconds: meter
270 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
271 .with_description("Duration of pre-restore safety backup creation")
272 .with_unit("s")
273 .build(),
274 }
275 }
276}
277
278#[derive(Clone)]
279struct RestoreContext {
280 client: Client,
281 recorder: Recorder,
282 metrics: RestoreMetrics,
283}
284
285pub async fn run(client: Client) {
286 let api = Api::<KanidmRestore>::all(client.clone());
287 let recorder = Recorder::new(
288 client.clone(),
289 Reporter {
290 controller: CONTROLLER_ID.into(),
291 instance: None,
292 },
293 );
294 let ctx = Arc::new(RestoreContext {
295 client,
296 recorder,
297 metrics: RestoreMetrics::new(),
298 });
299 info!("starting {CONTROLLER_ID} controller");
300 Controller::new(api, watcher::Config::default().any_semantic())
301 .shutdown_on_signal()
302 .run(reconcile_restore, error_policy, ctx)
303 .for_each(|result| async move {
304 if let Err(error) = result {
305 error!(%error, "KanidmRestore reconciliation failed");
306 }
307 })
308 .await;
309}
310
311fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
312 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
313 Action::requeue(Duration::from_secs(5))
314}
315
316async fn reconcile_restore(
317 restore: Arc<KanidmRestore>,
318 ctx: Arc<RestoreContext>,
319) -> Result<Action> {
320 let namespace = restore
321 .namespace()
322 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
323 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
324 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
325 let ctx = ctx.clone();
326 async move {
327 match event {
328 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
329 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
330 }
331 }
332 })
333 .await
334 .map_err(|error| {
335 Error::FinalizerError(
336 "failed on KanidmRestore finalizer".to_string(),
337 Box::new(error),
338 )
339 })
340}
341
342async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
343 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
344 match phase {
345 KanidmRestorePhase::Pending => {
346 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
347 Ok(Action::requeue(REQUEUE))
348 }
349 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
350 Ok(target) => {
351 mark_restoring(&restore, &target, &ctx).await?;
352 let mut status = restore.status.clone().unwrap_or_default();
353 status.observed_target_uid = target.uid();
354 status.original_replicas = capture_original_replicas(&target);
355 status.phase = KanidmRestorePhase::Quiescing;
356 status.message = None;
357 patch_status(&restore, &ctx, status).await?;
358 Ok(Action::requeue(REQUEUE))
359 }
360 Err(error) => {
361 set_phase(
362 &restore,
363 &ctx,
364 KanidmRestorePhase::Failed,
365 Some(error.to_string()),
366 )
367 .await?;
368 Ok(Action::requeue(Duration::from_secs(300)))
369 }
370 },
371 KanidmRestorePhase::Quiescing => {
372 let target = get_target(&restore, &ctx).await?;
373 scale_all(&target, &ctx, 0).await?;
374 if target_pods_stopped(&target, &ctx).await?
375 && target_volumes_detached(&target, &ctx).await?
376 {
377 let next_phase = if requires_safety_backup(&restore) {
378 KanidmRestorePhase::SafetyBackup
379 } else {
380 record_break_glass(&restore, &ctx).await?;
381 KanidmRestorePhase::PreparingSource
382 };
383 if next_phase == KanidmRestorePhase::PreparingSource {
384 let refreshed = Api::<KanidmRestore>::namespaced(
385 ctx.client.clone(),
386 restore.namespace().as_deref().unwrap_or_default(),
387 )
388 .get(&restore.name_any())
389 .await
390 .map_err(|error| {
391 Error::kube_error(
392 "get",
393 "KanidmRestore",
394 restore.namespace().as_deref().unwrap_or_default(),
395 restore.name_any(),
396 error,
397 )
398 })?;
399 set_phase(&refreshed, &ctx, next_phase, None).await?;
400 } else {
401 set_phase(&restore, &ctx, next_phase, None).await?;
402 }
403 }
404 Ok(Action::requeue(REQUEUE))
405 }
406 KanidmRestorePhase::SafetyBackup => {
407 let target = get_target(&restore, &ctx).await?;
408 let name = safety_backup_job_name(&restore);
409 let expected_backup_id = compute_safety_backup_id(&restore);
410 let expected_manifest_key = compute_safety_manifest_key(&restore);
411 ensure_safety_backup_job(
412 &restore,
413 &target,
414 &ctx,
415 &name,
416 &expected_backup_id,
417 &expected_manifest_key,
418 )
419 .await?;
420 let mut status = restore.status.clone().unwrap_or_default();
421 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
422 match job_state(&restore, &ctx, &name).await? {
423 JobState::Complete => {
424 let result =
425 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
426 match result {
427 Ok(verified) => {
428 status.safety_backup_job_name = Some(name);
429 status.safety_backup_ref =
430 Some(format!("safety-{}", restore.name_any()));
431 status.safety_backup_manifest_key = Some(verified.manifest_key);
432 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
433 record_safety_backup_duration(&restore, &ctx).await;
434 status.phase = KanidmRestorePhase::PreparingSource;
435 status.message = None;
436 patch_status(&restore, &ctx, status).await?;
437 }
438 Err(error) => {
439 resume_before_mutation(&restore, &ctx).await?;
440 set_phase(
441 &restore,
442 &ctx,
443 KanidmRestorePhase::Failed,
444 Some(format!("safety backup result verification failed: {error}")),
445 )
446 .await?;
447 }
448 }
449 }
450 JobState::Failed => {
451 resume_before_mutation(&restore, &ctx).await?;
452 set_phase(
453 &restore,
454 &ctx,
455 KanidmRestorePhase::Failed,
456 Some(
457 "safety backup job failed; target restored to original state"
458 .to_string(),
459 ),
460 )
461 .await?;
462 }
463 JobState::Running => {
464 patch_status(&restore, &ctx, status).await?;
465 }
466 }
467 Ok(Action::requeue(REQUEUE))
468 }
469 KanidmRestorePhase::PreparingSource => {
470 if is_remote_source(&restore) {
471 let target = get_target(&restore, &ctx).await?;
472 let name = source_prep_job_name(&restore);
473 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
474 match job_state(&restore, &ctx, &name).await? {
475 JobState::Complete => {
476 let backup_id = restore
477 .spec
478 .source
479 .backup_ref
480 .as_ref()
481 .map(|b| b.name.clone())
482 .unwrap_or_default();
483 let backup = {
484 let ns = restore.namespace().unwrap();
485 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
486 .get(&backup_id)
487 .await
488 .map_err(|e| {
489 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
490 })?
491 };
492 let expected_backup_id = &backup.spec.backup_id;
493 let result =
494 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
495 .await;
496 match result {
497 Ok(_verified) => {
498 let mut status = restore.status.clone().unwrap_or_default();
499 status.source_prep_job_name = Some(name);
500 status.phase = KanidmRestorePhase::RestoringPrimary;
501 status.message = None;
502 patch_status(&restore, &ctx, status).await?;
503 }
504 Err(error) => {
505 resume_before_mutation(&restore, &ctx).await?;
506 set_phase(
507 &restore,
508 &ctx,
509 KanidmRestorePhase::Failed,
510 Some(format!(
511 "source preparation result verification failed: {error}"
512 )),
513 )
514 .await?;
515 }
516 }
517 }
518 JobState::Failed => {
519 resume_before_mutation(&restore, &ctx).await?;
520 set_phase(
521 &restore,
522 &ctx,
523 KanidmRestorePhase::Failed,
524 Some(
525 "source preparation job failed; target restored to original state"
526 .to_string(),
527 ),
528 )
529 .await?;
530 }
531 JobState::Running => {}
532 }
533 } else {
534 let target = get_target(&restore, &ctx).await?;
535 let name = source_check_job_name(&restore);
536 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
537 match job_state(&restore, &ctx, &name).await? {
538 JobState::Complete => {
539 let mut status = restore.status.clone().unwrap_or_default();
540 status.source_prep_job_name = Some(name);
541 status.phase = KanidmRestorePhase::RestoringPrimary;
542 status.message = None;
543 patch_status(&restore, &ctx, status).await?;
544 }
545 JobState::Failed => {
546 resume_before_mutation(&restore, &ctx).await?;
547 set_phase(
548 &restore,
549 &ctx,
550 KanidmRestorePhase::Failed,
551 Some(
552 "local backup file check failed; target restored to original state"
553 .to_string(),
554 ),
555 )
556 .await?;
557 }
558 JobState::Running => {}
559 }
560 }
561 Ok(Action::requeue(REQUEUE))
562 }
563 KanidmRestorePhase::RestoringPrimary => {
564 let target = get_target(&restore, &ctx).await?;
565 let status = restore.status.as_ref().cloned().unwrap_or_default();
566 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
567 return Err(Error::MissingData(
568 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
569 ));
570 }
571 if !status.database_mutation_started {
572 let mut status = status;
573 status.database_mutation_started = true;
574 status.message = Some(
575 "database mutation boundary persisted; starting primary restore".to_string(),
576 );
577 patch_status(&restore, &ctx, status).await?;
578 return Ok(Action::requeue(REQUEUE));
579 }
580 let name = restore_job_name(&restore);
581 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
582 match job_state(&restore, &ctx, &name).await? {
583 JobState::Complete => {
584 let mut status = restore.status.clone().unwrap_or_default();
585 status.restore_job_name = Some(name);
586 status.phase = KanidmRestorePhase::Verifying;
587 status.message = None;
588 patch_status(&restore, &ctx, status).await?;
589 }
590 JobState::Failed => {
591 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
592 }
593 JobState::Running => {}
594 }
595 Ok(Action::requeue(REQUEUE))
596 }
597 KanidmRestorePhase::Verifying => {
598 let target = get_target(&restore, &ctx).await?;
599 let name = verify_job_name(&restore);
600 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
601 match job_state(&restore, &ctx, &name).await? {
602 JobState::Complete => {
603 let mut status = restore.status.clone().unwrap_or_default();
604 status.verify_job_name = Some(name);
605 status.phase = KanidmRestorePhase::RebuildingReplicas;
606 status.message = None;
607 patch_status(&restore, &ctx, status).await?;
608 }
609 JobState::Failed => {
610 fail_after_mutation(&restore, &ctx, "database verification failed").await?
611 }
612 JobState::Running => {}
613 }
614 Ok(Action::requeue(REQUEUE))
615 }
616 KanidmRestorePhase::RebuildingReplicas => {
617 let target = get_target(&restore, &ctx).await?;
618 let mut status = restore.status.clone().unwrap_or_default();
619 if !status.replicas_cleared {
620 if delete_secondary_pvcs(&target, &ctx).await? {
621 status.replicas_cleared = true;
622 status.message = Some("secondary database state cleared".to_string());
623 patch_status(&restore, &ctx, status).await?;
624 }
625 return Ok(Action::requeue(REQUEUE));
626 }
627
628 scale_primary(&target, &ctx, 1).await?;
629 if !primary_ready(&target, &ctx).await? {
630 return Ok(Action::requeue(REQUEUE));
631 }
632 if !status.certificates_cleared {
633 delete_replica_cert_secrets(&target, &ctx).await?;
634 delete_admin_secret(&target, &ctx).await?;
635 status.certificates_cleared = true;
636 status.message = Some(
637 "replica certificates and admin secret cleared for regeneration".to_string(),
638 );
639 patch_status(&restore, &ctx, status).await?;
640 return Ok(Action::requeue(REQUEUE));
641 }
642 status.phase = KanidmRestorePhase::Resuming;
643 status.message = None;
644 patch_status(&restore, &ctx, status).await?;
645 Ok(Action::requeue(REQUEUE))
646 }
647 KanidmRestorePhase::Resuming => {
648 let target = get_target(&restore, &ctx).await?;
649 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
650 clear_restoring(&restore, &target, &ctx).await?;
651 Ok(Action::requeue(Duration::from_secs(3600)))
652 }
653 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
654 Ok(Action::requeue(Duration::from_secs(3600)))
655 }
656 }
657}
658
659async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
660 let status = restore.status.clone().unwrap_or_default();
661 if status.database_mutation_started
662 && status.phase != KanidmRestorePhase::Completed
663 && status.phase != KanidmRestorePhase::Failed
664 {
665 reconcile_apply(restore.clone(), ctx.clone()).await?;
666 return Ok(Action::requeue(REQUEUE));
667 }
668 if let Ok(target) = get_target(&restore, &ctx).await {
669 let owns_maintenance =
670 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
671 if owns_maintenance && !status.database_mutation_started {
672 scale_desired(&target, &ctx).await?;
673 }
674 clear_restoring(&restore, &target, &ctx).await?;
675 }
676 Ok(Action::await_change())
677}
678
679async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
680 let target = get_target(restore, ctx).await?;
681 let actual_uid = target
682 .uid()
683 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
684 if actual_uid != restore.spec.target_ref.uid {
685 return Err(Error::MissingData(format!(
686 "target UID mismatch: expected {}, got {}",
687 restore.spec.target_ref.uid, actual_uid
688 )));
689 }
690 validate_source(restore)?;
691 if is_remote_source(restore) {
692 validate_backup_ref(restore, &target, ctx).await?;
693 }
694 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
695 {
696 return Err(Error::MissingData(format!(
697 "restoreImage must be the target's pinned Kanidm image (target image is {})",
698 target.spec.image
699 )));
700 }
701 let storage = target
702 .spec
703 .storage
704 .as_ref()
705 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
706 if storage.empty_dir.is_some()
707 || storage.ephemeral.is_some()
708 || storage.volume_claim_template.is_none()
709 {
710 return Err(Error::MissingData(
711 "restore requires PVC-backed Kanidm storage".to_string(),
712 ));
713 }
714 let primaries = target
715 .spec
716 .replica_groups
717 .iter()
718 .filter(|rg| rg.primary_node)
719 .count();
720 if primaries != 1 {
721 return Err(Error::MissingData(
722 "backup/restore requires exactly one primary replica group".to_string(),
723 ));
724 }
725 let ns = restore.namespace().unwrap();
726 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
727 .list(&ListParams::default())
728 .await
729 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
730 if restores.items.iter().any(|other| {
731 other.name_any() != restore.name_any()
732 && other.spec.target_ref.name == restore.spec.target_ref.name
733 && !matches!(
734 other.status.as_ref().map(|s| s.phase),
735 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
736 )
737 }) {
738 return Err(Error::MissingData(
739 "another active restore targets this Kanidm".to_string(),
740 ));
741 }
742 validate_safety_backup_config(restore)?;
743 Ok(target)
744}
745
746fn safe_basename(name: &str) -> bool {
747 !name.is_empty()
748 && name != "."
749 && name != ".."
750 && !name.contains('/')
751 && !name.contains('\\')
752 && !name.contains('\0')
753}
754
755fn mutable_image(image: &str) -> bool {
756 image == "kanidm/server:latest"
757 || image.ends_with(":latest")
758 || (!image.contains('@')
759 && !image
760 .rsplit('/')
761 .next()
762 .is_some_and(|part| part.contains(':')))
763}
764
765async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
766 let ns = restore
767 .namespace()
768 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
769 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
770 .get(&restore.spec.target_ref.name)
771 .await
772 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
773}
774
775async fn mark_restoring(
776 restore: &KanidmRestore,
777 target: &Kanidm,
778 ctx: &RestoreContext,
779) -> Result<()> {
780 let ns = target.namespace().unwrap();
781 let uid = restore
782 .uid()
783 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
784 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
785 .patch(
786 &target.name_any(),
787 &PatchParams::default(),
788 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
789 )
790 .await
791 .map(|_| ())
792 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
793}
794
795async fn clear_restoring(
796 restore: &KanidmRestore,
797 target: &Kanidm,
798 ctx: &RestoreContext,
799) -> Result<()> {
800 let ns = target.namespace().unwrap();
801 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
802 return Ok(());
803 }
804 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
805 .patch(
806 &target.name_any(),
807 &PatchParams::default(),
808 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
809 )
810 .await
811 .map(|_| ())
812 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
813}
814
815async fn patch_status(
816 restore: &KanidmRestore,
817 ctx: &RestoreContext,
818 mut status: KanidmRestoreStatus,
819) -> Result<()> {
820 let ns = restore.namespace().unwrap();
821 let previous_phase = restore.status.as_ref().map(|current| current.phase);
822 let next_phase = status.phase;
823 status.observed_generation = restore.metadata.generation;
824 update_restore_conditions(&mut status, restore.metadata.generation);
825
826 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
827 .patch_status(
828 &restore.name_any(),
829 &PatchParams::default(),
830 &Patch::Merge(json!({"status": &status})),
831 )
832 .await
833 .map_err(|e| {
834 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
835 })?;
836
837 if previous_phase != Some(next_phase) {
838 record_restore_transition(
839 restore,
840 ctx,
841 previous_phase,
842 next_phase,
843 status.message.as_deref(),
844 )
845 .await;
846 }
847 Ok(())
848}
849
850fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
851 let previous = status.conditions.clone();
852 let phase = status.phase;
853 let phase_reason = format!("{phase:?}");
854 let message = status
855 .message
856 .clone()
857 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
858 let terminal_success = phase == KanidmRestorePhase::Completed;
859 let terminal_failure = phase == KanidmRestorePhase::Failed;
860 let progressing = !terminal_success && !terminal_failure;
861
862 let break_glass = previous
863 .iter()
864 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
865 .cloned();
866 status.conditions = vec![
867 restore_condition(
868 &previous,
869 CONDITION_PROGRESSING,
870 if progressing {
871 CONDITION_TRUE
872 } else {
873 CONDITION_FALSE
874 },
875 &phase_reason,
876 &message,
877 generation,
878 ),
879 restore_condition(
880 &previous,
881 CONDITION_READY,
882 if terminal_success {
883 CONDITION_TRUE
884 } else {
885 CONDITION_FALSE
886 },
887 if terminal_success {
888 "RestoreCompleted"
889 } else {
890 "RestoreNotCompleted"
891 },
892 if terminal_success {
893 "Kanidm restore completed successfully."
894 } else {
895 "Kanidm restore has not completed successfully."
896 },
897 generation,
898 ),
899 restore_condition(
900 &previous,
901 CONDITION_FAILED,
902 if terminal_failure {
903 CONDITION_TRUE
904 } else {
905 CONDITION_FALSE
906 },
907 if terminal_failure {
908 "RestoreFailed"
909 } else {
910 "NoRestoreFailure"
911 },
912 if terminal_failure {
913 status
914 .message
915 .as_deref()
916 .unwrap_or("Kanidm restore failed.")
917 } else {
918 "No terminal restore failure has been recorded."
919 },
920 generation,
921 ),
922 ];
923 if let Some(condition) = break_glass {
924 status.conditions.push(condition);
925 }
926}
927
928fn restore_condition(
929 previous: &[Condition],
930 condition_type: &str,
931 condition_status: &str,
932 reason: &str,
933 message: &str,
934 generation: Option<i64>,
935) -> Condition {
936 let last_transition_time = previous
937 .iter()
938 .find(|condition| {
939 condition.type_ == condition_type
940 && condition.status == condition_status
941 && condition.reason == reason
942 })
943 .map(|condition| condition.last_transition_time.clone())
944 .unwrap_or_else(|| Time(Timestamp::now()));
945 Condition {
946 type_: condition_type.to_string(),
947 status: condition_status.to_string(),
948 reason: reason.to_string(),
949 message: message.to_string(),
950 last_transition_time,
951 observed_generation: generation,
952 }
953}
954
955async fn record_restore_transition(
956 restore: &KanidmRestore,
957 ctx: &RestoreContext,
958 previous_phase: Option<KanidmRestorePhase>,
959 phase: KanidmRestorePhase,
960 message: Option<&str>,
961) {
962 if phase == KanidmRestorePhase::Validating {
963 ctx.metrics.attempts.add(1, &[]);
964 }
965
966 let result = match phase {
967 KanidmRestorePhase::Completed => Some("success"),
968 KanidmRestorePhase::Failed => Some("failure"),
969 _ => None,
970 };
971 if let Some(result) = result {
972 let attributes = [KeyValue::new("result", result)];
973 ctx.metrics.outcomes.add(1, &attributes);
974 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
975 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
976 ctx.metrics.duration_seconds.record(elapsed, &attributes);
977 }
978 }
979
980 let reason = if phase == KanidmRestorePhase::Failed {
981 "RestoreFailed"
982 } else {
983 "RestorePhaseChanged"
984 };
985 let note = message.map(str::to_string).or_else(|| {
986 Some(format!(
987 "Kanidm restore phase changed from {} to {phase:?}.",
988 previous_phase
989 .map(|previous| format!("{previous:?}"))
990 .unwrap_or_else(|| "None".to_string())
991 ))
992 });
993 if let Err(error) = ctx
994 .recorder
995 .publish(
996 &Event {
997 type_: if phase == KanidmRestorePhase::Failed {
998 EventType::Warning
999 } else {
1000 EventType::Normal
1001 },
1002 reason: reason.to_string(),
1003 note,
1004 action: "Restore".to_string(),
1005 secondary: None,
1006 },
1007 &restore.object_ref(&()),
1008 )
1009 .await
1010 {
1011 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1012 }
1013}
1014
1015async fn set_phase(
1016 restore: &KanidmRestore,
1017 ctx: &RestoreContext,
1018 phase: KanidmRestorePhase,
1019 message: Option<String>,
1020) -> Result<()> {
1021 let mut status = restore.status.clone().unwrap_or_default();
1022 status.phase = phase;
1023 status.message = message;
1024 patch_status(restore, ctx, status).await
1025}
1026
1027async fn fail_after_mutation(
1028 restore: &KanidmRestore,
1029 ctx: &RestoreContext,
1030 message: &str,
1031) -> Result<()> {
1032 set_phase(
1033 restore,
1034 ctx,
1035 KanidmRestorePhase::Failed,
1036 Some(message.to_string()),
1037 )
1038 .await
1039}
1040
1041fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1042 target
1043 .spec
1044 .replica_groups
1045 .iter()
1046 .find(|rg| rg.primary_node)
1047 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1048}
1049
1050async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1051 let ns = target.namespace().unwrap();
1052 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1053 for rg in &target.spec.replica_groups {
1054 let name = target.statefulset_name(&rg.name);
1055 api.patch(
1056 &name,
1057 &PatchParams::default(),
1058 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1059 )
1060 .await
1061 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1062 }
1063 Ok(())
1064}
1065
1066async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1067 let ns = target.namespace().unwrap();
1068 let rg = primary_group(target)?;
1069 let name = target.statefulset_name(&rg.name);
1070 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1071 .patch(
1072 &name,
1073 &PatchParams::default(),
1074 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1075 )
1076 .await
1077 .map(|_| ())
1078 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1079}
1080
1081async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1082 let ns = target.namespace().unwrap();
1083 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1084 for rg in &target.spec.replica_groups {
1085 let name = target.statefulset_name(&rg.name);
1086 api.patch(
1087 &name,
1088 &PatchParams::default(),
1089 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1090 )
1091 .await
1092 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1093 }
1094 Ok(())
1095}
1096
1097async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098 let ns = target.namespace().unwrap();
1099 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1100 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1101 .await
1102 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1103 Ok(pods.items.is_empty())
1104}
1105
1106async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1107 let ns = target.namespace().unwrap();
1108 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1109 let mut pv_names = BTreeSet::new();
1110 let primary_name = primary_pvc_name(target)?;
1111 let primary_pvc = pvc_api
1112 .get(&primary_name)
1113 .await
1114 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1115 let primary_pv = primary_pvc
1116 .spec
1117 .as_ref()
1118 .and_then(|spec| spec.volume_name.clone())
1119 .ok_or_else(|| {
1120 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1121 })?;
1122 pv_names.insert(primary_pv);
1123
1124 for rg in &target.spec.replica_groups {
1125 let sts = target.statefulset_name(&rg.name);
1126 for ordinal in 0..rg.replicas {
1127 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1128 if name == primary_name {
1129 continue;
1130 }
1131 if let Some(pvc) = pvc_api
1132 .get_opt(&name)
1133 .await
1134 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1136 {
1137 pv_names.insert(pv_name);
1138 }
1139 }
1140 }
1141
1142 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1143 .list(&ListParams::default())
1144 .await
1145 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1146 Ok(!attachments.items.iter().any(|attachment| {
1147 attachment
1148 .spec
1149 .source
1150 .persistent_volume_name
1151 .as_ref()
1152 .is_some_and(|pv_name| pv_names.contains(pv_name))
1153 }))
1154}
1155
1156async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1157 let ns = target.namespace().unwrap();
1158 let name = target.statefulset_name(&primary_group(target)?.name);
1159 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1160 .get(&name)
1161 .await
1162 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1163 Ok(sts
1164 .status
1165 .as_ref()
1166 .and_then(|s| s.ready_replicas)
1167 .unwrap_or(0)
1168 >= 1)
1169}
1170
1171fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1172 let rg = primary_group(target)?;
1173 Ok(format!(
1174 "{DATA_VOLUME}-{}-0",
1175 target.statefulset_name(&rg.name)
1176 ))
1177}
1178
1179async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1180 let ns = target.namespace().unwrap();
1181 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1182 let mut all_absent = true;
1183 for rg in &target.spec.replica_groups {
1184 let sts = target.statefulset_name(&rg.name);
1185 for ordinal in 0..rg.replicas {
1186 if rg.primary_node && ordinal == 0 {
1187 continue;
1188 }
1189 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1190 match api
1191 .get_opt(&name)
1192 .await
1193 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1194 {
1195 None => {}
1196 Some(pvc) => {
1197 all_absent = false;
1198 if pvc.metadata.deletion_timestamp.is_none() {
1199 match api.delete(&name, &DeleteParams::default()).await {
1200 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1201 Err(kube::Error::Api(status)) if status.code == 404 => {}
1202 Err(error) => {
1203 return Err(Error::kube_error(
1204 "delete",
1205 "PersistentVolumeClaim",
1206 &ns,
1207 &name,
1208 error,
1209 ));
1210 }
1211 }
1212 }
1213 }
1214 }
1215 }
1216 }
1217 Ok(all_absent)
1218}
1219
1220async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1221 let ns = target.namespace().unwrap();
1222 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1223 let label_selector = format!(
1224 "{}={},{}={}",
1225 CLUSTER_LABEL,
1226 target.name_any(),
1227 SECRET_TYPE_LABEL,
1228 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1229 );
1230 let secrets = api
1231 .list(&ListParams::default().labels(&label_selector))
1232 .await
1233 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1234 let mut all_absent = secrets.items.is_empty();
1235 for secret in secrets.items {
1236 let name = secret.name_any();
1237 if secret.metadata.deletion_timestamp.is_none() {
1238 match api.delete(&name, &DeleteParams::default()).await {
1239 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1240 Err(kube::Error::Api(status)) if status.code == 404 => {}
1241 Err(error) => {
1242 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1243 }
1244 }
1245 all_absent = false;
1246 }
1247 }
1248 Ok(all_absent)
1249}
1250
1251async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1252 let ns = target.namespace().unwrap();
1253 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1254 let name = format!("{}-admin-passwords", target.name_any());
1255 match api
1256 .get_opt(&name)
1257 .await
1258 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1259 {
1260 None => Ok(true),
1261 Some(secret) => {
1262 if secret.metadata.deletion_timestamp.is_some() {
1263 return Ok(false);
1264 }
1265 match api.delete(&name, &DeleteParams::default()).await {
1266 Ok(_) => {
1267 debug!(secret = %name, "deleting stale admin secret");
1268 Ok(false)
1269 }
1270 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1271 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1272 }
1273 }
1274 }
1275}
1276
1277fn restore_job_name(restore: &KanidmRestore) -> String {
1278 format!("{}-restore", restore.name_any())
1279}
1280fn verify_job_name(restore: &KanidmRestore) -> String {
1281 format!("{}-verify", restore.name_any())
1282}
1283
1284fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1285 let origin = target
1286 .spec
1287 .origin
1288 .clone()
1289 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1290 vec![
1291 EnvVar {
1292 name: "KANIDM_DB_PATH".to_string(),
1293 value: Some("/data/kanidm.db".to_string()),
1294 ..Default::default()
1295 },
1296 EnvVar {
1297 name: "KANIDM_DOMAIN".to_string(),
1298 value: Some(target.spec.domain.clone()),
1299 ..Default::default()
1300 },
1301 EnvVar {
1302 name: "KANIDM_ORIGIN".to_string(),
1303 value: Some(origin),
1304 ..Default::default()
1305 },
1306 EnvVar {
1307 name: "KANIDM_TLS_CHAIN".to_string(),
1308 value: Some(format!("{TLS_PATH}/tls.crt")),
1309 ..Default::default()
1310 },
1311 EnvVar {
1312 name: "KANIDM_TLS_KEY".to_string(),
1313 value: Some(format!("{TLS_PATH}/tls.key")),
1314 ..Default::default()
1315 },
1316 ]
1317}
1318
1319async fn ensure_database_job(
1320 restore: &KanidmRestore,
1321 target: &Kanidm,
1322 ctx: &RestoreContext,
1323 name: &str,
1324 verify: bool,
1325) -> Result<()> {
1326 let ns = restore.namespace().unwrap();
1327 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1328 if api
1329 .get_opt(name)
1330 .await
1331 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1332 .is_some()
1333 {
1334 return Ok(());
1335 }
1336 let mut command = vec![
1337 "kanidmd".to_string(),
1338 "database".to_string(),
1339 if verify { "backup" } else { "restore" }.to_string(),
1340 ];
1341 if verify {
1342 command.push("/verify/verification.json.gz".to_string());
1343 } else {
1344 if is_remote_source(restore) {
1345 command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1346 } else {
1347 let file_name = restore
1348 .spec
1349 .source
1350 .local
1351 .as_ref()
1352 .map(|l| l.file_name.clone())
1353 .unwrap_or_default();
1354 command.push(format!("{BACKUP_PATH}/{file_name}"));
1355 }
1356 }
1357 let job = Job {
1358 metadata: ObjectMeta {
1359 name: Some(name.to_string()),
1360 namespace: Some(ns.clone()),
1361 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1362 ..Default::default()
1363 },
1364 spec: Some(JobSpec {
1365 backoff_limit: Some(0),
1366 template: PodTemplateSpec {
1367 metadata: Some(ObjectMeta::default()),
1368 spec: Some(PodSpec {
1369 automount_service_account_token: Some(false),
1370 restart_policy: Some("Never".to_string()),
1371 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1372 seccomp_profile: Some(SeccompProfile {
1373 type_: "RuntimeDefault".to_string(),
1374 ..Default::default()
1375 }),
1376 ..Default::default()
1377 }),
1378 containers: vec![Container {
1379 name: if verify { "verify" } else { "restore" }.to_string(),
1380 image: Some(restore.spec.restore_image.clone()),
1381 command: Some(command),
1382 env: Some(build_kanidm_db_env_vars(target)),
1383 security_context: Some(kanidm_job_security_context()),
1384 resources: Some(job_resource_requirements()),
1385 volume_mounts: Some(vec![
1386 VolumeMount {
1387 name: DATA_VOLUME.to_string(),
1388 mount_path: DATA_PATH.to_string(),
1389 ..Default::default()
1390 },
1391 VolumeMount {
1392 name: TLS_VOLUME.to_string(),
1393 mount_path: TLS_PATH.to_string(),
1394 read_only: Some(true),
1395 ..Default::default()
1396 },
1397 VolumeMount {
1398 name: "verification-output".to_string(),
1399 mount_path: "/verify".to_string(),
1400 ..Default::default()
1401 },
1402 ]),
1403 ..Default::default()
1404 }],
1405 volumes: Some(vec![
1406 Volume {
1407 name: DATA_VOLUME.to_string(),
1408 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1409 claim_name: primary_pvc_name(target)?,
1410 read_only: Some(false),
1411 }),
1412 ..Default::default()
1413 },
1414 Volume {
1415 name: TLS_VOLUME.to_string(),
1416 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1417 secret_name: Some(target.effective_tls_secret_name()),
1418 ..Default::default()
1419 }),
1420 ..Default::default()
1421 },
1422 Volume {
1423 name: "verification-output".to_string(),
1424 empty_dir: Some(Default::default()),
1425 ..Default::default()
1426 },
1427 ]),
1428 ..Default::default()
1429 }),
1430 },
1431 ..Default::default()
1432 }),
1433 ..Default::default()
1434 };
1435 api.create(&PostParams::default(), &job)
1436 .await
1437 .map(|_| ())
1438 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1439}
1440
1441enum JobState {
1442 Running,
1443 Complete,
1444 Failed,
1445}
1446
1447async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1448 let ns = restore.namespace().unwrap();
1449 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1450 .get(name)
1451 .await
1452 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1453 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1454 return Ok(JobState::Failed);
1455 }
1456 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1457 return Ok(JobState::Complete);
1458 }
1459 Ok(JobState::Running)
1460}
1461
1462fn is_remote_source(restore: &KanidmRestore) -> bool {
1463 restore.spec.source.backup_ref.is_some()
1464}
1465
1466fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1467 let safety = restore.spec.safety_backup.as_ref();
1468 let skip = safety.is_some_and(|s| s.skip);
1469 !skip
1470}
1471
1472fn validate_source(restore: &KanidmRestore) -> Result<()> {
1473 let has_local = restore.spec.source.local.is_some();
1474 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1475 if has_local && has_backup_ref {
1476 return Err(Error::MissingData(
1477 "source.local and source.backupRef are mutually exclusive".to_string(),
1478 ));
1479 }
1480 if !has_local && !has_backup_ref {
1481 return Err(Error::MissingData(
1482 "source must specify either local or backupRef".to_string(),
1483 ));
1484 }
1485 if let Some(local) = &restore.spec.source.local {
1486 if !safe_basename(&local.file_name) {
1487 return Err(Error::MissingData(
1488 "restore source fileName must be a safe basename".to_string(),
1489 ));
1490 }
1491 }
1492 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1493 if backup_ref.name.is_empty() {
1494 return Err(Error::MissingData(
1495 "source.backupRef.name must not be empty".to_string(),
1496 ));
1497 }
1498 }
1499 Ok(())
1500}
1501
1502fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1503 let safety = restore.spec.safety_backup.as_ref();
1504 let skip = safety.is_some_and(|s| s.skip);
1505 if skip {
1506 let annotations = restore.metadata.annotations.as_ref();
1507 let reason = annotations
1508 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1509 .map(|s| s.trim())
1510 .unwrap_or("");
1511 let approver = annotations
1512 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1513 .map(|s| s.trim())
1514 .unwrap_or("");
1515 if reason.is_empty() {
1516 return Err(Error::MissingData(format!(
1517 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1518 )));
1519 }
1520 if approver.is_empty() {
1521 return Err(Error::MissingData(format!(
1522 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1523 )));
1524 }
1525 }
1526 if is_remote_source(restore) && !skip {
1527 let has_repo = safety
1528 .and_then(|s| s.repository_ref.as_ref())
1529 .is_some_and(|r| !r.name.is_empty());
1530 if !has_repo {
1531 return Err(Error::MissingData(
1532 "remote restore requires safetyBackup.repositoryRef".to_string(),
1533 ));
1534 }
1535 }
1536 Ok(())
1537}
1538
1539fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1540 target
1541 .spec
1542 .replica_groups
1543 .iter()
1544 .map(|rg| ReplicaCountEntry {
1545 group: rg.name.clone(),
1546 replicas: rg.replicas,
1547 })
1548 .collect()
1549}
1550
1551async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1552 if let Ok(target) = get_target(restore, ctx).await {
1553 let owns_maintenance =
1554 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1555 if owns_maintenance {
1556 scale_desired(&target, ctx).await?;
1557 clear_restoring(restore, &target, ctx).await?;
1558 }
1559 }
1560 Ok(())
1561}
1562
1563async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564 ctx.metrics.break_glass_total.add(1, &[]);
1565 let reason = restore
1566 .metadata
1567 .annotations
1568 .as_ref()
1569 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1570 .cloned()
1571 .unwrap_or_default();
1572 let approver = restore
1573 .metadata
1574 .annotations
1575 .as_ref()
1576 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1577 .cloned()
1578 .unwrap_or_default();
1579 let mut status = restore.status.clone().unwrap_or_default();
1580 let break_glass_condition = restore_condition(
1581 &status.conditions,
1582 CONDITION_BREAK_GLASS,
1583 CONDITION_TRUE,
1584 "BreakGlassOverride",
1585 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1586 restore.metadata.generation,
1587 );
1588 status
1589 .conditions
1590 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1591 status.conditions.push(break_glass_condition);
1592 patch_status(restore, ctx, status).await?;
1593 if let Err(error) = ctx
1594 .recorder
1595 .publish(
1596 &Event {
1597 type_: EventType::Warning,
1598 reason: "SafetyBackupSkipped".to_string(),
1599 note: Some(format!(
1600 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1601 )),
1602 action: "Restore".to_string(),
1603 secondary: None,
1604 },
1605 &restore.object_ref(&()),
1606 )
1607 .await
1608 {
1609 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1610 }
1611 warn!(
1612 restore = %restore.name_any(),
1613 reason = %reason,
1614 approver = %approver,
1615 "break-glass safety backup override activated"
1616 );
1617 Ok(())
1618}
1619
1620async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1621 let status = restore.status.as_ref().cloned().unwrap_or_default();
1622 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1623 if let Ok(start_ts) = start.parse::<Timestamp>() {
1624 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1625 ctx.metrics
1626 .safety_backup_duration_seconds
1627 .record(elapsed, &[]);
1628 }
1629 }
1630}
1631
1632fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1633 format!("{}-safety-backup", restore.name_any())
1634}
1635
1636fn source_prep_job_name(restore: &KanidmRestore) -> String {
1637 format!("{}-source-prep", restore.name_any())
1638}
1639
1640fn source_check_job_name(restore: &KanidmRestore) -> String {
1641 format!("{}-source-check", restore.name_any())
1642}
1643
1644async fn ensure_source_check_job(
1645 restore: &KanidmRestore,
1646 target: &Kanidm,
1647 ctx: &RestoreContext,
1648 name: &str,
1649) -> Result<()> {
1650 let ns = restore.namespace().unwrap();
1651 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1652 if api
1653 .get_opt(name)
1654 .await
1655 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1656 .is_some()
1657 {
1658 return Ok(());
1659 }
1660 let file_name = restore
1661 .spec
1662 .source
1663 .local
1664 .as_ref()
1665 .map(|l| l.file_name.clone())
1666 .unwrap_or_default();
1667 let operation_doc = serde_json::json!({
1668 "apiVersion": "backup.kaniop.rs/v1alpha1",
1669 "kind": "OperationDocument",
1670 "operation": "check",
1671 "path": format!("{BACKUP_PATH}/{file_name}"),
1672 "resultPath": "/run/kaniop-result/result.json",
1673 "format": "kanidmJsonGzip",
1674 })
1675 .to_string();
1676 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1677 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1678 let data_mover_image = data_mover_image();
1679 let job = Job {
1680 metadata: ObjectMeta {
1681 name: Some(name.to_string()),
1682 namespace: Some(ns.clone()),
1683 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1684 ..Default::default()
1685 },
1686 spec: Some(JobSpec {
1687 backoff_limit: Some(0),
1688 template: PodTemplateSpec {
1689 metadata: Some(ObjectMeta::default()),
1690 spec: Some(PodSpec {
1691 automount_service_account_token: Some(false),
1692 restart_policy: Some("Never".to_string()),
1693 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1694 run_as_non_root: Some(true),
1695 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1696 type_: "RuntimeDefault".to_string(),
1697 ..Default::default()
1698 }),
1699 ..Default::default()
1700 }),
1701 containers: vec![Container {
1702 name: "source-check".to_string(),
1703 image: Some(data_mover_image),
1704 command: Some(vec![
1705 "/bin/kaniop-data-mover".to_string(),
1706 "check".to_string(),
1707 "--operation-doc".to_string(),
1708 "/run/kaniop/operation.json".to_string(),
1709 ]),
1710 security_context: Some(hardened_security_context()),
1711 resources: Some(job_resource_requirements()),
1712 env: Some(vec![EnvVar {
1713 name: "RUST_LOG".to_string(),
1714 value: Some("info".to_string()),
1715 ..Default::default()
1716 }]),
1717 volume_mounts: Some(vec![
1718 VolumeMount {
1719 name: DATA_VOLUME.to_string(),
1720 mount_path: DATA_PATH.to_string(),
1721 read_only: Some(true),
1722 ..Default::default()
1723 },
1724 VolumeMount {
1725 name: "operation".to_string(),
1726 mount_path: "/run/kaniop".to_string(),
1727 read_only: Some(true),
1728 ..Default::default()
1729 },
1730 VolumeMount {
1731 name: "result".to_string(),
1732 mount_path: "/run/kaniop-result".to_string(),
1733 ..Default::default()
1734 },
1735 ]),
1736 ..Default::default()
1737 }],
1738 volumes: Some(vec![
1739 Volume {
1740 name: DATA_VOLUME.to_string(),
1741 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1742 claim_name: primary_pvc_name(target)?,
1743 read_only: Some(true),
1744 }),
1745 ..Default::default()
1746 },
1747 Volume {
1748 name: "operation".to_string(),
1749 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1750 name: operation_cm_name,
1751 ..Default::default()
1752 }),
1753 ..Default::default()
1754 },
1755 Volume {
1756 name: "result".to_string(),
1757 empty_dir: Some(Default::default()),
1758 ..Default::default()
1759 },
1760 ]),
1761 ..Default::default()
1762 }),
1763 },
1764 ..Default::default()
1765 }),
1766 ..Default::default()
1767 };
1768 api.create(&PostParams::default(), &job)
1769 .await
1770 .map(|_| ())
1771 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1772}
1773
1774const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1775
1776fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1777 let ns = restore.namespace().unwrap_or_default();
1778 let name = restore.name_any();
1779 let namespace_uuid =
1780 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1781 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1782 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1783}
1784
1785fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1786 let ns = restore.namespace().unwrap_or_default();
1787 let kanidm_uid = &restore.spec.target_ref.uid;
1788 let backup_id = compute_safety_backup_id(restore);
1789 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1790}
1791
1792struct VerifiedSafetyBackup {
1793 manifest_key: String,
1794 payload_sha256: String,
1795}
1796
1797async fn read_safety_backup_result(
1798 restore: &KanidmRestore,
1799 ctx: &RestoreContext,
1800 job_name: &str,
1801 expected_backup_id: &str,
1802) -> Result<VerifiedSafetyBackup> {
1803 let ns = restore.namespace().unwrap();
1804 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1805 let pod_list = pods
1806 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1807 .await
1808 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1809 let pod = pod_list.items.first().ok_or_else(|| {
1810 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1811 })?;
1812 let pod_name = pod.name_any();
1813 let container_status = pod
1814 .status
1815 .as_ref()
1816 .and_then(|s| s.container_statuses.as_ref())
1817 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1818 .ok_or_else(|| {
1819 Error::MissingData(format!(
1820 "safety-upload container status not found in pod {pod_name}"
1821 ))
1822 })?;
1823 let termination_message = container_status
1824 .state
1825 .as_ref()
1826 .and_then(|state| state.terminated.as_ref())
1827 .and_then(|t| t.message.as_ref())
1828 .ok_or_else(|| {
1829 Error::MissingData(format!(
1830 "termination message not found in pod {pod_name}; result transport failed"
1831 ))
1832 })?;
1833 if termination_message.is_empty() {
1834 return Err(Error::MissingData(format!(
1835 "termination message is empty in pod {pod_name}; result document was not written"
1836 )));
1837 }
1838 let result_doc = parse_result_document(termination_message).map_err(|e| {
1839 Error::ParseError(format!(
1840 "failed to parse result document from pod {pod_name}: {e}"
1841 ))
1842 })?;
1843 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1844 return Err(Error::ParseError(format!(
1845 "unsupported result document version: {}",
1846 result_doc.api_version
1847 )));
1848 }
1849 if result_doc.kind != "ResultDocument" {
1850 return Err(Error::ParseError(format!(
1851 "invalid result document kind: {}",
1852 result_doc.kind
1853 )));
1854 }
1855 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1856 return Err(Error::ParseError(format!(
1857 "result document operation mismatch: expected '{}', got '{}'",
1858 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1859 )));
1860 }
1861 if !result_doc.success {
1862 return Err(Error::ParseError(format!(
1863 "safety backup reported failure with exit code {:?}",
1864 result_doc.exit_code
1865 )));
1866 }
1867 if result_doc.exit_code != ExitCode::Success {
1868 return Err(Error::ParseError(format!(
1869 "safety backup exit code is not Success: {:?}",
1870 result_doc.exit_code
1871 )));
1872 }
1873 let doc_backup_id = result_doc
1874 .backup_id
1875 .as_deref()
1876 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1877 if doc_backup_id != expected_backup_id {
1878 return Err(Error::ParseError(format!(
1879 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1880 )));
1881 }
1882 let manifest_key = result_doc
1883 .manifest_key
1884 .clone()
1885 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1886 if manifest_key.is_empty() {
1887 return Err(Error::ParseError(
1888 "result document manifestKey is empty".to_string(),
1889 ));
1890 }
1891 let payload_sha256 = result_doc
1892 .payload_sha256
1893 .clone()
1894 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1895 if payload_sha256.is_empty() {
1896 return Err(Error::ParseError(
1897 "result document payloadSha256 is empty".to_string(),
1898 ));
1899 }
1900 info!(
1901 restore = %restore.name_any(),
1902 backup_id = %expected_backup_id,
1903 manifest_key = %manifest_key,
1904 "safety backup result verified"
1905 );
1906 Ok(VerifiedSafetyBackup {
1907 manifest_key,
1908 payload_sha256,
1909 })
1910}
1911
1912struct VerifiedSourcePrepResult {
1913 #[allow(dead_code)]
1914 manifest_key: String,
1915 #[allow(dead_code)]
1916 payload_sha256: String,
1917}
1918
1919async fn read_source_prep_result(
1920 restore: &KanidmRestore,
1921 ctx: &RestoreContext,
1922 job_name: &str,
1923 expected_backup_id: &str,
1924) -> Result<VerifiedSourcePrepResult> {
1925 let ns = restore.namespace().unwrap();
1926 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1927 let pod_list = pods
1928 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1929 .await
1930 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1931 let pod = pod_list.items.first().ok_or_else(|| {
1932 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1933 })?;
1934 let pod_name = pod.name_any();
1935 let container_status = pod
1936 .status
1937 .as_ref()
1938 .and_then(|s| s.container_statuses.as_ref())
1939 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1940 .ok_or_else(|| {
1941 Error::MissingData(format!(
1942 "source-prep container status not found in pod {pod_name}"
1943 ))
1944 })?;
1945 let termination_message = container_status
1946 .state
1947 .as_ref()
1948 .and_then(|state| state.terminated.as_ref())
1949 .and_then(|t| t.message.as_ref())
1950 .ok_or_else(|| {
1951 Error::MissingData(format!(
1952 "termination message not found in pod {pod_name}; result transport failed"
1953 ))
1954 })?;
1955 if termination_message.is_empty() {
1956 return Err(Error::MissingData(format!(
1957 "termination message is empty in pod {pod_name}; result document was not written"
1958 )));
1959 }
1960 let result_doc = parse_result_document(termination_message).map_err(|e| {
1961 Error::ParseError(format!(
1962 "failed to parse result document from pod {pod_name}: {e}"
1963 ))
1964 })?;
1965 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1966 return Err(Error::ParseError(format!(
1967 "unsupported result document version: {}",
1968 result_doc.api_version
1969 )));
1970 }
1971 if result_doc.kind != "ResultDocument" {
1972 return Err(Error::ParseError(format!(
1973 "invalid result document kind: {}",
1974 result_doc.kind
1975 )));
1976 }
1977 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1978 return Err(Error::ParseError(format!(
1979 "result document operation mismatch: expected '{}', got '{}'",
1980 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1981 )));
1982 }
1983 if !result_doc.success {
1984 return Err(Error::ParseError(format!(
1985 "source prep reported failure with exit code {:?}",
1986 result_doc.exit_code
1987 )));
1988 }
1989 if result_doc.exit_code != ExitCode::Success {
1990 return Err(Error::ParseError(format!(
1991 "source prep exit code is not Success: {:?}",
1992 result_doc.exit_code
1993 )));
1994 }
1995 let doc_backup_id = result_doc
1996 .backup_id
1997 .as_deref()
1998 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1999 if doc_backup_id != expected_backup_id {
2000 return Err(Error::ParseError(format!(
2001 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2002 )));
2003 }
2004 let manifest_key = result_doc
2005 .manifest_key
2006 .clone()
2007 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2008 if manifest_key.is_empty() {
2009 return Err(Error::ParseError(
2010 "result document manifestKey is empty".to_string(),
2011 ));
2012 }
2013 let payload_sha256 = result_doc
2014 .payload_sha256
2015 .clone()
2016 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2017 if payload_sha256.is_empty() {
2018 return Err(Error::ParseError(
2019 "result document payloadSha256 is empty".to_string(),
2020 ));
2021 }
2022 info!(
2023 restore = %restore.name_any(),
2024 backup_id = %expected_backup_id,
2025 manifest_key = %manifest_key,
2026 payload_sha256 = %payload_sha256,
2027 "source prep result verified"
2028 );
2029 Ok(VerifiedSourcePrepResult {
2030 manifest_key,
2031 payload_sha256,
2032 })
2033}
2034
2035fn hardened_security_context() -> SecurityContext {
2036 SecurityContext {
2037 allow_privilege_escalation: Some(false),
2038 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2039 drop: Some(vec!["ALL".to_string()]),
2040 ..Default::default()
2041 }),
2042 read_only_root_filesystem: Some(true),
2043 run_as_non_root: Some(true),
2044 seccomp_profile: Some(SeccompProfile {
2045 type_: "RuntimeDefault".to_string(),
2046 ..Default::default()
2047 }),
2048 ..Default::default()
2049 }
2050}
2051
2052fn kanidm_job_security_context() -> SecurityContext {
2053 SecurityContext {
2054 allow_privilege_escalation: Some(false),
2055 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2056 drop: Some(vec!["ALL".to_string()]),
2057 ..Default::default()
2058 }),
2059 seccomp_profile: Some(SeccompProfile {
2060 type_: "RuntimeDefault".to_string(),
2061 ..Default::default()
2062 }),
2063 ..Default::default()
2064 }
2065}
2066
2067fn job_resource_requirements() -> ResourceRequirements {
2068 ResourceRequirements {
2069 requests: Some(std::collections::BTreeMap::from([
2070 ("cpu".to_string(), Quantity("100m".to_string())),
2071 ("memory".to_string(), Quantity("128Mi".to_string())),
2072 ])),
2073 limits: Some(std::collections::BTreeMap::from([
2074 ("cpu".to_string(), Quantity("2".to_string())),
2075 ("memory".to_string(), Quantity("2Gi".to_string())),
2076 ])),
2077 ..Default::default()
2078 }
2079}
2080
2081async fn ensure_safety_backup_job(
2082 restore: &KanidmRestore,
2083 target: &Kanidm,
2084 ctx: &RestoreContext,
2085 name: &str,
2086 backup_id: &str,
2087 manifest_key: &str,
2088) -> Result<()> {
2089 let ns = restore.namespace().unwrap();
2090 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2091 if api
2092 .get_opt(name)
2093 .await
2094 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2095 .is_some()
2096 {
2097 return Ok(());
2098 }
2099 let safety = restore
2100 .spec
2101 .safety_backup
2102 .as_ref()
2103 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2104 let repo_ref = safety
2105 .repository_ref
2106 .as_ref()
2107 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2108 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2109 .get(&repo_ref.name)
2110 .await
2111 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2112 let _namespace_uid = &ns;
2113 let _kanidm_uid = &restore.spec.target_ref.uid;
2114 let endpoint = &repo.spec.s3.endpoint;
2115 let region = &repo.spec.s3.region;
2116 let operation_doc = build_safety_upload_operation_doc(
2117 restore,
2118 target,
2119 backup_id,
2120 manifest_key,
2121 &repo.spec.s3.bucket,
2122 &repo.spec.s3.prefix,
2123 endpoint,
2124 region,
2125 repo.spec.s3.force_path_style,
2126 repo.spec.s3.insecure,
2127 repo.spec
2128 .s3
2129 .ca_bundle_ref
2130 .as_deref()
2131 .map(|_| ca_bundle_path())
2132 .as_deref(),
2133 repo.spec.encryption.as_ref(),
2134 )?;
2135 let operation_cm_name = format!("{name}-op");
2136 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2137 let auth_method = &repo.spec.authentication.writer;
2138 let data_mover_image = data_mover_image();
2139 let init_command = vec![
2140 "kanidmd".to_string(),
2141 "database".to_string(),
2142 "backup".to_string(),
2143 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2144 ];
2145 let upload_script = format!(
2146 r#"set -eu
2147/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2148RESULT_FILE="/run/kaniop-result/result.json"
2149TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2150if [ ! -f "$RESULT_FILE" ]; then
2151 echo "ERROR: result document not found at $RESULT_FILE"
2152 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2153 exit 1
2154fi
2155if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2156 echo "ERROR: safety backup upload reported failure"
2157 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2158 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2159 exit 1
2160fi
2161dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2162echo "safety backup upload completed successfully"
2163"#
2164 );
2165 let job = Job {
2166 metadata: ObjectMeta {
2167 name: Some(name.to_string()),
2168 namespace: Some(ns.clone()),
2169 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2170 ..Default::default()
2171 },
2172 spec: Some(JobSpec {
2173 backoff_limit: Some(0),
2174 template: PodTemplateSpec {
2175 metadata: Some(ObjectMeta::default()),
2176 spec: Some(PodSpec {
2177 automount_service_account_token: Some(false),
2178 restart_policy: Some("Never".to_string()),
2179 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2180 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2181 type_: "RuntimeDefault".to_string(),
2182 ..Default::default()
2183 }),
2184 ..Default::default()
2185 }),
2186 init_containers: Some(vec![Container {
2187 name: "safety-backup".to_string(),
2188 image: Some(restore.spec.restore_image.clone()),
2189 command: Some(init_command),
2190 security_context: Some(SecurityContext {
2191 allow_privilege_escalation: Some(false),
2192 capabilities: Some(Capabilities {
2193 drop: Some(vec!["ALL".to_string()]),
2194 ..Default::default()
2195 }),
2196 read_only_root_filesystem: Some(true),
2197 seccomp_profile: Some(SeccompProfile {
2198 type_: "RuntimeDefault".to_string(),
2199 ..Default::default()
2200 }),
2201 ..Default::default()
2202 }),
2203 resources: Some(job_resource_requirements()),
2204 env: Some(build_kanidm_db_env_vars(target)),
2205 volume_mounts: Some(vec![
2206 VolumeMount {
2207 name: DATA_VOLUME.to_string(),
2208 mount_path: DATA_PATH.to_string(),
2209 read_only: Some(false),
2210 ..Default::default()
2211 },
2212 VolumeMount {
2213 name: TLS_VOLUME.to_string(),
2214 mount_path: TLS_PATH.to_string(),
2215 read_only: Some(true),
2216 ..Default::default()
2217 },
2218 VolumeMount {
2219 name: SHARED_VOLUME.to_string(),
2220 mount_path: SHARED_VOL_PATH.to_string(),
2221 ..Default::default()
2222 },
2223 ]),
2224 ..Default::default()
2225 }]),
2226 containers: vec![Container {
2227 name: "safety-upload".to_string(),
2228 image: Some(data_mover_image),
2229 command: Some(vec!["/bin/sh".to_string()]),
2230 args: Some(vec!["-c".to_string(), upload_script]),
2231 security_context: Some(hardened_security_context()),
2232 resources: Some(job_resource_requirements()),
2233 env: {
2234 let mut env_vars =
2235 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2236 env_vars.push(EnvVar {
2237 name: "RUST_LOG".to_string(),
2238 value: Some("info".to_string()),
2239 ..Default::default()
2240 });
2241 if repo.spec.s3.ca_bundle_ref.is_some() {
2242 env_vars.push(ca_bundle_env_var());
2243 }
2244 env_vars.extend(build_encryption_env_vars(
2245 repo.spec
2246 .encryption
2247 .as_ref()
2248 .and_then(|e| e.key_ref.as_ref()),
2249 ));
2250 Some(env_vars)
2251 },
2252 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2253 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2254 volume_mounts: {
2255 let mut mounts = vec![
2256 VolumeMount {
2257 name: SHARED_VOLUME.to_string(),
2258 mount_path: SHARED_VOL_PATH.to_string(),
2259 read_only: Some(true),
2260 ..Default::default()
2261 },
2262 VolumeMount {
2263 name: "operation".to_string(),
2264 mount_path: "/run/kaniop".to_string(),
2265 read_only: Some(true),
2266 ..Default::default()
2267 },
2268 VolumeMount {
2269 name: "result".to_string(),
2270 mount_path: "/run/kaniop-result".to_string(),
2271 ..Default::default()
2272 },
2273 ];
2274 mounts.extend(build_auth_volume_mounts(auth_method));
2275 if repo.spec.s3.ca_bundle_ref.is_some() {
2276 mounts.push(build_ca_bundle_volume_mount());
2277 }
2278 Some(mounts)
2279 },
2280 ..Default::default()
2281 }],
2282 volumes: {
2283 let mut vols = vec![
2284 Volume {
2285 name: DATA_VOLUME.to_string(),
2286 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2287 claim_name: primary_pvc_name(target)?,
2288 read_only: Some(false),
2289 }),
2290 ..Default::default()
2291 },
2292 Volume {
2293 name: TLS_VOLUME.to_string(),
2294 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2295 secret_name: Some(target.effective_tls_secret_name()),
2296 ..Default::default()
2297 }),
2298 ..Default::default()
2299 },
2300 Volume {
2301 name: SHARED_VOLUME.to_string(),
2302 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2303 size_limit: Some(crate::controller::backup_job_volume_size()),
2304 ..Default::default()
2305 }),
2306 ..Default::default()
2307 },
2308 Volume {
2309 name: "operation".to_string(),
2310 config_map: Some(ConfigMapVolumeSource {
2311 name: operation_cm_name,
2312 ..Default::default()
2313 }),
2314 ..Default::default()
2315 },
2316 Volume {
2317 name: "result".to_string(),
2318 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2319 size_limit: Some(Quantity("16Mi".to_string())),
2320 ..Default::default()
2321 }),
2322 ..Default::default()
2323 },
2324 ];
2325 vols.extend(build_auth_volumes(auth_method));
2326 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2327 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2328 }
2329 Some(vols)
2330 },
2331 ..Default::default()
2332 }),
2333 },
2334 ..Default::default()
2335 }),
2336 ..Default::default()
2337 };
2338 api.create(&PostParams::default(), &job)
2339 .await
2340 .map(|_| ())
2341 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2342}
2343
2344async fn ensure_source_prep_job(
2345 restore: &KanidmRestore,
2346 target: &Kanidm,
2347 ctx: &RestoreContext,
2348 name: &str,
2349) -> Result<()> {
2350 let ns = restore.namespace().unwrap();
2351 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2352 if api
2353 .get_opt(name)
2354 .await
2355 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2356 .is_some()
2357 {
2358 return Ok(());
2359 }
2360 let backup_name = restore
2361 .spec
2362 .source
2363 .backup_ref
2364 .as_ref()
2365 .map(|b| b.name.clone())
2366 .unwrap_or_default();
2367 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2368 .get(&backup_name)
2369 .await
2370 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2371 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2372 .get(&backup.spec.repository_ref.name)
2373 .await
2374 .map_err(|e| {
2375 Error::kube_error(
2376 "get",
2377 "KanidmBackupRepository",
2378 &ns,
2379 &backup.spec.repository_ref.name,
2380 e,
2381 )
2382 })?;
2383 let endpoint = &repo.spec.s3.endpoint;
2384 let region = &repo.spec.s3.region;
2385 let operation_doc = build_download_operation_doc(
2386 restore,
2387 target,
2388 &backup.spec.manifest_key,
2389 &backup.spec.backup_id,
2390 &repo.spec.s3.bucket,
2391 &repo.spec.s3.prefix,
2392 endpoint,
2393 region,
2394 repo.spec.s3.force_path_style,
2395 repo.spec.s3.insecure,
2396 repo.spec
2397 .s3
2398 .ca_bundle_ref
2399 .as_deref()
2400 .map(|_| ca_bundle_path())
2401 .as_deref(),
2402 repo.spec.encryption.as_ref(),
2403 );
2404 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2405 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2406 let auth_method = &repo.spec.authentication.reader;
2407 let data_mover_image = data_mover_image();
2408 let download_script = format!(
2409 r#"set -eu
2410/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2411RESULT_FILE="/run/kaniop-result/result.json"
2412TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2413if [ ! -f "$RESULT_FILE" ]; then
2414 echo "ERROR: result document not found at $RESULT_FILE"
2415 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2416 exit 1
2417fi
2418if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2419 echo "ERROR: source preparation download reported failure"
2420 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2421 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2422 exit 1
2423fi
2424dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2425echo "source preparation download completed successfully"
2426"#
2427 );
2428 let job = Job {
2429 metadata: ObjectMeta {
2430 name: Some(name.to_string()),
2431 namespace: Some(ns.clone()),
2432 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2433 ..Default::default()
2434 },
2435 spec: Some(JobSpec {
2436 backoff_limit: Some(0),
2437 template: PodTemplateSpec {
2438 metadata: Some(ObjectMeta::default()),
2439 spec: Some(PodSpec {
2440 automount_service_account_token: Some(false),
2441 restart_policy: Some("Never".to_string()),
2442 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2443 run_as_non_root: Some(true),
2444 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2445 type_: "RuntimeDefault".to_string(),
2446 ..Default::default()
2447 }),
2448 ..Default::default()
2449 }),
2450 containers: vec![Container {
2451 name: "source-prep".to_string(),
2452 image: Some(data_mover_image),
2453 command: Some(vec!["/bin/sh".to_string()]),
2454 args: Some(vec!["-c".to_string(), download_script]),
2455 security_context: Some(hardened_security_context()),
2456 resources: Some(job_resource_requirements()),
2457 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2458 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2459 env: {
2460 let mut env_vars = build_auth_env_vars(
2461 auth_method,
2462 &backup.spec.repository_ref.name,
2463 AuthRole::Reader,
2464 );
2465 env_vars.push(EnvVar {
2466 name: "RUST_LOG".to_string(),
2467 value: Some("info".to_string()),
2468 ..Default::default()
2469 });
2470 if repo.spec.s3.ca_bundle_ref.is_some() {
2471 env_vars.push(ca_bundle_env_var());
2472 }
2473 env_vars.extend(build_encryption_env_vars(
2474 repo.spec
2475 .encryption
2476 .as_ref()
2477 .and_then(|e| e.key_ref.as_ref()),
2478 ));
2479 Some(env_vars)
2480 },
2481 volume_mounts: {
2482 let mut mounts = vec![
2483 VolumeMount {
2484 name: STAGING_VOLUME.to_string(),
2485 mount_path: STAGING_PATH.to_string(),
2486 ..Default::default()
2487 },
2488 VolumeMount {
2489 name: "operation".to_string(),
2490 mount_path: "/run/kaniop".to_string(),
2491 read_only: Some(true),
2492 ..Default::default()
2493 },
2494 VolumeMount {
2495 name: "result".to_string(),
2496 mount_path: "/run/kaniop-result".to_string(),
2497 ..Default::default()
2498 },
2499 ];
2500 mounts.extend(build_auth_volume_mounts(auth_method));
2501 if repo.spec.s3.ca_bundle_ref.is_some() {
2502 mounts.push(build_ca_bundle_volume_mount());
2503 }
2504 Some(mounts)
2505 },
2506 ..Default::default()
2507 }],
2508 volumes: {
2509 let mut vols = vec![
2510 Volume {
2511 name: STAGING_VOLUME.to_string(),
2512 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2513 size_limit: Some(crate::controller::backup_job_volume_size()),
2514 ..Default::default()
2515 }),
2516 ..Default::default()
2517 },
2518 Volume {
2519 name: "operation".to_string(),
2520 config_map: Some(ConfigMapVolumeSource {
2521 name: operation_cm_name,
2522 ..Default::default()
2523 }),
2524 ..Default::default()
2525 },
2526 Volume {
2527 name: "result".to_string(),
2528 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2529 size_limit: Some(Quantity("16Mi".to_string())),
2530 ..Default::default()
2531 }),
2532 ..Default::default()
2533 },
2534 ];
2535 vols.extend(build_auth_volumes(auth_method));
2536 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2537 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2538 }
2539 Some(vols)
2540 },
2541 ..Default::default()
2542 }),
2543 },
2544 ..Default::default()
2545 }),
2546 ..Default::default()
2547 };
2548 api.create(&PostParams::default(), &job)
2549 .await
2550 .map(|_| ())
2551 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2552}
2553
2554#[allow(clippy::too_many_arguments)]
2555fn build_safety_upload_operation_doc(
2556 restore: &KanidmRestore,
2557 target: &Kanidm,
2558 backup_id: &str,
2559 manifest_key: &str,
2560 bucket: &str,
2561 prefix: &str,
2562 endpoint: &str,
2563 region: &str,
2564 force_path_style: bool,
2565 insecure: bool,
2566 ca_bundle_path: Option<&str>,
2567 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2568) -> Result<String> {
2569 let enc_mode =
2570 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2571 let enc_key_id = encryption
2572 .and_then(|e| e.key_id.as_ref())
2573 .map(|s| serde_json::Value::String(s.clone()));
2574 let op = UploadOperation {
2575 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2576 bucket: bucket.to_string(),
2577 prefix: prefix.to_string(),
2578 endpoint: endpoint.to_string(),
2579 region: region.to_string(),
2580 force_path_style,
2581 insecure,
2582 ca_bundle_path: ca_bundle_path.map(str::to_string),
2583 backup_id: backup_id.to_string(),
2584 namespace_uid: restore.namespace().unwrap_or_default(),
2585 kanidm_uid: restore.spec.target_ref.uid.clone(),
2586 kanidm_name: restore.spec.target_ref.name.clone(),
2587 domain: target.spec.domain.clone(),
2588 kanidm_version: target
2589 .status
2590 .as_ref()
2591 .and_then(|s| s.version.as_ref())
2592 .map(|v| v.image_tag.clone())
2593 .unwrap_or_default(),
2594 image_digest: None,
2595 consistency: "kanidm-offline".to_string(),
2596 reason: "restore-safety".to_string(),
2597 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2598 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2599 result_path: "/run/kaniop-result/result.json".to_string(),
2600 max_concurrent_parts: 4,
2601 max_retries: 3,
2602 };
2603 let _ = manifest_key;
2604 let doc = OperationDocument {
2605 api_version: OPERATION_DOC_VERSION.to_string(),
2606 kind: "OperationDocument".to_string(),
2607 spec: OperationSpec::Upload(op),
2608 };
2609 serde_json::to_string(&doc)
2610 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2611}
2612
2613async fn validate_backup_ref(
2614 restore: &KanidmRestore,
2615 target: &Kanidm,
2616 ctx: &RestoreContext,
2617) -> Result<()> {
2618 let backup_ref = restore
2619 .spec
2620 .source
2621 .backup_ref
2622 .as_ref()
2623 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2624 let ns = restore.namespace().unwrap();
2625 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2626 .get(&backup_ref.name)
2627 .await
2628 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2629 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2630 return Err(Error::MissingData(format!(
2631 "KanidmBackup '{}' is not Ready (phase: {:?})",
2632 backup_ref.name,
2633 backup.status.as_ref().map(|s| &s.phase)
2634 )));
2635 }
2636 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2637 return Err(Error::MissingData(format!(
2638 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2639 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2640 )));
2641 }
2642 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2643 return Err(Error::MissingData(format!(
2644 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2645 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2646 )));
2647 }
2648 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2649 .get(&backup.spec.repository_ref.name)
2650 .await
2651 .map_err(|e| {
2652 Error::kube_error(
2653 "get",
2654 "KanidmBackupRepository",
2655 &ns,
2656 &backup.spec.repository_ref.name,
2657 e,
2658 )
2659 })?;
2660 if !has_accepted_condition(
2661 &repo
2662 .status
2663 .as_ref()
2664 .map(|s| &s.conditions)
2665 .cloned()
2666 .unwrap_or_default(),
2667 ) {
2668 return Err(Error::MissingData(format!(
2669 "KanidmBackupRepository '{}' configuration has not been accepted",
2670 backup.spec.repository_ref.name
2671 )));
2672 }
2673 validate_backup_compatibility(restore, target, &backup)?;
2674 Ok(())
2675}
2676
2677fn has_accepted_condition(conditions: &[Condition]) -> bool {
2678 conditions
2679 .iter()
2680 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2681}
2682
2683fn validate_backup_compatibility(
2684 restore: &KanidmRestore,
2685 target: &Kanidm,
2686 backup: &KanidmBackup,
2687) -> Result<()> {
2688 let status = backup.status.as_ref();
2689 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2690 let target_version = target
2691 .status
2692 .as_ref()
2693 .and_then(|s| s.version.as_ref())
2694 .map(|v| v.image_tag.as_str());
2695 if let Some(tv) = target_version {
2696 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2697 return Err(Error::MissingData(format!(
2698 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2699 )));
2700 }
2701 }
2702 }
2703 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2704 if !backup_digest.is_empty()
2705 && restore.spec.restore_image.contains('@')
2706 && !restore.spec.restore_image.contains(backup_digest)
2707 {
2708 return Err(Error::MissingData(format!(
2709 "restore image digest does not match backup image digest '{backup_digest}'"
2710 )));
2711 }
2712 }
2713 Ok(())
2714}
2715
2716#[allow(clippy::too_many_arguments)]
2717fn build_download_operation_doc(
2718 restore: &KanidmRestore,
2719 target: &Kanidm,
2720 manifest_key: &str,
2721 expected_backup_id: &str,
2722 bucket: &str,
2723 prefix: &str,
2724 endpoint: &str,
2725 region: &str,
2726 force_path_style: bool,
2727 insecure: bool,
2728 ca_bundle_path: Option<&str>,
2729 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2730) -> String {
2731 let enc_mode =
2732 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2733 let enc_key_id = encryption
2734 .and_then(|e| e.key_id.as_ref())
2735 .map(|s| serde_json::Value::String(s.clone()));
2736 serde_json::json!({
2737 "apiVersion": "backup.kaniop.rs/v1alpha1",
2738 "kind": "OperationDocument",
2739 "operation": "download",
2740 "manifestKey": manifest_key,
2741 "bucket": bucket,
2742 "prefix": prefix,
2743 "endpoint": endpoint,
2744 "region": region,
2745 "forcePathStyle": force_path_style,
2746 "insecure": insecure,
2747 "caBundlePath": ca_bundle_path,
2748 "expectedBackupId": expected_backup_id,
2749 "expectedKanidmUid": restore.spec.target_ref.uid,
2750 "expectedDomain": target.spec.domain,
2751 "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2752 "resultPath": "/run/kaniop-result/result.json",
2753 "maxRetries": 3,
2754 "encryptionMode": enc_mode,
2755 "encryptionKeyId": enc_key_id,
2756 })
2757 .to_string()
2758}
2759
2760async fn ensure_operation_configmap(
2761 restore: &KanidmRestore,
2762 name: &str,
2763 data: &str,
2764 ns: &str,
2765 ctx: &RestoreContext,
2766) -> Result<()> {
2767 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2768 if api
2769 .get_opt(name)
2770 .await
2771 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2772 .is_some()
2773 {
2774 return Ok(());
2775 }
2776 let cm = ConfigMap {
2777 metadata: ObjectMeta {
2778 name: Some(name.to_string()),
2779 namespace: Some(ns.to_string()),
2780 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2781 ..Default::default()
2782 },
2783 data: Some(std::collections::BTreeMap::from([(
2784 "operation.json".to_string(),
2785 data.to_string(),
2786 )])),
2787 ..Default::default()
2788 };
2789 api.create(&PostParams::default(), &cm)
2790 .await
2791 .map(|_| ())
2792 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2793}
2794
2795#[cfg(test)]
2796mod tests {
2797 use super::{
2798 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2799 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2800 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2801 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2802 hardened_security_context, has_accepted_condition, is_remote_source,
2803 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2804 validate_safety_backup_config, validate_source,
2805 };
2806 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2807 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2808 use kube::api::ObjectMeta as ApiObjectMeta;
2809 use std::collections::BTreeMap;
2810
2811 fn make_restore(
2812 source: KanidmRestoreSource,
2813 safety: Option<SafetyBackupConfig>,
2814 ) -> KanidmRestore {
2815 KanidmRestore {
2816 metadata: ApiObjectMeta {
2817 name: Some("test-restore".to_string()),
2818 namespace: Some("default".to_string()),
2819 ..Default::default()
2820 },
2821 spec: KanidmRestoreSpec {
2822 target_ref: KanidmRestoreTargetRef {
2823 name: "corp-idm".to_string(),
2824 uid: "test-uid-123".to_string(),
2825 },
2826 source,
2827 restore_image: "kanidm/server@sha256:abc".to_string(),
2828 safety_backup: safety,
2829 },
2830 status: None,
2831 }
2832 }
2833
2834 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2835 super::super::crd::Kanidm {
2836 status: Some(super::super::crd::KanidmStatus {
2837 available_replicas: 1,
2838 replicas: 1,
2839 unavailable_replicas: 0,
2840 updated_replicas: 1,
2841 replica_statuses: vec![],
2842 replica_column: "1/1".to_string(),
2843 secret_name: None,
2844 version: Some(super::super::crd::KanidmVersionStatus {
2845 image_tag: image_tag.to_string(),
2846 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2847 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2848 }),
2849 domain_appearance_image: None,
2850 mail_sender: None,
2851 conditions: None,
2852 }),
2853 ..Default::default()
2854 }
2855 }
2856
2857 #[test]
2858 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2859 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2860 }
2861
2862 #[test]
2863 fn completed_restore_sets_ready_condition() {
2864 let mut status = KanidmRestoreStatus {
2865 phase: KanidmRestorePhase::Completed,
2866 ..Default::default()
2867 };
2868 super::update_restore_conditions(&mut status, Some(7));
2869 assert!(status.conditions.iter().any(|condition| {
2870 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2871 }));
2872 assert!(!status.conditions.iter().any(|condition| {
2873 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2874 }));
2875 }
2876
2877 #[test]
2878 fn failed_restore_sets_failed_condition() {
2879 let mut status = KanidmRestoreStatus {
2880 phase: KanidmRestorePhase::Failed,
2881 message: Some("verification failed".to_string()),
2882 ..Default::default()
2883 };
2884 super::update_restore_conditions(&mut status, Some(8));
2885 assert!(status.conditions.iter().any(|condition| {
2886 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2887 }));
2888 }
2889
2890 #[test]
2891 fn restore_filename_is_confined_to_backup_directory() {
2892 assert!(safe_basename("backup.json.gz"));
2893 assert!(!safe_basename("../backup.json.gz"));
2894 assert!(!safe_basename("a/b"));
2895 assert!(!safe_basename(""));
2896 }
2897
2898 #[test]
2899 fn restore_image_rejects_latest_and_unpinned_names() {
2900 assert!(mutable_image("kanidm/server:latest"));
2901 assert!(mutable_image("kanidm/server"));
2902 assert!(!mutable_image("kanidm/server:1.10.0"));
2903 assert!(!mutable_image("kanidm/server@sha256:abc"));
2904 }
2905
2906 #[test]
2907 fn source_validation_rejects_both_local_and_backup_ref() {
2908 let source = KanidmRestoreSource {
2909 local: Some(super::KanidmRestoreLocalSource {
2910 file_name: "backup.json".to_string(),
2911 }),
2912 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2913 name: "some-backup".to_string(),
2914 }),
2915 };
2916 let restore = make_restore(source, None);
2917 assert!(validate_source(&restore).is_err());
2918 }
2919
2920 #[test]
2921 fn source_validation_rejects_empty_source() {
2922 let source = KanidmRestoreSource {
2923 local: None,
2924 backup_ref: None,
2925 };
2926 let restore = make_restore(source, None);
2927 assert!(validate_source(&restore).is_err());
2928 }
2929
2930 #[test]
2931 fn source_validation_accepts_local_source() {
2932 let source = KanidmRestoreSource {
2933 local: Some(super::KanidmRestoreLocalSource {
2934 file_name: "backup.json".to_string(),
2935 }),
2936 backup_ref: None,
2937 };
2938 let restore = make_restore(source, None);
2939 assert!(validate_source(&restore).is_ok());
2940 }
2941
2942 #[test]
2943 fn source_validation_accepts_backup_ref_source() {
2944 let source = KanidmRestoreSource {
2945 local: None,
2946 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2947 name: "corp-idm-019c7c76".to_string(),
2948 }),
2949 };
2950 let restore = make_restore(source, None);
2951 assert!(validate_source(&restore).is_ok());
2952 }
2953
2954 #[test]
2955 fn source_validation_rejects_empty_backup_ref_name() {
2956 let source = KanidmRestoreSource {
2957 local: None,
2958 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2959 name: "".to_string(),
2960 }),
2961 };
2962 let restore = make_restore(source, None);
2963 assert!(validate_source(&restore).is_err());
2964 }
2965
2966 #[test]
2967 fn is_remote_source_true_for_backup_ref() {
2968 let source = KanidmRestoreSource {
2969 local: None,
2970 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2971 name: "backup-1".to_string(),
2972 }),
2973 };
2974 let restore = make_restore(source, None);
2975 assert!(is_remote_source(&restore));
2976 }
2977
2978 #[test]
2979 fn is_remote_source_false_for_local() {
2980 let source = KanidmRestoreSource {
2981 local: Some(super::KanidmRestoreLocalSource {
2982 file_name: "backup.json".to_string(),
2983 }),
2984 backup_ref: None,
2985 };
2986 let restore = make_restore(source, None);
2987 assert!(!is_remote_source(&restore));
2988 }
2989
2990 #[test]
2991 fn requires_safety_backup_default_true() {
2992 let source = KanidmRestoreSource {
2993 local: None,
2994 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2995 name: "backup-1".to_string(),
2996 }),
2997 };
2998 let restore = make_restore(source, None);
2999 assert!(requires_safety_backup(&restore));
3000 }
3001
3002 #[test]
3003 fn requires_safety_backup_false_when_skip() {
3004 let source = KanidmRestoreSource {
3005 local: None,
3006 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3007 name: "backup-1".to_string(),
3008 }),
3009 };
3010 let safety = SafetyBackupConfig {
3011 repository_ref: Some(SafetyBackupRepositoryRef {
3012 name: "offsite".to_string(),
3013 }),
3014 skip: true,
3015 };
3016 let restore = make_restore(source, Some(safety));
3017 assert!(!requires_safety_backup(&restore));
3018 }
3019
3020 #[test]
3021 fn break_glass_validation_rejects_skip_without_reason() {
3022 let source = KanidmRestoreSource {
3023 local: None,
3024 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3025 name: "backup-1".to_string(),
3026 }),
3027 };
3028 let safety = SafetyBackupConfig {
3029 repository_ref: Some(SafetyBackupRepositoryRef {
3030 name: "offsite".to_string(),
3031 }),
3032 skip: true,
3033 };
3034 let mut restore = make_restore(source, Some(safety));
3035 restore.metadata.annotations = Some(BTreeMap::from([(
3036 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3037 "commander@example.com".to_string(),
3038 )]));
3039 assert!(validate_safety_backup_config(&restore).is_err());
3040 }
3041
3042 #[test]
3043 fn break_glass_validation_rejects_skip_without_approver() {
3044 let source = KanidmRestoreSource {
3045 local: None,
3046 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3047 name: "backup-1".to_string(),
3048 }),
3049 };
3050 let safety = SafetyBackupConfig {
3051 repository_ref: Some(SafetyBackupRepositoryRef {
3052 name: "offsite".to_string(),
3053 }),
3054 skip: true,
3055 };
3056 let mut restore = make_restore(source, Some(safety));
3057 restore.metadata.annotations = Some(BTreeMap::from([(
3058 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3059 "PVC is unreadable".to_string(),
3060 )]));
3061 assert!(validate_safety_backup_config(&restore).is_err());
3062 }
3063
3064 #[test]
3065 fn break_glass_validation_accepts_valid_annotations() {
3066 let source = KanidmRestoreSource {
3067 local: None,
3068 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3069 name: "backup-1".to_string(),
3070 }),
3071 };
3072 let safety = SafetyBackupConfig {
3073 repository_ref: Some(SafetyBackupRepositoryRef {
3074 name: "offsite".to_string(),
3075 }),
3076 skip: true,
3077 };
3078 let mut restore = make_restore(source, Some(safety));
3079 restore.metadata.annotations = Some(BTreeMap::from([
3080 (
3081 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3082 "PVC is unreadable".to_string(),
3083 ),
3084 (
3085 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3086 "commander@example.com".to_string(),
3087 ),
3088 ]));
3089 assert!(validate_safety_backup_config(&restore).is_ok());
3090 }
3091
3092 #[test]
3093 fn remote_source_requires_repository_ref_when_not_skipping() {
3094 let source = KanidmRestoreSource {
3095 local: None,
3096 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3097 name: "backup-1".to_string(),
3098 }),
3099 };
3100 let safety = SafetyBackupConfig {
3101 repository_ref: None,
3102 skip: false,
3103 };
3104 let restore = make_restore(source, Some(safety));
3105 assert!(validate_safety_backup_config(&restore).is_err());
3106 }
3107
3108 #[test]
3109 fn remote_source_accepts_repository_ref() {
3110 let source = KanidmRestoreSource {
3111 local: None,
3112 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3113 name: "backup-1".to_string(),
3114 }),
3115 };
3116 let safety = SafetyBackupConfig {
3117 repository_ref: Some(SafetyBackupRepositoryRef {
3118 name: "offsite".to_string(),
3119 }),
3120 skip: false,
3121 };
3122 let restore = make_restore(source, Some(safety));
3123 assert!(validate_safety_backup_config(&restore).is_ok());
3124 }
3125
3126 #[test]
3127 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3128 let phases = vec![
3129 KanidmRestorePhase::Pending,
3130 KanidmRestorePhase::Validating,
3131 KanidmRestorePhase::Quiescing,
3132 KanidmRestorePhase::SafetyBackup,
3133 KanidmRestorePhase::PreparingSource,
3134 KanidmRestorePhase::RestoringPrimary,
3135 KanidmRestorePhase::Verifying,
3136 KanidmRestorePhase::RebuildingReplicas,
3137 KanidmRestorePhase::Resuming,
3138 KanidmRestorePhase::Completed,
3139 ];
3140 assert_eq!(phases.len(), 10);
3141 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3142 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3143 }
3144
3145 #[test]
3146 fn status_default_has_empty_phase_timestamps() {
3147 let status = KanidmRestoreStatus::default();
3148 assert!(status.phase_timestamps.is_empty());
3149 }
3150
3151 #[test]
3152 fn status_default_has_empty_original_replicas() {
3153 let status = KanidmRestoreStatus::default();
3154 assert!(status.original_replicas.is_empty());
3155 }
3156
3157 #[test]
3158 fn hardened_security_context_drops_all_capabilities() {
3159 let ctx = hardened_security_context();
3160 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3161 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3162 assert_eq!(ctx.run_as_non_root, Some(true));
3163 let caps = ctx.capabilities.unwrap();
3164 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3165 assert_eq!(
3166 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3167 Some("RuntimeDefault")
3168 );
3169 }
3170
3171 #[test]
3172 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3173 let ctx = kanidm_job_security_context();
3174 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3175 assert!(ctx.run_as_non_root.is_none());
3176 assert!(ctx.read_only_root_filesystem.is_none());
3177 let caps = ctx.capabilities.unwrap();
3178 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3179 assert_eq!(
3180 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3181 Some("RuntimeDefault")
3182 );
3183 }
3184
3185 #[test]
3186 fn safety_backup_job_name_is_deterministic() {
3187 let source = KanidmRestoreSource {
3188 local: Some(super::KanidmRestoreLocalSource {
3189 file_name: "backup.json".to_string(),
3190 }),
3191 backup_ref: None,
3192 };
3193 let restore = make_restore(source, None);
3194 let name1 = super::safety_backup_job_name(&restore);
3195 let name2 = super::safety_backup_job_name(&restore);
3196 assert_eq!(name1, name2);
3197 assert_eq!(name1, "test-restore-safety-backup");
3198 }
3199
3200 #[test]
3201 fn source_prep_job_name_is_deterministic() {
3202 let source = KanidmRestoreSource {
3203 local: None,
3204 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3205 name: "backup-1".to_string(),
3206 }),
3207 };
3208 let restore = make_restore(source, None);
3209 let name = super::source_prep_job_name(&restore);
3210 assert_eq!(name, "test-restore-source-prep");
3211 }
3212
3213 #[test]
3214 fn break_glass_condition_is_recorded_in_status() {
3215 let mut status = KanidmRestoreStatus::default();
3216 let condition = super::restore_condition(
3217 &status.conditions,
3218 CONDITION_BREAK_GLASS,
3219 CONDITION_TRUE,
3220 "BreakGlassOverride",
3221 "Safety backup skipped",
3222 Some(1),
3223 );
3224 status.conditions.push(condition);
3225 assert!(status.conditions.iter().any(|c| {
3226 c.type_ == CONDITION_BREAK_GLASS
3227 && c.status == CONDITION_TRUE
3228 && c.reason == "BreakGlassOverride"
3229 }));
3230 }
3231
3232 #[test]
3233 fn replica_count_entry_serialization() {
3234 let entry = ReplicaCountEntry {
3235 group: "primary".to_string(),
3236 replicas: 3,
3237 };
3238 let json = serde_json::to_string(&entry).unwrap();
3239 assert!(json.contains("\"group\":\"primary\""));
3240 assert!(json.contains("\"replicas\":3"));
3241 }
3242
3243 #[test]
3244 fn local_source_does_not_require_repository_ref() {
3245 let source = KanidmRestoreSource {
3246 local: Some(super::KanidmRestoreLocalSource {
3247 file_name: "backup.json".to_string(),
3248 }),
3249 backup_ref: None,
3250 };
3251 let safety = SafetyBackupConfig {
3252 repository_ref: None,
3253 skip: false,
3254 };
3255 let restore = make_restore(source, Some(safety));
3256 assert!(validate_safety_backup_config(&restore).is_ok());
3257 }
3258
3259 #[test]
3260 fn skip_without_break_glass_annotations_is_rejected() {
3261 let source = KanidmRestoreSource {
3262 local: Some(super::KanidmRestoreLocalSource {
3263 file_name: "backup.json".to_string(),
3264 }),
3265 backup_ref: None,
3266 };
3267 let safety = SafetyBackupConfig {
3268 repository_ref: None,
3269 skip: true,
3270 };
3271 let restore = make_restore(source, Some(safety));
3272 assert!(validate_safety_backup_config(&restore).is_err());
3273 }
3274
3275 #[test]
3276 fn has_accepted_condition_true() {
3277 let conditions = vec![Condition {
3278 type_: "Ready".to_string(),
3279 status: "True".to_string(),
3280 reason: "Accepted".to_string(),
3281 message: String::new(),
3282 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3283 k8s_openapi::jiff::Timestamp::now(),
3284 ),
3285 observed_generation: None,
3286 }];
3287 assert!(has_accepted_condition(&conditions));
3288 }
3289
3290 #[test]
3291 fn has_accepted_condition_false_when_not_accepted_reason() {
3292 let conditions = vec![Condition {
3293 type_: "Ready".to_string(),
3294 status: "True".to_string(),
3295 reason: "SomeOtherReason".to_string(),
3296 message: String::new(),
3297 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3298 k8s_openapi::jiff::Timestamp::now(),
3299 ),
3300 observed_generation: None,
3301 }];
3302 assert!(!has_accepted_condition(&conditions));
3303 }
3304
3305 #[test]
3306 fn has_accepted_condition_false_when_status_false() {
3307 let conditions = vec![Condition {
3308 type_: "Ready".to_string(),
3309 status: "False".to_string(),
3310 reason: "Accepted".to_string(),
3311 message: String::new(),
3312 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3313 k8s_openapi::jiff::Timestamp::now(),
3314 ),
3315 observed_generation: None,
3316 }];
3317 assert!(!has_accepted_condition(&conditions));
3318 }
3319
3320 #[test]
3321 fn has_accepted_condition_empty() {
3322 assert!(!has_accepted_condition(&[]));
3323 }
3324
3325 #[test]
3326 fn safety_backup_upload_operation_doc_is_valid() {
3327 let source = KanidmRestoreSource {
3328 local: None,
3329 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3330 name: "backup-1".to_string(),
3331 }),
3332 };
3333 let mut restore = make_restore(source, None);
3334 restore.metadata.namespace = Some("test-ns".to_string());
3335 let target = super::super::crd::Kanidm::default();
3336 let doc_str = super::build_safety_upload_operation_doc(
3337 &restore,
3338 &target,
3339 "test-backup-id",
3340 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3341 "my-bucket",
3342 "prod",
3343 "https://s3.example.com",
3344 "us-east-1",
3345 false,
3346 false,
3347 None,
3348 None,
3349 )
3350 .unwrap();
3351 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3352 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3353 assert_eq!(parsed["kind"], "OperationDocument");
3354 assert_eq!(parsed["operation"], "upload");
3355 assert_eq!(parsed["bucket"], "my-bucket");
3356 assert_eq!(parsed["prefix"], "prod");
3357 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3358 assert_eq!(parsed["region"], "us-east-1");
3359 assert_eq!(parsed["backupId"], "test-backup-id");
3360 assert_eq!(parsed["consistency"], "kanidm-offline");
3361 assert_eq!(parsed["reason"], "restore-safety");
3362 assert_eq!(
3363 parsed["payloadPath"],
3364 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3365 );
3366 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3367 }
3368
3369 #[test]
3370 fn download_operation_doc_uses_real_repository_data() {
3371 let source = KanidmRestoreSource {
3372 local: None,
3373 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3374 name: "backup-1".to_string(),
3375 }),
3376 };
3377 let restore = make_restore(source, None);
3378 let target = super::super::crd::Kanidm::default();
3379 let doc_str = super::build_download_operation_doc(
3380 &restore,
3381 &target,
3382 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3383 "019c7c76-f423-7a12-8f41-2bea7588a303",
3384 "real-bucket",
3385 "real-prefix",
3386 "https://real-endpoint.com",
3387 "eu-west-1",
3388 true,
3389 false,
3390 None,
3391 None,
3392 );
3393 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3394 assert_eq!(parsed["bucket"], "real-bucket");
3395 assert_eq!(parsed["prefix"], "real-prefix");
3396 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3397 assert_eq!(parsed["region"], "eu-west-1");
3398 assert_eq!(parsed["forcePathStyle"], true);
3399 assert_eq!(
3400 parsed["manifestKey"],
3401 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3402 );
3403 assert_eq!(
3404 parsed["expectedBackupId"],
3405 "019c7c76-f423-7a12-8f41-2bea7588a303"
3406 );
3407 }
3408
3409 #[test]
3410 fn safety_backup_ref_is_set_after_verification() {
3411 let status = KanidmRestoreStatus {
3412 phase: KanidmRestorePhase::SafetyBackup,
3413 safety_backup_job_name: Some("test-safety-backup".to_string()),
3414 safety_backup_ref: Some("safety-test-restore".to_string()),
3415 safety_backup_expected_backup_id: Some(
3416 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3417 ),
3418 safety_backup_manifest_key: Some(
3419 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3420 ),
3421 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3422 ..Default::default()
3423 };
3424 assert!(status.safety_backup_ref.is_some());
3425 assert!(
3426 status
3427 .safety_backup_ref
3428 .as_ref()
3429 .unwrap()
3430 .starts_with("safety-")
3431 );
3432 assert!(status.safety_backup_expected_backup_id.is_some());
3433 assert!(status.safety_backup_manifest_key.is_some());
3434 assert!(status.safety_backup_payload_sha256.is_some());
3435 }
3436
3437 #[test]
3438 fn database_mutation_not_started_without_safety_backup_ref() {
3439 let status = KanidmRestoreStatus {
3440 phase: KanidmRestorePhase::PreparingSource,
3441 safety_backup_ref: None,
3442 database_mutation_started: false,
3443 ..Default::default()
3444 };
3445 assert!(!status.database_mutation_started);
3446 assert!(status.safety_backup_ref.is_none());
3447 }
3448
3449 #[test]
3450 fn pre_boundary_failure_does_not_set_mutation_started() {
3451 let status = KanidmRestoreStatus {
3452 phase: KanidmRestorePhase::Failed,
3453 message: Some(
3454 "safety backup job failed; target restored to original state".to_string(),
3455 ),
3456 database_mutation_started: false,
3457 ..Default::default()
3458 };
3459 assert!(!status.database_mutation_started);
3460 }
3461
3462 #[test]
3463 fn shared_volume_constants_are_distinct() {
3464 assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3465 assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3466 }
3467
3468 #[test]
3469 fn validate_backup_compatibility_matching_versions_passes() {
3470 let source = KanidmRestoreSource {
3471 local: None,
3472 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3473 name: "backup-1".to_string(),
3474 }),
3475 };
3476 let restore = make_restore(source, None);
3477 let target = kanidm_with_version("1.10.4");
3478 let backup = kaniop_backup_core::crd::KanidmBackup {
3479 metadata: kube::api::ObjectMeta::default(),
3480 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3481 backup_id: "id".to_string(),
3482 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3483 name: "k".to_string(),
3484 uid: "uid".to_string(),
3485 },
3486 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3487 name: "repo".to_string(),
3488 },
3489 manifest_key: "key".to_string(),
3490 },
3491 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3492 kanidm_version: Some("1.10.4".to_string()),
3493 ..Default::default()
3494 }),
3495 };
3496 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3497 }
3498
3499 #[test]
3500 fn validate_backup_compatibility_version_mismatch_fails() {
3501 let source = KanidmRestoreSource {
3502 local: None,
3503 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3504 name: "backup-1".to_string(),
3505 }),
3506 };
3507 let restore = make_restore(source, None);
3508 let target = kanidm_with_version("1.11.0");
3509 let backup = kaniop_backup_core::crd::KanidmBackup {
3510 metadata: kube::api::ObjectMeta::default(),
3511 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3512 backup_id: "id".to_string(),
3513 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3514 name: "k".to_string(),
3515 uid: "uid".to_string(),
3516 },
3517 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3518 name: "repo".to_string(),
3519 },
3520 manifest_key: "key".to_string(),
3521 },
3522 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3523 kanidm_version: Some("1.10.4".to_string()),
3524 ..Default::default()
3525 }),
3526 };
3527 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3528 }
3529
3530 #[test]
3531 fn validate_backup_compatibility_no_backup_version_passes() {
3532 let source = KanidmRestoreSource {
3533 local: None,
3534 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3535 name: "backup-1".to_string(),
3536 }),
3537 };
3538 let restore = make_restore(source, None);
3539 let target = super::super::crd::Kanidm::default();
3540 let backup = kaniop_backup_core::crd::KanidmBackup {
3541 metadata: kube::api::ObjectMeta::default(),
3542 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3543 backup_id: "id".to_string(),
3544 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3545 name: "k".to_string(),
3546 uid: "uid".to_string(),
3547 },
3548 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3549 name: "repo".to_string(),
3550 },
3551 manifest_key: "key".to_string(),
3552 },
3553 status: None,
3554 };
3555 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3556 }
3557
3558 #[test]
3559 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3560 let source = KanidmRestoreSource {
3561 local: None,
3562 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3563 name: "backup-1".to_string(),
3564 }),
3565 };
3566 let mut restore = make_restore(source, None);
3567 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3568 let target = super::super::crd::Kanidm::default();
3569 let backup = kaniop_backup_core::crd::KanidmBackup {
3570 metadata: kube::api::ObjectMeta::default(),
3571 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3572 backup_id: "id".to_string(),
3573 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3574 name: "k".to_string(),
3575 uid: "uid".to_string(),
3576 },
3577 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3578 name: "repo".to_string(),
3579 },
3580 manifest_key: "key".to_string(),
3581 },
3582 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3583 image_digest: Some("sha256:originaldigest".to_string()),
3584 ..Default::default()
3585 }),
3586 };
3587 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3588 }
3589
3590 #[test]
3591 fn validate_backup_compatibility_image_digest_match_passes() {
3592 let source = KanidmRestoreSource {
3593 local: None,
3594 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3595 name: "backup-1".to_string(),
3596 }),
3597 };
3598 let mut restore = make_restore(source, None);
3599 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3600 let target = super::super::crd::Kanidm::default();
3601 let backup = kaniop_backup_core::crd::KanidmBackup {
3602 metadata: kube::api::ObjectMeta::default(),
3603 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3604 backup_id: "id".to_string(),
3605 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3606 name: "k".to_string(),
3607 uid: "uid".to_string(),
3608 },
3609 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3610 name: "repo".to_string(),
3611 },
3612 manifest_key: "key".to_string(),
3613 },
3614 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3615 image_digest: Some("sha256:abc123".to_string()),
3616 ..Default::default()
3617 }),
3618 };
3619 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3620 }
3621
3622 #[test]
3623 fn safety_backup_id_is_deterministic() {
3624 let source = KanidmRestoreSource {
3625 local: None,
3626 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3627 name: "backup-1".to_string(),
3628 }),
3629 };
3630 let restore = make_restore(source, None);
3631 let id1 = super::compute_safety_backup_id(&restore);
3632 let id2 = super::compute_safety_backup_id(&restore);
3633 assert_eq!(id1, id2);
3634 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3635 }
3636
3637 #[test]
3638 fn safety_backup_id_differs_for_different_restores() {
3639 let source1 = KanidmRestoreSource {
3640 local: None,
3641 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3642 name: "backup-1".to_string(),
3643 }),
3644 };
3645 let restore1 = make_restore(source1, None);
3646 let mut restore2 = make_restore(
3647 KanidmRestoreSource {
3648 local: None,
3649 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3650 name: "backup-2".to_string(),
3651 }),
3652 },
3653 None,
3654 );
3655 restore2.metadata.name = Some("different-restore".to_string());
3656 let id1 = super::compute_safety_backup_id(&restore1);
3657 let id2 = super::compute_safety_backup_id(&restore2);
3658 assert_ne!(id1, id2);
3659 }
3660
3661 #[test]
3662 fn safety_manifest_key_is_deterministic() {
3663 let source = KanidmRestoreSource {
3664 local: None,
3665 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3666 name: "backup-1".to_string(),
3667 }),
3668 };
3669 let restore = make_restore(source, None);
3670 let key1 = super::compute_safety_manifest_key(&restore);
3671 let key2 = super::compute_safety_manifest_key(&restore);
3672 assert_eq!(key1, key2);
3673 assert!(key1.contains("manifest.json"));
3674 assert!(key1.starts_with("v1/tenants/"));
3675 }
3676
3677 #[test]
3678 fn safety_backup_id_survives_restart() {
3679 let source = KanidmRestoreSource {
3680 local: None,
3681 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3682 name: "backup-1".to_string(),
3683 }),
3684 };
3685 let mut restore = make_restore(source, None);
3686 let original_id = super::compute_safety_backup_id(&restore);
3687 restore.status = Some(KanidmRestoreStatus {
3688 phase: KanidmRestorePhase::SafetyBackup,
3689 safety_backup_expected_backup_id: Some(original_id.clone()),
3690 ..Default::default()
3691 });
3692 let recomputed_id = super::compute_safety_backup_id(&restore);
3693 assert_eq!(original_id, recomputed_id);
3694 }
3695
3696 #[test]
3697 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3698 let source = KanidmRestoreSource {
3699 local: None,
3700 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3701 name: "backup-1".to_string(),
3702 }),
3703 };
3704 let restore = make_restore(source, None);
3705 assert!(super::requires_safety_backup(&restore));
3706 let status_without_ref = KanidmRestoreStatus {
3707 phase: KanidmRestorePhase::RestoringPrimary,
3708 safety_backup_ref: None,
3709 database_mutation_started: false,
3710 ..Default::default()
3711 };
3712 assert!(status_without_ref.safety_backup_ref.is_none());
3713 assert!(!status_without_ref.database_mutation_started);
3714 }
3715
3716 #[test]
3717 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3718 let status_with_ref = KanidmRestoreStatus {
3719 phase: KanidmRestorePhase::RestoringPrimary,
3720 safety_backup_ref: Some("safety-test-restore".to_string()),
3721 safety_backup_manifest_key: Some(
3722 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3723 ),
3724 safety_backup_payload_sha256: Some("abc123".to_string()),
3725 database_mutation_started: false,
3726 ..Default::default()
3727 };
3728 assert!(status_with_ref.safety_backup_ref.is_some());
3729 assert!(!status_with_ref.database_mutation_started);
3730 }
3731
3732 #[test]
3733 fn result_document_verification_rejects_missing_backup_id() {
3734 let result_json = r#"{
3735 "apiVersion": "backup.kaniop.rs/v1alpha1",
3736 "kind": "ResultDocument",
3737 "operation": "upload",
3738 "success": true,
3739 "exitCode": "success",
3740 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3741 "payloadSha256": "abc123"
3742 }"#;
3743 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3744 assert!(result.backup_id.is_none());
3745 }
3746
3747 #[test]
3748 fn result_document_verification_rejects_missing_manifest_key() {
3749 let result_json = r#"{
3750 "apiVersion": "backup.kaniop.rs/v1alpha1",
3751 "kind": "ResultDocument",
3752 "operation": "upload",
3753 "success": true,
3754 "exitCode": "success",
3755 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3756 "payloadSha256": "abc123"
3757 }"#;
3758 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3759 assert!(result.manifest_key.is_none());
3760 }
3761
3762 #[test]
3763 fn result_document_verification_rejects_missing_payload_sha256() {
3764 let result_json = r#"{
3765 "apiVersion": "backup.kaniop.rs/v1alpha1",
3766 "kind": "ResultDocument",
3767 "operation": "upload",
3768 "success": true,
3769 "exitCode": "success",
3770 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3771 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3772 }"#;
3773 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3774 assert!(result.payload_sha256.is_none());
3775 }
3776
3777 #[test]
3778 fn result_document_verification_rejects_failure_exit_code() {
3779 let result_json = r#"{
3780 "apiVersion": "backup.kaniop.rs/v1alpha1",
3781 "kind": "ResultDocument",
3782 "operation": "upload",
3783 "success": false,
3784 "exitCode": "integrity",
3785 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3786 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3787 "payloadSha256": "abc123"
3788 }"#;
3789 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3790 assert!(!result.success);
3791 assert_eq!(
3792 result.exit_code,
3793 kaniop_backup_core::result::ExitCode::Integrity
3794 );
3795 }
3796
3797 #[test]
3798 fn result_document_verification_rejects_wrong_operation() {
3799 let result_json = r#"{
3800 "apiVersion": "backup.kaniop.rs/v1alpha1",
3801 "kind": "ResultDocument",
3802 "operation": "download",
3803 "success": true,
3804 "exitCode": "success",
3805 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3806 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3807 "payloadSha256": "abc123"
3808 }"#;
3809 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3810 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3811 }
3812
3813 #[test]
3814 fn result_document_verification_accepts_valid_document() {
3815 let result_json = r#"{
3816 "apiVersion": "backup.kaniop.rs/v1alpha1",
3817 "kind": "ResultDocument",
3818 "operation": "upload",
3819 "success": true,
3820 "exitCode": "success",
3821 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3822 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3823 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3824 }"#;
3825 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3826 assert!(result.success);
3827 assert_eq!(
3828 result.exit_code,
3829 kaniop_backup_core::result::ExitCode::Success
3830 );
3831 assert_eq!(result.operation, "upload");
3832 assert!(result.backup_id.is_some());
3833 assert!(result.manifest_key.is_some());
3834 assert!(result.payload_sha256.is_some());
3835 }
3836
3837 #[test]
3838 fn termination_message_path_constant_is_valid() {
3839 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3840 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3841 }
3842
3843 #[test]
3844 fn status_tracks_safety_backup_evidence() {
3845 let status = KanidmRestoreStatus {
3846 phase: KanidmRestorePhase::PreparingSource,
3847 safety_backup_ref: Some("safety-test-restore".to_string()),
3848 safety_backup_expected_backup_id: Some(
3849 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3850 ),
3851 safety_backup_manifest_key: Some(
3852 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3853 ),
3854 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3855 ..Default::default()
3856 };
3857 assert!(status.safety_backup_ref.is_some());
3858 assert!(status.safety_backup_expected_backup_id.is_some());
3859 assert!(status.safety_backup_manifest_key.is_some());
3860 assert!(status.safety_backup_payload_sha256.is_some());
3861 }
3862
3863 #[test]
3864 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3865 let invalid_json = "not valid json";
3866 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3867 assert!(result.is_err());
3868 }
3869
3870 #[test]
3871 fn fail_closed_semantics_preserved_on_oversized_result() {
3872 let oversized_json = format!(
3873 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3874 kaniop_backup_core::result::RESULT_DOC_VERSION,
3875 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3876 );
3877 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3878 assert!(result.is_err());
3879 }
3880
3881 #[test]
3882 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3883 let source = KanidmRestoreSource {
3884 local: None,
3885 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3886 name: "backup-1".to_string(),
3887 }),
3888 };
3889 let mut restore = make_restore(source, None);
3890 restore.metadata.namespace = Some("test-ns".to_string());
3891 let target = super::super::crd::Kanidm::default();
3892 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3893 let doc_str = super::build_safety_upload_operation_doc(
3894 &restore,
3895 &target,
3896 "test-backup-id",
3897 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3898 "my-bucket",
3899 "prod",
3900 "https://s3.example.com",
3901 "us-east-1",
3902 false,
3903 false,
3904 Some(&ca_path),
3905 None,
3906 )
3907 .unwrap();
3908 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3909 assert_eq!(parsed["caBundlePath"], ca_path);
3910 }
3911
3912 #[test]
3913 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3914 let source = KanidmRestoreSource {
3915 local: None,
3916 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3917 name: "backup-1".to_string(),
3918 }),
3919 };
3920 let mut restore = make_restore(source, None);
3921 restore.metadata.namespace = Some("test-ns".to_string());
3922 let target = super::super::crd::Kanidm::default();
3923 let doc_str = super::build_safety_upload_operation_doc(
3924 &restore,
3925 &target,
3926 "test-backup-id",
3927 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3928 "my-bucket",
3929 "prod",
3930 "https://s3.example.com",
3931 "us-east-1",
3932 false,
3933 false,
3934 None,
3935 None,
3936 )
3937 .unwrap();
3938 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3939 assert!(parsed["caBundlePath"].is_null());
3940 }
3941
3942 #[test]
3943 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3944 let source = KanidmRestoreSource {
3945 local: None,
3946 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3947 name: "backup-1".to_string(),
3948 }),
3949 };
3950 let restore = make_restore(source, None);
3951 let target = super::super::crd::Kanidm::default();
3952 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3953 let doc_str = super::build_download_operation_doc(
3954 &restore,
3955 &target,
3956 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3957 "019c7c76-f423-7a12-8f41-2bea7588a303",
3958 "real-bucket",
3959 "real-prefix",
3960 "https://real-endpoint.com",
3961 "eu-west-1",
3962 true,
3963 false,
3964 Some(&ca_path),
3965 None,
3966 );
3967 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3968 assert_eq!(parsed["caBundlePath"], ca_path);
3969 }
3970
3971 #[test]
3972 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3973 let source = KanidmRestoreSource {
3974 local: None,
3975 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3976 name: "backup-1".to_string(),
3977 }),
3978 };
3979 let restore = make_restore(source, None);
3980 let target = super::super::crd::Kanidm::default();
3981 let doc_str = super::build_download_operation_doc(
3982 &restore,
3983 &target,
3984 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3985 "019c7c76-f423-7a12-8f41-2bea7588a303",
3986 "real-bucket",
3987 "real-prefix",
3988 "https://real-endpoint.com",
3989 "eu-west-1",
3990 true,
3991 false,
3992 None,
3993 None,
3994 );
3995 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3996 assert!(parsed["caBundlePath"].is_null());
3997 }
3998
3999 #[test]
4000 fn source_prep_result_operation_is_download() {
4001 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4002 assert_ne!(
4003 super::SOURCE_PREP_RESULT_OPERATION,
4004 super::SAFETY_BACKUP_RESULT_OPERATION
4005 );
4006 }
4007
4008 #[test]
4009 fn download_result_document_verification_accepts_valid_document() {
4010 let result_json = r#"{
4011 "apiVersion": "backup.kaniop.rs/v1alpha1",
4012 "kind": "ResultDocument",
4013 "operation": "download",
4014 "success": true,
4015 "exitCode": "success",
4016 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4019 }"#;
4020 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021 assert!(result.success);
4022 assert_eq!(
4023 result.exit_code,
4024 kaniop_backup_core::result::ExitCode::Success
4025 );
4026 assert_eq!(result.operation, "download");
4027 assert_eq!(
4028 result.backup_id.as_deref(),
4029 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4030 );
4031 assert!(result.manifest_key.is_some());
4032 assert!(result.payload_sha256.is_some());
4033 }
4034
4035 #[test]
4036 fn download_result_document_rejects_wrong_operation() {
4037 let result_json = r#"{
4038 "apiVersion": "backup.kaniop.rs/v1alpha1",
4039 "kind": "ResultDocument",
4040 "operation": "upload",
4041 "success": true,
4042 "exitCode": "success",
4043 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4044 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4045 "payloadSha256": "abc123"
4046 }"#;
4047 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4048 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4049 }
4050
4051 #[test]
4052 fn download_result_document_rejects_failure_success_field() {
4053 let result_json = r#"{
4054 "apiVersion": "backup.kaniop.rs/v1alpha1",
4055 "kind": "ResultDocument",
4056 "operation": "download",
4057 "success": false,
4058 "exitCode": "retryable",
4059 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4060 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4061 "payloadSha256": "abc123"
4062 }"#;
4063 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4064 assert!(!result.success);
4065 assert_ne!(
4066 result.exit_code,
4067 kaniop_backup_core::result::ExitCode::Success
4068 );
4069 }
4070
4071 #[test]
4072 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4073 let status = KanidmRestoreStatus {
4074 phase: KanidmRestorePhase::PreparingSource,
4075 database_mutation_started: false,
4076 ..Default::default()
4077 };
4078 assert!(!status.database_mutation_started);
4079 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4080 }
4081
4082 #[test]
4083 fn backup_job_volume_size_default_is_10gi() {
4084 let qty = crate::controller::backup_job_volume_size();
4085 assert_eq!(qty, Quantity("10Gi".to_string()));
4086 }
4087
4088 #[test]
4089 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4090 use kaniop_backup_core::auth::build_encryption_env_vars;
4091 use kaniop_backup_core::crd::SecretRef;
4092
4093 let key_ref = SecretRef {
4094 name: "kek-secret".to_string(),
4095 };
4096 let env_vars = build_encryption_env_vars(Some(&key_ref));
4097 assert_eq!(env_vars.len(), 1);
4098 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4099 let secret_ref = env_vars[0]
4100 .value_from
4101 .as_ref()
4102 .unwrap()
4103 .secret_key_ref
4104 .as_ref()
4105 .unwrap();
4106 assert_eq!(secret_ref.name, "kek-secret");
4107 assert_eq!(secret_ref.key, "encryption-key");
4108 }
4109
4110 #[test]
4111 fn encryption_env_vars_absent_when_no_key_ref() {
4112 use kaniop_backup_core::auth::build_encryption_env_vars;
4113
4114 let env_vars = build_encryption_env_vars(None);
4115 assert!(
4116 env_vars.is_empty(),
4117 "encryption env vars must be empty when key_ref is None"
4118 );
4119 }
4120
4121 #[test]
4122 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4123 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4124
4125 let source = KanidmRestoreSource {
4126 local: None,
4127 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4128 name: "backup-1".to_string(),
4129 }),
4130 };
4131 let mut restore = make_restore(source, None);
4132 restore.metadata.namespace = Some("test-ns".to_string());
4133 let target = super::super::crd::Kanidm::default();
4134 let encryption = RepositoryEncryption {
4135 mode: EncryptionMode::ClientSide,
4136 key_id: None,
4137 key_ref: Some(SecretRef {
4138 name: "kek-secret".to_string(),
4139 }),
4140 };
4141 let doc_str = super::build_safety_upload_operation_doc(
4142 &restore,
4143 &target,
4144 "test-backup-id",
4145 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4146 "my-bucket",
4147 "prod",
4148 "https://s3.example.com",
4149 "us-east-1",
4150 false,
4151 false,
4152 None,
4153 Some(&encryption),
4154 )
4155 .unwrap();
4156 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4157 assert_eq!(parsed["encryptionMode"], "clientSide");
4158 assert!(parsed["encryptionKeyId"].is_null());
4159 }
4160
4161 #[test]
4162 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4163 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4164
4165 let source = KanidmRestoreSource {
4166 local: None,
4167 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4168 name: "backup-1".to_string(),
4169 }),
4170 };
4171 let restore = make_restore(source, None);
4172 let target = super::super::crd::Kanidm::default();
4173 let encryption = RepositoryEncryption {
4174 mode: EncryptionMode::ProviderKms,
4175 key_id: Some("alias/kaniop-backups".to_string()),
4176 key_ref: None,
4177 };
4178 let doc_str = super::build_download_operation_doc(
4179 &restore,
4180 &target,
4181 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4182 "019c7c76-f423-7a12-8f41-2bea7588a303",
4183 "real-bucket",
4184 "real-prefix",
4185 "https://real-endpoint.com",
4186 "eu-west-1",
4187 true,
4188 false,
4189 None,
4190 Some(&encryption),
4191 );
4192 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4193 assert_eq!(parsed["encryptionMode"], "providerKms");
4194 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4195 }
4196}