1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const STAGING_VOLUME: &str = "restore-staging";
54const SHARED_VOLUME: &str = "safety-backup-shared";
55const DATA_PATH: &str = "/data";
56const TLS_VOLUME: &str = "kanidm-certs";
57const TLS_PATH: &str = "/etc/kanidm/tls";
58const BACKUP_PATH: &str = "/data";
59const STAGING_PATH: &str = "/staging";
60const SHARED_VOL_PATH: &str = "/shared";
61const REQUEUE: Duration = Duration::from_secs(2);
62const CONDITION_TRUE: &str = "True";
63const CONDITION_FALSE: &str = "False";
64const CONDITION_PROGRESSING: &str = "Progressing";
65const CONDITION_READY: &str = "Ready";
66const CONDITION_FAILED: &str = "Failed";
67
68pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
69pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
70const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
71const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
72const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
73const SOURCE_PREP_RESULT_OPERATION: &str = "download";
74
75#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
76#[cfg_attr(feature = "schemars", derive(JsonSchema))]
77#[cfg_attr(
78 feature = "schemars",
79 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
80)]
81#[kube(
82 category = "kaniop",
83 group = "kaniop.rs",
84 version = "v1beta1",
85 kind = "KanidmRestore",
86 plural = "kanidmrestores",
87 singular = "kanidmrestore",
88 shortname = "idmrestore",
89 namespaced,
90 status = "KanidmRestoreStatus",
91 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
92 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
93 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
94)]
95#[serde(rename_all = "camelCase")]
96pub struct KanidmRestoreSpec {
97 pub target_ref: KanidmRestoreTargetRef,
98 pub source: KanidmRestoreSource,
99 pub restore_image: String,
100 #[serde(skip_serializing_if = "Option::is_none")]
101 pub safety_backup: Option<SafetyBackupConfig>,
102}
103
104#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
105#[cfg_attr(feature = "schemars", derive(JsonSchema))]
106#[serde(rename_all = "camelCase")]
107pub struct KanidmRestoreTargetRef {
108 pub name: String,
109 pub uid: String,
110}
111
112#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
113#[cfg_attr(feature = "schemars", derive(JsonSchema))]
114#[cfg_attr(
115 feature = "schemars",
116 schemars(extend("x-kubernetes-validations" = [
117 {
118 "message": "exactly one of local or backupRef must be set",
119 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
120 }
121 ]))
122)]
123#[serde(rename_all = "camelCase")]
124pub struct KanidmRestoreSource {
125 #[serde(skip_serializing_if = "Option::is_none")]
127 pub local: Option<KanidmRestoreLocalSource>,
128 #[serde(skip_serializing_if = "Option::is_none")]
130 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
131}
132
133#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
134#[cfg_attr(feature = "schemars", derive(JsonSchema))]
135#[serde(rename_all = "camelCase")]
136pub struct KanidmRestoreLocalSource {
137 pub file_name: String,
139}
140
141#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
142#[cfg_attr(feature = "schemars", derive(JsonSchema))]
143#[serde(rename_all = "camelCase")]
144pub struct KanidmRestoreBackupRefSource {
145 pub name: String,
147}
148
149#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "schemars", derive(JsonSchema))]
151#[serde(rename_all = "camelCase")]
152pub struct SafetyBackupConfig {
153 #[serde(skip_serializing_if = "Option::is_none")]
155 pub repository_ref: Option<SafetyBackupRepositoryRef>,
156 #[serde(default)]
158 pub skip: bool,
159}
160
161#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
162#[cfg_attr(feature = "schemars", derive(JsonSchema))]
163#[serde(rename_all = "camelCase")]
164pub struct SafetyBackupRepositoryRef {
165 pub name: String,
166}
167
168#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
169#[cfg_attr(feature = "schemars", derive(JsonSchema))]
170pub enum KanidmRestorePhase {
171 #[default]
172 Pending,
173 Validating,
174 Quiescing,
175 SafetyBackup,
176 PreparingSource,
177 RestoringPrimary,
178 Verifying,
179 RebuildingReplicas,
180 Resuming,
181 Completed,
182 Failed,
183}
184
185#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
186#[cfg_attr(feature = "schemars", derive(JsonSchema))]
187#[serde(rename_all = "camelCase")]
188pub struct KanidmRestoreStatus {
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_generation: Option<i64>,
191 #[serde(skip_serializing_if = "Option::is_none")]
192 pub observed_target_uid: Option<String>,
193 #[serde(default)]
194 pub phase: KanidmRestorePhase,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub restore_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub verify_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub safety_backup_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub source_prep_job_name: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_ref: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_expected_backup_id: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_manifest_key: Option<String>,
209 #[serde(skip_serializing_if = "Option::is_none")]
210 pub safety_backup_payload_sha256: Option<String>,
211 #[serde(default)]
212 pub replicas_cleared: bool,
213 #[serde(default)]
214 pub certificates_cleared: bool,
215 #[serde(default)]
216 pub database_mutation_started: bool,
217 #[serde(default, skip_serializing_if = "Vec::is_empty")]
218 pub original_replicas: Vec<ReplicaCountEntry>,
219 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
220 pub phase_timestamps: std::collections::BTreeMap<String, String>,
221 #[serde(default, skip_serializing_if = "Vec::is_empty")]
222 #[cfg_attr(
223 feature = "schemars",
224 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
225 )]
226 pub conditions: Vec<Condition>,
227 #[serde(skip_serializing_if = "Option::is_none")]
228 pub message: Option<String>,
229}
230
231#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "schemars", derive(JsonSchema))]
233#[serde(rename_all = "camelCase")]
234pub struct ReplicaCountEntry {
235 pub group: String,
236 pub replicas: i32,
237}
238
239#[derive(Clone)]
240struct RestoreMetrics {
241 attempts: Counter<u64>,
242 outcomes: Counter<u64>,
243 duration_seconds: Histogram<f64>,
244 break_glass_total: Counter<u64>,
245 safety_backup_duration_seconds: Histogram<f64>,
246}
247
248impl RestoreMetrics {
249 fn new() -> Self {
250 let meter = global::meter("kaniop");
251 Self {
252 attempts: meter
253 .u64_counter("kaniop_restore_attempts_total")
254 .with_description("Number of Kanidm restore attempts started")
255 .build(),
256 outcomes: meter
257 .u64_counter("kaniop_restore_outcomes_total")
258 .with_description("Number of terminal Kanidm restore outcomes")
259 .build(),
260 duration_seconds: meter
261 .f64_histogram("kaniop_restore_duration_seconds")
262 .with_description("Kanidm restore duration from object creation to terminal phase")
263 .with_unit("s")
264 .build(),
265 break_glass_total: meter
266 .u64_counter("kaniop_restore_break_glass_total")
267 .with_description("Number of break-glass safety backup overrides")
268 .build(),
269 safety_backup_duration_seconds: meter
270 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
271 .with_description("Duration of pre-restore safety backup creation")
272 .with_unit("s")
273 .build(),
274 }
275 }
276}
277
278#[derive(Clone)]
279struct RestoreContext {
280 client: Client,
281 recorder: Recorder,
282 metrics: RestoreMetrics,
283}
284
285pub async fn run(client: Client) {
286 let api = Api::<KanidmRestore>::all(client.clone());
287 let recorder = Recorder::new(
288 client.clone(),
289 Reporter {
290 controller: CONTROLLER_ID.into(),
291 instance: None,
292 },
293 );
294 let ctx = Arc::new(RestoreContext {
295 client,
296 recorder,
297 metrics: RestoreMetrics::new(),
298 });
299 info!("starting {CONTROLLER_ID} controller");
300 Controller::new(api, watcher::Config::default().any_semantic())
301 .shutdown_on_signal()
302 .run(reconcile_restore, error_policy, ctx)
303 .for_each(|result| async move {
304 if let Err(error) = result {
305 error!(%error, "KanidmRestore reconciliation failed");
306 }
307 })
308 .await;
309}
310
311fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
312 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
313 Action::requeue(Duration::from_secs(5))
314}
315
316async fn reconcile_restore(
317 restore: Arc<KanidmRestore>,
318 ctx: Arc<RestoreContext>,
319) -> Result<Action> {
320 let namespace = restore
321 .namespace()
322 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
323 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
324 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
325 let ctx = ctx.clone();
326 async move {
327 match event {
328 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
329 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
330 }
331 }
332 })
333 .await
334 .map_err(|error| {
335 Error::FinalizerError(
336 "failed on KanidmRestore finalizer".to_string(),
337 Box::new(error),
338 )
339 })
340}
341
342async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
343 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
344 match phase {
345 KanidmRestorePhase::Pending => {
346 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
347 Ok(Action::requeue(REQUEUE))
348 }
349 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
350 Ok(target) => {
351 mark_restoring(&restore, &target, &ctx).await?;
352 let mut status = restore.status.clone().unwrap_or_default();
353 status.observed_target_uid = target.uid();
354 status.original_replicas = capture_original_replicas(&target);
355 status.phase = KanidmRestorePhase::Quiescing;
356 status.message = None;
357 patch_status(&restore, &ctx, status).await?;
358 Ok(Action::requeue(REQUEUE))
359 }
360 Err(error) => {
361 set_phase(
362 &restore,
363 &ctx,
364 KanidmRestorePhase::Failed,
365 Some(error.to_string()),
366 )
367 .await?;
368 Ok(Action::requeue(Duration::from_secs(300)))
369 }
370 },
371 KanidmRestorePhase::Quiescing => {
372 let target = get_target(&restore, &ctx).await?;
373 scale_all(&target, &ctx, 0).await?;
374 if target_pods_stopped(&target, &ctx).await?
375 && target_volumes_detached(&target, &ctx).await?
376 {
377 let next_phase = if requires_safety_backup(&restore) {
378 KanidmRestorePhase::SafetyBackup
379 } else {
380 record_break_glass(&restore, &ctx).await?;
381 KanidmRestorePhase::PreparingSource
382 };
383 if next_phase == KanidmRestorePhase::PreparingSource {
384 let refreshed = Api::<KanidmRestore>::namespaced(
385 ctx.client.clone(),
386 restore.namespace().as_deref().unwrap_or_default(),
387 )
388 .get(&restore.name_any())
389 .await
390 .map_err(|error| {
391 Error::kube_error(
392 "get",
393 "KanidmRestore",
394 restore.namespace().as_deref().unwrap_or_default(),
395 restore.name_any(),
396 error,
397 )
398 })?;
399 set_phase(&refreshed, &ctx, next_phase, None).await?;
400 } else {
401 set_phase(&restore, &ctx, next_phase, None).await?;
402 }
403 }
404 Ok(Action::requeue(REQUEUE))
405 }
406 KanidmRestorePhase::SafetyBackup => {
407 let target = get_target(&restore, &ctx).await?;
408 let name = safety_backup_job_name(&restore);
409 let expected_backup_id = compute_safety_backup_id(&restore);
410 let expected_manifest_key = compute_safety_manifest_key(&restore);
411 ensure_safety_backup_job(
412 &restore,
413 &target,
414 &ctx,
415 &name,
416 &expected_backup_id,
417 &expected_manifest_key,
418 )
419 .await?;
420 let mut status = restore.status.clone().unwrap_or_default();
421 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
422 match job_state(&restore, &ctx, &name).await? {
423 JobState::Complete => {
424 let result =
425 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
426 match result {
427 Ok(verified) => {
428 status.safety_backup_job_name = Some(name);
429 status.safety_backup_ref =
430 Some(format!("safety-{}", restore.name_any()));
431 status.safety_backup_manifest_key = Some(verified.manifest_key);
432 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
433 record_safety_backup_duration(&restore, &ctx).await;
434 status.phase = KanidmRestorePhase::PreparingSource;
435 status.message = None;
436 patch_status(&restore, &ctx, status).await?;
437 }
438 Err(error) => {
439 resume_before_mutation(&restore, &ctx).await?;
440 set_phase(
441 &restore,
442 &ctx,
443 KanidmRestorePhase::Failed,
444 Some(format!("safety backup result verification failed: {error}")),
445 )
446 .await?;
447 }
448 }
449 }
450 JobState::Failed => {
451 resume_before_mutation(&restore, &ctx).await?;
452 set_phase(
453 &restore,
454 &ctx,
455 KanidmRestorePhase::Failed,
456 Some(
457 "safety backup job failed; target restored to original state"
458 .to_string(),
459 ),
460 )
461 .await?;
462 }
463 JobState::Running => {
464 patch_status(&restore, &ctx, status).await?;
465 }
466 }
467 Ok(Action::requeue(REQUEUE))
468 }
469 KanidmRestorePhase::PreparingSource => {
470 if is_remote_source(&restore) {
471 let target = get_target(&restore, &ctx).await?;
472 let name = source_prep_job_name(&restore);
473 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
474 match job_state(&restore, &ctx, &name).await? {
475 JobState::Complete => {
476 let backup_id = restore
477 .spec
478 .source
479 .backup_ref
480 .as_ref()
481 .map(|b| b.name.clone())
482 .unwrap_or_default();
483 let backup = {
484 let ns = restore.namespace().unwrap();
485 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
486 .get(&backup_id)
487 .await
488 .map_err(|e| {
489 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
490 })?
491 };
492 let expected_backup_id = &backup.spec.backup_id;
493 let result =
494 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
495 .await;
496 match result {
497 Ok(_verified) => {
498 let mut status = restore.status.clone().unwrap_or_default();
499 status.source_prep_job_name = Some(name);
500 status.phase = KanidmRestorePhase::RestoringPrimary;
501 status.message = None;
502 patch_status(&restore, &ctx, status).await?;
503 }
504 Err(error) => {
505 resume_before_mutation(&restore, &ctx).await?;
506 set_phase(
507 &restore,
508 &ctx,
509 KanidmRestorePhase::Failed,
510 Some(format!(
511 "source preparation result verification failed: {error}"
512 )),
513 )
514 .await?;
515 }
516 }
517 }
518 JobState::Failed => {
519 resume_before_mutation(&restore, &ctx).await?;
520 set_phase(
521 &restore,
522 &ctx,
523 KanidmRestorePhase::Failed,
524 Some(
525 "source preparation job failed; target restored to original state"
526 .to_string(),
527 ),
528 )
529 .await?;
530 }
531 JobState::Running => {}
532 }
533 } else {
534 let target = get_target(&restore, &ctx).await?;
535 let name = source_check_job_name(&restore);
536 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
537 match job_state(&restore, &ctx, &name).await? {
538 JobState::Complete => {
539 let mut status = restore.status.clone().unwrap_or_default();
540 status.source_prep_job_name = Some(name);
541 status.phase = KanidmRestorePhase::RestoringPrimary;
542 status.message = None;
543 patch_status(&restore, &ctx, status).await?;
544 }
545 JobState::Failed => {
546 resume_before_mutation(&restore, &ctx).await?;
547 set_phase(
548 &restore,
549 &ctx,
550 KanidmRestorePhase::Failed,
551 Some(
552 "local backup file check failed; target restored to original state"
553 .to_string(),
554 ),
555 )
556 .await?;
557 }
558 JobState::Running => {}
559 }
560 }
561 Ok(Action::requeue(REQUEUE))
562 }
563 KanidmRestorePhase::RestoringPrimary => {
564 let target = get_target(&restore, &ctx).await?;
565 let status = restore.status.as_ref().cloned().unwrap_or_default();
566 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
567 return Err(Error::MissingData(
568 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
569 ));
570 }
571 if !status.database_mutation_started {
572 let mut status = status;
573 status.database_mutation_started = true;
574 status.message = Some(
575 "database mutation boundary persisted; starting primary restore".to_string(),
576 );
577 patch_status(&restore, &ctx, status).await?;
578 return Ok(Action::requeue(REQUEUE));
579 }
580 let name = restore_job_name(&restore);
581 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
582 match job_state(&restore, &ctx, &name).await? {
583 JobState::Complete => {
584 let mut status = restore.status.clone().unwrap_or_default();
585 status.restore_job_name = Some(name);
586 status.phase = KanidmRestorePhase::Verifying;
587 status.message = None;
588 patch_status(&restore, &ctx, status).await?;
589 }
590 JobState::Failed => {
591 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
592 }
593 JobState::Running => {}
594 }
595 Ok(Action::requeue(REQUEUE))
596 }
597 KanidmRestorePhase::Verifying => {
598 let target = get_target(&restore, &ctx).await?;
599 let name = verify_job_name(&restore);
600 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
601 match job_state(&restore, &ctx, &name).await? {
602 JobState::Complete => {
603 let mut status = restore.status.clone().unwrap_or_default();
604 status.verify_job_name = Some(name);
605 status.phase = KanidmRestorePhase::RebuildingReplicas;
606 status.message = None;
607 patch_status(&restore, &ctx, status).await?;
608 }
609 JobState::Failed => {
610 fail_after_mutation(&restore, &ctx, "database verification failed").await?
611 }
612 JobState::Running => {}
613 }
614 Ok(Action::requeue(REQUEUE))
615 }
616 KanidmRestorePhase::RebuildingReplicas => {
617 let target = get_target(&restore, &ctx).await?;
618 let mut status = restore.status.clone().unwrap_or_default();
619 if !status.replicas_cleared {
620 if delete_secondary_pvcs(&target, &ctx).await? {
621 status.replicas_cleared = true;
622 status.message = Some("secondary database state cleared".to_string());
623 patch_status(&restore, &ctx, status).await?;
624 }
625 return Ok(Action::requeue(REQUEUE));
626 }
627
628 scale_primary(&target, &ctx, 1).await?;
629 if !primary_ready(&target, &ctx).await? {
630 return Ok(Action::requeue(REQUEUE));
631 }
632 if !status.certificates_cleared {
633 delete_replica_cert_secrets(&target, &ctx).await?;
634 delete_admin_secret(&target, &ctx).await?;
635 status.certificates_cleared = true;
636 status.message = Some(
637 "replica certificates and admin secret cleared for regeneration".to_string(),
638 );
639 patch_status(&restore, &ctx, status).await?;
640 return Ok(Action::requeue(REQUEUE));
641 }
642 status.phase = KanidmRestorePhase::Resuming;
643 status.message = None;
644 patch_status(&restore, &ctx, status).await?;
645 Ok(Action::requeue(REQUEUE))
646 }
647 KanidmRestorePhase::Resuming => {
648 let target = get_target(&restore, &ctx).await?;
649 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
650 clear_restoring(&restore, &target, &ctx).await?;
651 Ok(Action::requeue(Duration::from_secs(3600)))
652 }
653 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
654 Ok(Action::requeue(Duration::from_secs(3600)))
655 }
656 }
657}
658
659async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
660 let status = restore.status.clone().unwrap_or_default();
661 if status.database_mutation_started
662 && status.phase != KanidmRestorePhase::Completed
663 && status.phase != KanidmRestorePhase::Failed
664 {
665 reconcile_apply(restore.clone(), ctx.clone()).await?;
666 return Ok(Action::requeue(REQUEUE));
667 }
668 if let Ok(target) = get_target(&restore, &ctx).await {
669 let owns_maintenance =
670 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
671 if owns_maintenance && !status.database_mutation_started {
672 scale_desired(&target, &ctx).await?;
673 }
674 clear_restoring(&restore, &target, &ctx).await?;
675 }
676 Ok(Action::await_change())
677}
678
679async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
680 let target = get_target(restore, ctx).await?;
681 let actual_uid = target
682 .uid()
683 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
684 if actual_uid != restore.spec.target_ref.uid {
685 return Err(Error::MissingData(format!(
686 "target UID mismatch: expected {}, got {}",
687 restore.spec.target_ref.uid, actual_uid
688 )));
689 }
690 validate_source(restore)?;
691 if is_remote_source(restore) {
692 validate_backup_ref(restore, &target, ctx).await?;
693 }
694 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
695 {
696 return Err(Error::MissingData(format!(
697 "restoreImage must be the target's pinned Kanidm image (target image is {})",
698 target.spec.image
699 )));
700 }
701 let storage = target
702 .spec
703 .storage
704 .as_ref()
705 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
706 if storage.empty_dir.is_some()
707 || storage.ephemeral.is_some()
708 || storage.volume_claim_template.is_none()
709 {
710 return Err(Error::MissingData(
711 "restore requires PVC-backed Kanidm storage".to_string(),
712 ));
713 }
714 let primaries = target
715 .spec
716 .replica_groups
717 .iter()
718 .filter(|rg| rg.primary_node)
719 .count();
720 if primaries != 1 {
721 return Err(Error::MissingData(
722 "backup/restore requires exactly one primary replica group".to_string(),
723 ));
724 }
725 let ns = restore.namespace().unwrap();
726 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
727 .list(&ListParams::default())
728 .await
729 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
730 if restores.items.iter().any(|other| {
731 other.name_any() != restore.name_any()
732 && other.spec.target_ref.name == restore.spec.target_ref.name
733 && !matches!(
734 other.status.as_ref().map(|s| s.phase),
735 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
736 )
737 }) {
738 return Err(Error::MissingData(
739 "another active restore targets this Kanidm".to_string(),
740 ));
741 }
742 validate_safety_backup_config(restore)?;
743 Ok(target)
744}
745
746fn safe_basename(name: &str) -> bool {
747 !name.is_empty()
748 && name != "."
749 && name != ".."
750 && !name.contains('/')
751 && !name.contains('\\')
752 && !name.contains('\0')
753}
754
755fn mutable_image(image: &str) -> bool {
756 image == "kanidm/server:latest"
757 || image.ends_with(":latest")
758 || (!image.contains('@')
759 && !image
760 .rsplit('/')
761 .next()
762 .is_some_and(|part| part.contains(':')))
763}
764
765async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
766 let ns = restore
767 .namespace()
768 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
769 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
770 .get(&restore.spec.target_ref.name)
771 .await
772 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
773}
774
775async fn mark_restoring(
776 restore: &KanidmRestore,
777 target: &Kanidm,
778 ctx: &RestoreContext,
779) -> Result<()> {
780 let ns = target.namespace().unwrap();
781 let uid = restore
782 .uid()
783 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
784 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
785 .patch(
786 &target.name_any(),
787 &PatchParams::default(),
788 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
789 )
790 .await
791 .map(|_| ())
792 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
793}
794
795async fn clear_restoring(
796 restore: &KanidmRestore,
797 target: &Kanidm,
798 ctx: &RestoreContext,
799) -> Result<()> {
800 let ns = target.namespace().unwrap();
801 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
802 return Ok(());
803 }
804 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
805 .patch(
806 &target.name_any(),
807 &PatchParams::default(),
808 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
809 )
810 .await
811 .map(|_| ())
812 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
813}
814
815async fn patch_status(
816 restore: &KanidmRestore,
817 ctx: &RestoreContext,
818 mut status: KanidmRestoreStatus,
819) -> Result<()> {
820 let ns = restore.namespace().unwrap();
821 let previous_phase = restore.status.as_ref().map(|current| current.phase);
822 let next_phase = status.phase;
823 status.observed_generation = restore.metadata.generation;
824 update_restore_conditions(&mut status, restore.metadata.generation);
825
826 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
827 .patch_status(
828 &restore.name_any(),
829 &PatchParams::default(),
830 &Patch::Merge(json!({"status": &status})),
831 )
832 .await
833 .map_err(|e| {
834 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
835 })?;
836
837 if previous_phase != Some(next_phase) {
838 record_restore_transition(
839 restore,
840 ctx,
841 previous_phase,
842 next_phase,
843 status.message.as_deref(),
844 )
845 .await;
846 }
847 Ok(())
848}
849
850fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
851 let previous = status.conditions.clone();
852 let phase = status.phase;
853 let phase_reason = format!("{phase:?}");
854 let message = status
855 .message
856 .clone()
857 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
858 let terminal_success = phase == KanidmRestorePhase::Completed;
859 let terminal_failure = phase == KanidmRestorePhase::Failed;
860 let progressing = !terminal_success && !terminal_failure;
861
862 let break_glass = previous
863 .iter()
864 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
865 .cloned();
866 status.conditions = vec![
867 restore_condition(
868 &previous,
869 CONDITION_PROGRESSING,
870 if progressing {
871 CONDITION_TRUE
872 } else {
873 CONDITION_FALSE
874 },
875 &phase_reason,
876 &message,
877 generation,
878 ),
879 restore_condition(
880 &previous,
881 CONDITION_READY,
882 if terminal_success {
883 CONDITION_TRUE
884 } else {
885 CONDITION_FALSE
886 },
887 if terminal_success {
888 "RestoreCompleted"
889 } else {
890 "RestoreNotCompleted"
891 },
892 if terminal_success {
893 "Kanidm restore completed successfully."
894 } else {
895 "Kanidm restore has not completed successfully."
896 },
897 generation,
898 ),
899 restore_condition(
900 &previous,
901 CONDITION_FAILED,
902 if terminal_failure {
903 CONDITION_TRUE
904 } else {
905 CONDITION_FALSE
906 },
907 if terminal_failure {
908 "RestoreFailed"
909 } else {
910 "NoRestoreFailure"
911 },
912 if terminal_failure {
913 status
914 .message
915 .as_deref()
916 .unwrap_or("Kanidm restore failed.")
917 } else {
918 "No terminal restore failure has been recorded."
919 },
920 generation,
921 ),
922 ];
923 if let Some(condition) = break_glass {
924 status.conditions.push(condition);
925 }
926}
927
928fn restore_condition(
929 previous: &[Condition],
930 condition_type: &str,
931 condition_status: &str,
932 reason: &str,
933 message: &str,
934 generation: Option<i64>,
935) -> Condition {
936 let last_transition_time = previous
937 .iter()
938 .find(|condition| {
939 condition.type_ == condition_type
940 && condition.status == condition_status
941 && condition.reason == reason
942 })
943 .map(|condition| condition.last_transition_time.clone())
944 .unwrap_or_else(|| Time(Timestamp::now()));
945 Condition {
946 type_: condition_type.to_string(),
947 status: condition_status.to_string(),
948 reason: reason.to_string(),
949 message: message.to_string(),
950 last_transition_time,
951 observed_generation: generation,
952 }
953}
954
955async fn record_restore_transition(
956 restore: &KanidmRestore,
957 ctx: &RestoreContext,
958 previous_phase: Option<KanidmRestorePhase>,
959 phase: KanidmRestorePhase,
960 message: Option<&str>,
961) {
962 if phase == KanidmRestorePhase::Validating {
963 ctx.metrics.attempts.add(1, &[]);
964 }
965
966 let result = match phase {
967 KanidmRestorePhase::Completed => Some("success"),
968 KanidmRestorePhase::Failed => Some("failure"),
969 _ => None,
970 };
971 if let Some(result) = result {
972 let attributes = [KeyValue::new("result", result)];
973 ctx.metrics.outcomes.add(1, &attributes);
974 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
975 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
976 ctx.metrics.duration_seconds.record(elapsed, &attributes);
977 }
978 }
979
980 let reason = if phase == KanidmRestorePhase::Failed {
981 "RestoreFailed"
982 } else {
983 "RestorePhaseChanged"
984 };
985 let note = message.map(str::to_string).or_else(|| {
986 Some(format!(
987 "Kanidm restore phase changed from {} to {phase:?}.",
988 previous_phase
989 .map(|previous| format!("{previous:?}"))
990 .unwrap_or_else(|| "None".to_string())
991 ))
992 });
993 if let Err(error) = ctx
994 .recorder
995 .publish(
996 &Event {
997 type_: if phase == KanidmRestorePhase::Failed {
998 EventType::Warning
999 } else {
1000 EventType::Normal
1001 },
1002 reason: reason.to_string(),
1003 note,
1004 action: "Restore".to_string(),
1005 secondary: None,
1006 },
1007 &restore.object_ref(&()),
1008 )
1009 .await
1010 {
1011 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1012 }
1013}
1014
1015async fn set_phase(
1016 restore: &KanidmRestore,
1017 ctx: &RestoreContext,
1018 phase: KanidmRestorePhase,
1019 message: Option<String>,
1020) -> Result<()> {
1021 let mut status = restore.status.clone().unwrap_or_default();
1022 status.phase = phase;
1023 status.message = message;
1024 patch_status(restore, ctx, status).await
1025}
1026
1027async fn fail_after_mutation(
1028 restore: &KanidmRestore,
1029 ctx: &RestoreContext,
1030 message: &str,
1031) -> Result<()> {
1032 set_phase(
1033 restore,
1034 ctx,
1035 KanidmRestorePhase::Failed,
1036 Some(message.to_string()),
1037 )
1038 .await
1039}
1040
1041fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1042 target
1043 .spec
1044 .replica_groups
1045 .iter()
1046 .find(|rg| rg.primary_node)
1047 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1048}
1049
1050async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1051 let ns = target.namespace().unwrap();
1052 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1053 for rg in &target.spec.replica_groups {
1054 let name = target.statefulset_name(&rg.name);
1055 api.patch(
1056 &name,
1057 &PatchParams::default(),
1058 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1059 )
1060 .await
1061 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1062 }
1063 Ok(())
1064}
1065
1066async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1067 let ns = target.namespace().unwrap();
1068 let rg = primary_group(target)?;
1069 let name = target.statefulset_name(&rg.name);
1070 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1071 .patch(
1072 &name,
1073 &PatchParams::default(),
1074 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1075 )
1076 .await
1077 .map(|_| ())
1078 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1079}
1080
1081async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1082 let ns = target.namespace().unwrap();
1083 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1084 for rg in &target.spec.replica_groups {
1085 let name = target.statefulset_name(&rg.name);
1086 api.patch(
1087 &name,
1088 &PatchParams::default(),
1089 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1090 )
1091 .await
1092 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1093 }
1094 Ok(())
1095}
1096
1097async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1098 let ns = target.namespace().unwrap();
1099 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1100 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1101 .await
1102 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1103 Ok(pods.items.is_empty())
1104}
1105
1106async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1107 let ns = target.namespace().unwrap();
1108 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1109 let mut pv_names = BTreeSet::new();
1110 let primary_name = primary_pvc_name(target)?;
1111 let primary_pvc = pvc_api
1112 .get(&primary_name)
1113 .await
1114 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1115 let primary_pv = primary_pvc
1116 .spec
1117 .as_ref()
1118 .and_then(|spec| spec.volume_name.clone())
1119 .ok_or_else(|| {
1120 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1121 })?;
1122 pv_names.insert(primary_pv);
1123
1124 for rg in &target.spec.replica_groups {
1125 let sts = target.statefulset_name(&rg.name);
1126 for ordinal in 0..rg.replicas {
1127 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1128 if name == primary_name {
1129 continue;
1130 }
1131 if let Some(pvc) = pvc_api
1132 .get_opt(&name)
1133 .await
1134 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1135 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1136 {
1137 pv_names.insert(pv_name);
1138 }
1139 }
1140 }
1141
1142 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1143 .list(&ListParams::default())
1144 .await
1145 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1146 Ok(!attachments.items.iter().any(|attachment| {
1147 attachment
1148 .spec
1149 .source
1150 .persistent_volume_name
1151 .as_ref()
1152 .is_some_and(|pv_name| pv_names.contains(pv_name))
1153 }))
1154}
1155
1156async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1157 let ns = target.namespace().unwrap();
1158 let name = target.statefulset_name(&primary_group(target)?.name);
1159 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1160 .get(&name)
1161 .await
1162 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1163 Ok(sts
1164 .status
1165 .as_ref()
1166 .and_then(|s| s.ready_replicas)
1167 .unwrap_or(0)
1168 >= 1)
1169}
1170
1171fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1172 let rg = primary_group(target)?;
1173 Ok(format!(
1174 "{DATA_VOLUME}-{}-0",
1175 target.statefulset_name(&rg.name)
1176 ))
1177}
1178
1179async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1180 let ns = target.namespace().unwrap();
1181 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1182 let mut all_absent = true;
1183 for rg in &target.spec.replica_groups {
1184 let sts = target.statefulset_name(&rg.name);
1185 for ordinal in 0..rg.replicas {
1186 if rg.primary_node && ordinal == 0 {
1187 continue;
1188 }
1189 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1190 match api
1191 .get_opt(&name)
1192 .await
1193 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1194 {
1195 None => {}
1196 Some(pvc) => {
1197 all_absent = false;
1198 if pvc.metadata.deletion_timestamp.is_none() {
1199 match api.delete(&name, &DeleteParams::default()).await {
1200 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1201 Err(kube::Error::Api(status)) if status.code == 404 => {}
1202 Err(error) => {
1203 return Err(Error::kube_error(
1204 "delete",
1205 "PersistentVolumeClaim",
1206 &ns,
1207 &name,
1208 error,
1209 ));
1210 }
1211 }
1212 }
1213 }
1214 }
1215 }
1216 }
1217 Ok(all_absent)
1218}
1219
1220async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1221 let ns = target.namespace().unwrap();
1222 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1223 let label_selector = format!(
1224 "{}={},{}={}",
1225 CLUSTER_LABEL,
1226 target.name_any(),
1227 SECRET_TYPE_LABEL,
1228 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1229 );
1230 let secrets = api
1231 .list(&ListParams::default().labels(&label_selector))
1232 .await
1233 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1234 let mut all_absent = secrets.items.is_empty();
1235 for secret in secrets.items {
1236 let name = secret.name_any();
1237 if secret.metadata.deletion_timestamp.is_none() {
1238 match api.delete(&name, &DeleteParams::default()).await {
1239 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1240 Err(kube::Error::Api(status)) if status.code == 404 => {}
1241 Err(error) => {
1242 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1243 }
1244 }
1245 all_absent = false;
1246 }
1247 }
1248 Ok(all_absent)
1249}
1250
1251async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1252 let ns = target.namespace().unwrap();
1253 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1254 let name = format!("{}-admin-passwords", target.name_any());
1255 match api
1256 .get_opt(&name)
1257 .await
1258 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1259 {
1260 None => Ok(true),
1261 Some(secret) => {
1262 if secret.metadata.deletion_timestamp.is_some() {
1263 return Ok(false);
1264 }
1265 match api.delete(&name, &DeleteParams::default()).await {
1266 Ok(_) => {
1267 debug!(secret = %name, "deleting stale admin secret");
1268 Ok(false)
1269 }
1270 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1271 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1272 }
1273 }
1274 }
1275}
1276
1277fn restore_job_name(restore: &KanidmRestore) -> String {
1278 format!("{}-restore", restore.name_any())
1279}
1280fn verify_job_name(restore: &KanidmRestore) -> String {
1281 format!("{}-verify", restore.name_any())
1282}
1283
1284fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1285 let origin = target
1286 .spec
1287 .origin
1288 .clone()
1289 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1290 vec![
1291 EnvVar {
1292 name: "KANIDM_DB_PATH".to_string(),
1293 value: Some("/data/kanidm.db".to_string()),
1294 ..Default::default()
1295 },
1296 EnvVar {
1297 name: "KANIDM_DOMAIN".to_string(),
1298 value: Some(target.spec.domain.clone()),
1299 ..Default::default()
1300 },
1301 EnvVar {
1302 name: "KANIDM_ORIGIN".to_string(),
1303 value: Some(origin),
1304 ..Default::default()
1305 },
1306 EnvVar {
1307 name: "KANIDM_TLS_CHAIN".to_string(),
1308 value: Some(format!("{TLS_PATH}/tls.crt")),
1309 ..Default::default()
1310 },
1311 EnvVar {
1312 name: "KANIDM_TLS_KEY".to_string(),
1313 value: Some(format!("{TLS_PATH}/tls.key")),
1314 ..Default::default()
1315 },
1316 ]
1317}
1318
1319async fn ensure_database_job(
1320 restore: &KanidmRestore,
1321 target: &Kanidm,
1322 ctx: &RestoreContext,
1323 name: &str,
1324 verify: bool,
1325) -> Result<()> {
1326 let ns = restore.namespace().unwrap();
1327 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1328 if api
1329 .get_opt(name)
1330 .await
1331 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1332 .is_some()
1333 {
1334 return Ok(());
1335 }
1336 let mut command = vec![
1337 "kanidmd".to_string(),
1338 "database".to_string(),
1339 if verify { "backup" } else { "restore" }.to_string(),
1340 ];
1341 if verify {
1342 command.push("/verify/verification.json.gz".to_string());
1343 } else {
1344 if is_remote_source(restore) {
1345 command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1346 } else {
1347 let file_name = restore
1348 .spec
1349 .source
1350 .local
1351 .as_ref()
1352 .map(|l| l.file_name.clone())
1353 .unwrap_or_default();
1354 command.push(format!("{BACKUP_PATH}/{file_name}"));
1355 }
1356 }
1357 let job = Job {
1358 metadata: ObjectMeta {
1359 name: Some(name.to_string()),
1360 namespace: Some(ns.clone()),
1361 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1362 ..Default::default()
1363 },
1364 spec: Some(JobSpec {
1365 backoff_limit: Some(0),
1366 template: PodTemplateSpec {
1367 metadata: Some(ObjectMeta::default()),
1368 spec: Some(PodSpec {
1369 automount_service_account_token: Some(false),
1370 restart_policy: Some("Never".to_string()),
1371 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1372 seccomp_profile: Some(SeccompProfile {
1373 type_: "RuntimeDefault".to_string(),
1374 ..Default::default()
1375 }),
1376 ..Default::default()
1377 }),
1378 containers: vec![Container {
1379 name: if verify { "verify" } else { "restore" }.to_string(),
1380 image: Some(restore.spec.restore_image.clone()),
1381 command: Some(command),
1382 env: Some(build_kanidm_db_env_vars(target)),
1383 security_context: Some(kanidm_job_security_context()),
1384 resources: Some(job_resource_requirements()),
1385 volume_mounts: Some(vec![
1386 VolumeMount {
1387 name: DATA_VOLUME.to_string(),
1388 mount_path: DATA_PATH.to_string(),
1389 ..Default::default()
1390 },
1391 VolumeMount {
1392 name: TLS_VOLUME.to_string(),
1393 mount_path: TLS_PATH.to_string(),
1394 read_only: Some(true),
1395 ..Default::default()
1396 },
1397 VolumeMount {
1398 name: "verification-output".to_string(),
1399 mount_path: "/verify".to_string(),
1400 ..Default::default()
1401 },
1402 ]),
1403 ..Default::default()
1404 }],
1405 volumes: Some(vec![
1406 Volume {
1407 name: DATA_VOLUME.to_string(),
1408 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1409 claim_name: primary_pvc_name(target)?,
1410 read_only: Some(false),
1411 }),
1412 ..Default::default()
1413 },
1414 Volume {
1415 name: TLS_VOLUME.to_string(),
1416 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1417 secret_name: Some(target.effective_tls_secret_name()),
1418 ..Default::default()
1419 }),
1420 ..Default::default()
1421 },
1422 Volume {
1423 name: "verification-output".to_string(),
1424 empty_dir: Some(Default::default()),
1425 ..Default::default()
1426 },
1427 ]),
1428 ..Default::default()
1429 }),
1430 },
1431 ..Default::default()
1432 }),
1433 ..Default::default()
1434 };
1435 api.create(&PostParams::default(), &job)
1436 .await
1437 .map(|_| ())
1438 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1439}
1440
1441enum JobState {
1442 Running,
1443 Complete,
1444 Failed,
1445}
1446
1447async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1448 let ns = restore.namespace().unwrap();
1449 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1450 .get(name)
1451 .await
1452 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1453 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1454 return Ok(JobState::Failed);
1455 }
1456 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1457 return Ok(JobState::Complete);
1458 }
1459 Ok(JobState::Running)
1460}
1461
1462fn is_remote_source(restore: &KanidmRestore) -> bool {
1463 restore.spec.source.backup_ref.is_some()
1464}
1465
1466fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1467 let safety = restore.spec.safety_backup.as_ref();
1468 let skip = safety.is_some_and(|s| s.skip);
1469 !skip
1470}
1471
1472fn validate_source(restore: &KanidmRestore) -> Result<()> {
1473 let has_local = restore.spec.source.local.is_some();
1474 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1475 if has_local && has_backup_ref {
1476 return Err(Error::MissingData(
1477 "source.local and source.backupRef are mutually exclusive".to_string(),
1478 ));
1479 }
1480 if !has_local && !has_backup_ref {
1481 return Err(Error::MissingData(
1482 "source must specify either local or backupRef".to_string(),
1483 ));
1484 }
1485 if let Some(local) = &restore.spec.source.local {
1486 if !safe_basename(&local.file_name) {
1487 return Err(Error::MissingData(
1488 "restore source fileName must be a safe basename".to_string(),
1489 ));
1490 }
1491 }
1492 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1493 if backup_ref.name.is_empty() {
1494 return Err(Error::MissingData(
1495 "source.backupRef.name must not be empty".to_string(),
1496 ));
1497 }
1498 }
1499 Ok(())
1500}
1501
1502fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1503 let safety = restore.spec.safety_backup.as_ref();
1504 let skip = safety.is_some_and(|s| s.skip);
1505 if skip {
1506 let annotations = restore.metadata.annotations.as_ref();
1507 let reason = annotations
1508 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1509 .map(|s| s.trim())
1510 .unwrap_or("");
1511 let approver = annotations
1512 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1513 .map(|s| s.trim())
1514 .unwrap_or("");
1515 if reason.is_empty() {
1516 return Err(Error::MissingData(format!(
1517 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1518 )));
1519 }
1520 if approver.is_empty() {
1521 return Err(Error::MissingData(format!(
1522 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1523 )));
1524 }
1525 }
1526 if is_remote_source(restore) && !skip {
1527 let has_repo = safety
1528 .and_then(|s| s.repository_ref.as_ref())
1529 .is_some_and(|r| !r.name.is_empty());
1530 if !has_repo {
1531 return Err(Error::MissingData(
1532 "remote restore requires safetyBackup.repositoryRef".to_string(),
1533 ));
1534 }
1535 }
1536 Ok(())
1537}
1538
1539fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1540 target
1541 .spec
1542 .replica_groups
1543 .iter()
1544 .map(|rg| ReplicaCountEntry {
1545 group: rg.name.clone(),
1546 replicas: rg.replicas,
1547 })
1548 .collect()
1549}
1550
1551async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1552 if let Ok(target) = get_target(restore, ctx).await {
1553 let owns_maintenance =
1554 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1555 if owns_maintenance {
1556 scale_desired(&target, ctx).await?;
1557 clear_restoring(restore, &target, ctx).await?;
1558 }
1559 }
1560 Ok(())
1561}
1562
1563async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1564 ctx.metrics.break_glass_total.add(1, &[]);
1565 let reason = restore
1566 .metadata
1567 .annotations
1568 .as_ref()
1569 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1570 .cloned()
1571 .unwrap_or_default();
1572 let approver = restore
1573 .metadata
1574 .annotations
1575 .as_ref()
1576 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1577 .cloned()
1578 .unwrap_or_default();
1579 let mut status = restore.status.clone().unwrap_or_default();
1580 let break_glass_condition = restore_condition(
1581 &status.conditions,
1582 CONDITION_BREAK_GLASS,
1583 CONDITION_TRUE,
1584 "BreakGlassOverride",
1585 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1586 restore.metadata.generation,
1587 );
1588 status
1589 .conditions
1590 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1591 status.conditions.push(break_glass_condition);
1592 patch_status(restore, ctx, status).await?;
1593 if let Err(error) = ctx
1594 .recorder
1595 .publish(
1596 &Event {
1597 type_: EventType::Warning,
1598 reason: "SafetyBackupSkipped".to_string(),
1599 note: Some(format!(
1600 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1601 )),
1602 action: "Restore".to_string(),
1603 secondary: None,
1604 },
1605 &restore.object_ref(&()),
1606 )
1607 .await
1608 {
1609 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1610 }
1611 warn!(
1612 restore = %restore.name_any(),
1613 reason = %reason,
1614 approver = %approver,
1615 "break-glass safety backup override activated"
1616 );
1617 Ok(())
1618}
1619
1620async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1621 let status = restore.status.as_ref().cloned().unwrap_or_default();
1622 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1623 if let Ok(start_ts) = start.parse::<Timestamp>() {
1624 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1625 ctx.metrics
1626 .safety_backup_duration_seconds
1627 .record(elapsed, &[]);
1628 }
1629 }
1630}
1631
1632fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1633 format!("{}-safety-backup", restore.name_any())
1634}
1635
1636fn source_prep_job_name(restore: &KanidmRestore) -> String {
1637 format!("{}-source-prep", restore.name_any())
1638}
1639
1640fn source_check_job_name(restore: &KanidmRestore) -> String {
1641 format!("{}-source-check", restore.name_any())
1642}
1643
1644async fn ensure_source_check_job(
1645 restore: &KanidmRestore,
1646 target: &Kanidm,
1647 ctx: &RestoreContext,
1648 name: &str,
1649) -> Result<()> {
1650 let ns = restore.namespace().unwrap();
1651 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1652 if api
1653 .get_opt(name)
1654 .await
1655 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1656 .is_some()
1657 {
1658 return Ok(());
1659 }
1660 let file_name = restore
1661 .spec
1662 .source
1663 .local
1664 .as_ref()
1665 .map(|l| l.file_name.clone())
1666 .unwrap_or_default();
1667 let operation_doc = serde_json::json!({
1668 "apiVersion": "backup.kaniop.rs/v1alpha1",
1669 "kind": "OperationDocument",
1670 "operation": "check",
1671 "path": format!("{BACKUP_PATH}/{file_name}"),
1672 "resultPath": "/run/kaniop-result/result.json",
1673 "format": "kanidmJsonGzip",
1674 })
1675 .to_string();
1676 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1677 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1678 let data_mover_image = data_mover_image();
1679 let job = Job {
1680 metadata: ObjectMeta {
1681 name: Some(name.to_string()),
1682 namespace: Some(ns.clone()),
1683 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1684 ..Default::default()
1685 },
1686 spec: Some(JobSpec {
1687 backoff_limit: Some(0),
1688 template: PodTemplateSpec {
1689 metadata: Some(ObjectMeta::default()),
1690 spec: Some(PodSpec {
1691 automount_service_account_token: Some(false),
1692 restart_policy: Some("Never".to_string()),
1693 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1694 run_as_non_root: Some(true),
1695 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1696 type_: "RuntimeDefault".to_string(),
1697 ..Default::default()
1698 }),
1699 ..Default::default()
1700 }),
1701 containers: vec![Container {
1702 name: "source-check".to_string(),
1703 image: Some(data_mover_image),
1704 command: Some(vec![
1705 "/bin/kaniop-data-mover".to_string(),
1706 "check".to_string(),
1707 "--operation-doc".to_string(),
1708 "/run/kaniop/operation.json".to_string(),
1709 ]),
1710 security_context: Some(hardened_security_context()),
1711 resources: Some(job_resource_requirements()),
1712 env: Some(vec![EnvVar {
1713 name: "RUST_LOG".to_string(),
1714 value: Some("info".to_string()),
1715 ..Default::default()
1716 }]),
1717 volume_mounts: Some(vec![
1718 VolumeMount {
1719 name: DATA_VOLUME.to_string(),
1720 mount_path: DATA_PATH.to_string(),
1721 read_only: Some(true),
1722 ..Default::default()
1723 },
1724 VolumeMount {
1725 name: "operation".to_string(),
1726 mount_path: "/run/kaniop".to_string(),
1727 read_only: Some(true),
1728 ..Default::default()
1729 },
1730 VolumeMount {
1731 name: "result".to_string(),
1732 mount_path: "/run/kaniop-result".to_string(),
1733 ..Default::default()
1734 },
1735 ]),
1736 ..Default::default()
1737 }],
1738 volumes: Some(vec![
1739 Volume {
1740 name: DATA_VOLUME.to_string(),
1741 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1742 claim_name: primary_pvc_name(target)?,
1743 read_only: Some(true),
1744 }),
1745 ..Default::default()
1746 },
1747 Volume {
1748 name: "operation".to_string(),
1749 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1750 name: operation_cm_name,
1751 ..Default::default()
1752 }),
1753 ..Default::default()
1754 },
1755 Volume {
1756 name: "result".to_string(),
1757 empty_dir: Some(Default::default()),
1758 ..Default::default()
1759 },
1760 ]),
1761 ..Default::default()
1762 }),
1763 },
1764 ..Default::default()
1765 }),
1766 ..Default::default()
1767 };
1768 api.create(&PostParams::default(), &job)
1769 .await
1770 .map(|_| ())
1771 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1772}
1773
1774const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1775
1776fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1777 let ns = restore.namespace().unwrap_or_default();
1778 let name = restore.name_any();
1779 let namespace_uuid =
1780 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1781 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1782 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1783}
1784
1785fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1786 let ns = restore.namespace().unwrap_or_default();
1787 let kanidm_uid = &restore.spec.target_ref.uid;
1788 let backup_id = compute_safety_backup_id(restore);
1789 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1790}
1791
1792struct VerifiedSafetyBackup {
1793 manifest_key: String,
1794 payload_sha256: String,
1795}
1796
1797async fn read_safety_backup_result(
1798 restore: &KanidmRestore,
1799 ctx: &RestoreContext,
1800 job_name: &str,
1801 expected_backup_id: &str,
1802) -> Result<VerifiedSafetyBackup> {
1803 let ns = restore.namespace().unwrap();
1804 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1805 let pod_list = pods
1806 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1807 .await
1808 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1809 let pod = pod_list.items.first().ok_or_else(|| {
1810 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1811 })?;
1812 let pod_name = pod.name_any();
1813 let container_status = pod
1814 .status
1815 .as_ref()
1816 .and_then(|s| s.container_statuses.as_ref())
1817 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1818 .ok_or_else(|| {
1819 Error::MissingData(format!(
1820 "safety-upload container status not found in pod {pod_name}"
1821 ))
1822 })?;
1823 let termination_message = container_status
1824 .state
1825 .as_ref()
1826 .and_then(|state| state.terminated.as_ref())
1827 .and_then(|t| t.message.as_ref())
1828 .ok_or_else(|| {
1829 Error::MissingData(format!(
1830 "termination message not found in pod {pod_name}; result transport failed"
1831 ))
1832 })?;
1833 if termination_message.is_empty() {
1834 return Err(Error::MissingData(format!(
1835 "termination message is empty in pod {pod_name}; result document was not written"
1836 )));
1837 }
1838 let result_doc = parse_result_document(termination_message).map_err(|e| {
1839 Error::ParseError(format!(
1840 "failed to parse result document from pod {pod_name}: {e}"
1841 ))
1842 })?;
1843 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1844 return Err(Error::ParseError(format!(
1845 "unsupported result document version: {}",
1846 result_doc.api_version
1847 )));
1848 }
1849 if result_doc.kind != "ResultDocument" {
1850 return Err(Error::ParseError(format!(
1851 "invalid result document kind: {}",
1852 result_doc.kind
1853 )));
1854 }
1855 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1856 return Err(Error::ParseError(format!(
1857 "result document operation mismatch: expected '{}', got '{}'",
1858 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1859 )));
1860 }
1861 if !result_doc.success {
1862 return Err(Error::ParseError(format!(
1863 "safety backup reported failure with exit code {:?}",
1864 result_doc.exit_code
1865 )));
1866 }
1867 if result_doc.exit_code != ExitCode::Success {
1868 return Err(Error::ParseError(format!(
1869 "safety backup exit code is not Success: {:?}",
1870 result_doc.exit_code
1871 )));
1872 }
1873 let doc_backup_id = result_doc
1874 .backup_id
1875 .as_deref()
1876 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1877 if doc_backup_id != expected_backup_id {
1878 return Err(Error::ParseError(format!(
1879 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1880 )));
1881 }
1882 let manifest_key = result_doc
1883 .manifest_key
1884 .clone()
1885 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1886 if manifest_key.is_empty() {
1887 return Err(Error::ParseError(
1888 "result document manifestKey is empty".to_string(),
1889 ));
1890 }
1891 let payload_sha256 = result_doc
1892 .payload_sha256
1893 .clone()
1894 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1895 if payload_sha256.is_empty() {
1896 return Err(Error::ParseError(
1897 "result document payloadSha256 is empty".to_string(),
1898 ));
1899 }
1900 info!(
1901 restore = %restore.name_any(),
1902 backup_id = %expected_backup_id,
1903 manifest_key = %manifest_key,
1904 "safety backup result verified"
1905 );
1906 Ok(VerifiedSafetyBackup {
1907 manifest_key,
1908 payload_sha256,
1909 })
1910}
1911
1912struct VerifiedSourcePrepResult {
1913 #[allow(dead_code)]
1914 manifest_key: String,
1915 #[allow(dead_code)]
1916 payload_sha256: String,
1917}
1918
1919async fn read_source_prep_result(
1920 restore: &KanidmRestore,
1921 ctx: &RestoreContext,
1922 job_name: &str,
1923 expected_backup_id: &str,
1924) -> Result<VerifiedSourcePrepResult> {
1925 let ns = restore.namespace().unwrap();
1926 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1927 let pod_list = pods
1928 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1929 .await
1930 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1931 let pod = pod_list.items.first().ok_or_else(|| {
1932 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1933 })?;
1934 let pod_name = pod.name_any();
1935 let container_status = pod
1936 .status
1937 .as_ref()
1938 .and_then(|s| s.container_statuses.as_ref())
1939 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1940 .ok_or_else(|| {
1941 Error::MissingData(format!(
1942 "source-prep container status not found in pod {pod_name}"
1943 ))
1944 })?;
1945 let termination_message = container_status
1946 .state
1947 .as_ref()
1948 .and_then(|state| state.terminated.as_ref())
1949 .and_then(|t| t.message.as_ref())
1950 .ok_or_else(|| {
1951 Error::MissingData(format!(
1952 "termination message not found in pod {pod_name}; result transport failed"
1953 ))
1954 })?;
1955 if termination_message.is_empty() {
1956 return Err(Error::MissingData(format!(
1957 "termination message is empty in pod {pod_name}; result document was not written"
1958 )));
1959 }
1960 let result_doc = parse_result_document(termination_message).map_err(|e| {
1961 Error::ParseError(format!(
1962 "failed to parse result document from pod {pod_name}: {e}"
1963 ))
1964 })?;
1965 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1966 return Err(Error::ParseError(format!(
1967 "unsupported result document version: {}",
1968 result_doc.api_version
1969 )));
1970 }
1971 if result_doc.kind != "ResultDocument" {
1972 return Err(Error::ParseError(format!(
1973 "invalid result document kind: {}",
1974 result_doc.kind
1975 )));
1976 }
1977 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1978 return Err(Error::ParseError(format!(
1979 "result document operation mismatch: expected '{}', got '{}'",
1980 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1981 )));
1982 }
1983 if !result_doc.success {
1984 return Err(Error::ParseError(format!(
1985 "source prep reported failure with exit code {:?}",
1986 result_doc.exit_code
1987 )));
1988 }
1989 if result_doc.exit_code != ExitCode::Success {
1990 return Err(Error::ParseError(format!(
1991 "source prep exit code is not Success: {:?}",
1992 result_doc.exit_code
1993 )));
1994 }
1995 let doc_backup_id = result_doc
1996 .backup_id
1997 .as_deref()
1998 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1999 if doc_backup_id != expected_backup_id {
2000 return Err(Error::ParseError(format!(
2001 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2002 )));
2003 }
2004 let manifest_key = result_doc
2005 .manifest_key
2006 .clone()
2007 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2008 if manifest_key.is_empty() {
2009 return Err(Error::ParseError(
2010 "result document manifestKey is empty".to_string(),
2011 ));
2012 }
2013 let payload_sha256 = result_doc
2014 .payload_sha256
2015 .clone()
2016 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2017 if payload_sha256.is_empty() {
2018 return Err(Error::ParseError(
2019 "result document payloadSha256 is empty".to_string(),
2020 ));
2021 }
2022 info!(
2023 restore = %restore.name_any(),
2024 backup_id = %expected_backup_id,
2025 manifest_key = %manifest_key,
2026 payload_sha256 = %payload_sha256,
2027 "source prep result verified"
2028 );
2029 Ok(VerifiedSourcePrepResult {
2030 manifest_key,
2031 payload_sha256,
2032 })
2033}
2034
2035fn hardened_security_context() -> SecurityContext {
2036 SecurityContext {
2037 allow_privilege_escalation: Some(false),
2038 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2039 drop: Some(vec!["ALL".to_string()]),
2040 ..Default::default()
2041 }),
2042 read_only_root_filesystem: Some(true),
2043 run_as_non_root: Some(true),
2044 seccomp_profile: Some(SeccompProfile {
2045 type_: "RuntimeDefault".to_string(),
2046 ..Default::default()
2047 }),
2048 ..Default::default()
2049 }
2050}
2051
2052fn kanidm_job_security_context() -> SecurityContext {
2053 SecurityContext {
2054 allow_privilege_escalation: Some(false),
2055 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2056 drop: Some(vec!["ALL".to_string()]),
2057 ..Default::default()
2058 }),
2059 seccomp_profile: Some(SeccompProfile {
2060 type_: "RuntimeDefault".to_string(),
2061 ..Default::default()
2062 }),
2063 ..Default::default()
2064 }
2065}
2066
2067fn job_resource_requirements() -> ResourceRequirements {
2068 ResourceRequirements {
2069 requests: Some(std::collections::BTreeMap::from([
2070 ("cpu".to_string(), Quantity("100m".to_string())),
2071 ("memory".to_string(), Quantity("128Mi".to_string())),
2072 ])),
2073 limits: Some(std::collections::BTreeMap::from([
2074 ("cpu".to_string(), Quantity("2".to_string())),
2075 ("memory".to_string(), Quantity("2Gi".to_string())),
2076 ])),
2077 ..Default::default()
2078 }
2079}
2080
2081async fn ensure_safety_backup_job(
2082 restore: &KanidmRestore,
2083 target: &Kanidm,
2084 ctx: &RestoreContext,
2085 name: &str,
2086 backup_id: &str,
2087 manifest_key: &str,
2088) -> Result<()> {
2089 let ns = restore.namespace().unwrap();
2090 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2091 if api
2092 .get_opt(name)
2093 .await
2094 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2095 .is_some()
2096 {
2097 return Ok(());
2098 }
2099 let safety = restore
2100 .spec
2101 .safety_backup
2102 .as_ref()
2103 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2104 let repo_ref = safety
2105 .repository_ref
2106 .as_ref()
2107 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2108 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2109 .get(&repo_ref.name)
2110 .await
2111 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2112 let _namespace_uid = &ns;
2113 let _kanidm_uid = &restore.spec.target_ref.uid;
2114 let endpoint = repo
2115 .spec
2116 .s3
2117 .endpoint
2118 .as_deref()
2119 .unwrap_or("https://s3.amazonaws.com");
2120 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2121 let operation_doc = build_safety_upload_operation_doc(
2122 restore,
2123 target,
2124 backup_id,
2125 manifest_key,
2126 &repo.spec.s3.bucket,
2127 &repo.spec.s3.prefix,
2128 endpoint,
2129 region,
2130 repo.spec.s3.force_path_style,
2131 repo.spec.s3.insecure,
2132 repo.spec
2133 .s3
2134 .ca_bundle_ref
2135 .as_deref()
2136 .map(|_| ca_bundle_path())
2137 .as_deref(),
2138 repo.spec.encryption.as_ref(),
2139 )?;
2140 let operation_cm_name = format!("{name}-op");
2141 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2142 let auth_method = &repo.spec.authentication.writer;
2143 let data_mover_image = data_mover_image();
2144 let init_command = vec![
2145 "kanidmd".to_string(),
2146 "database".to_string(),
2147 "backup".to_string(),
2148 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2149 ];
2150 let upload_script = format!(
2151 r#"set -eu
2152/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2153RESULT_FILE="/run/kaniop-result/result.json"
2154TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2155if [ ! -f "$RESULT_FILE" ]; then
2156 echo "ERROR: result document not found at $RESULT_FILE"
2157 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2158 exit 1
2159fi
2160if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2161 echo "ERROR: safety backup upload reported failure"
2162 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2163 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2164 exit 1
2165fi
2166dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2167echo "safety backup upload completed successfully"
2168"#
2169 );
2170 let job = Job {
2171 metadata: ObjectMeta {
2172 name: Some(name.to_string()),
2173 namespace: Some(ns.clone()),
2174 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2175 ..Default::default()
2176 },
2177 spec: Some(JobSpec {
2178 backoff_limit: Some(0),
2179 template: PodTemplateSpec {
2180 metadata: Some(ObjectMeta::default()),
2181 spec: Some(PodSpec {
2182 automount_service_account_token: Some(false),
2183 restart_policy: Some("Never".to_string()),
2184 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2185 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2186 type_: "RuntimeDefault".to_string(),
2187 ..Default::default()
2188 }),
2189 ..Default::default()
2190 }),
2191 init_containers: Some(vec![Container {
2192 name: "safety-backup".to_string(),
2193 image: Some(restore.spec.restore_image.clone()),
2194 command: Some(init_command),
2195 security_context: Some(SecurityContext {
2196 allow_privilege_escalation: Some(false),
2197 capabilities: Some(Capabilities {
2198 drop: Some(vec!["ALL".to_string()]),
2199 ..Default::default()
2200 }),
2201 read_only_root_filesystem: Some(true),
2202 seccomp_profile: Some(SeccompProfile {
2203 type_: "RuntimeDefault".to_string(),
2204 ..Default::default()
2205 }),
2206 ..Default::default()
2207 }),
2208 resources: Some(job_resource_requirements()),
2209 env: Some(build_kanidm_db_env_vars(target)),
2210 volume_mounts: Some(vec![
2211 VolumeMount {
2212 name: DATA_VOLUME.to_string(),
2213 mount_path: DATA_PATH.to_string(),
2214 read_only: Some(false),
2215 ..Default::default()
2216 },
2217 VolumeMount {
2218 name: TLS_VOLUME.to_string(),
2219 mount_path: TLS_PATH.to_string(),
2220 read_only: Some(true),
2221 ..Default::default()
2222 },
2223 VolumeMount {
2224 name: SHARED_VOLUME.to_string(),
2225 mount_path: SHARED_VOL_PATH.to_string(),
2226 ..Default::default()
2227 },
2228 ]),
2229 ..Default::default()
2230 }]),
2231 containers: vec![Container {
2232 name: "safety-upload".to_string(),
2233 image: Some(data_mover_image),
2234 command: Some(vec!["/bin/sh".to_string()]),
2235 args: Some(vec!["-c".to_string(), upload_script]),
2236 security_context: Some(hardened_security_context()),
2237 resources: Some(job_resource_requirements()),
2238 env: {
2239 let mut env_vars =
2240 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2241 env_vars.push(EnvVar {
2242 name: "RUST_LOG".to_string(),
2243 value: Some("info".to_string()),
2244 ..Default::default()
2245 });
2246 if repo.spec.s3.ca_bundle_ref.is_some() {
2247 env_vars.push(ca_bundle_env_var());
2248 }
2249 env_vars.extend(build_encryption_env_vars(
2250 repo.spec
2251 .encryption
2252 .as_ref()
2253 .and_then(|e| e.key_ref.as_ref()),
2254 ));
2255 Some(env_vars)
2256 },
2257 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2258 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2259 volume_mounts: {
2260 let mut mounts = vec![
2261 VolumeMount {
2262 name: SHARED_VOLUME.to_string(),
2263 mount_path: SHARED_VOL_PATH.to_string(),
2264 read_only: Some(true),
2265 ..Default::default()
2266 },
2267 VolumeMount {
2268 name: "operation".to_string(),
2269 mount_path: "/run/kaniop".to_string(),
2270 read_only: Some(true),
2271 ..Default::default()
2272 },
2273 VolumeMount {
2274 name: "result".to_string(),
2275 mount_path: "/run/kaniop-result".to_string(),
2276 ..Default::default()
2277 },
2278 ];
2279 mounts.extend(build_auth_volume_mounts(auth_method));
2280 if repo.spec.s3.ca_bundle_ref.is_some() {
2281 mounts.push(build_ca_bundle_volume_mount());
2282 }
2283 Some(mounts)
2284 },
2285 ..Default::default()
2286 }],
2287 volumes: {
2288 let mut vols = vec![
2289 Volume {
2290 name: DATA_VOLUME.to_string(),
2291 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2292 claim_name: primary_pvc_name(target)?,
2293 read_only: Some(false),
2294 }),
2295 ..Default::default()
2296 },
2297 Volume {
2298 name: TLS_VOLUME.to_string(),
2299 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2300 secret_name: Some(target.effective_tls_secret_name()),
2301 ..Default::default()
2302 }),
2303 ..Default::default()
2304 },
2305 Volume {
2306 name: SHARED_VOLUME.to_string(),
2307 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2308 size_limit: Some(crate::controller::backup_job_volume_size()),
2309 ..Default::default()
2310 }),
2311 ..Default::default()
2312 },
2313 Volume {
2314 name: "operation".to_string(),
2315 config_map: Some(ConfigMapVolumeSource {
2316 name: operation_cm_name,
2317 ..Default::default()
2318 }),
2319 ..Default::default()
2320 },
2321 Volume {
2322 name: "result".to_string(),
2323 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2324 size_limit: Some(Quantity("16Mi".to_string())),
2325 ..Default::default()
2326 }),
2327 ..Default::default()
2328 },
2329 ];
2330 vols.extend(build_auth_volumes(auth_method));
2331 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2332 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2333 }
2334 Some(vols)
2335 },
2336 ..Default::default()
2337 }),
2338 },
2339 ..Default::default()
2340 }),
2341 ..Default::default()
2342 };
2343 api.create(&PostParams::default(), &job)
2344 .await
2345 .map(|_| ())
2346 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2347}
2348
2349async fn ensure_source_prep_job(
2350 restore: &KanidmRestore,
2351 target: &Kanidm,
2352 ctx: &RestoreContext,
2353 name: &str,
2354) -> Result<()> {
2355 let ns = restore.namespace().unwrap();
2356 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2357 if api
2358 .get_opt(name)
2359 .await
2360 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2361 .is_some()
2362 {
2363 return Ok(());
2364 }
2365 let backup_name = restore
2366 .spec
2367 .source
2368 .backup_ref
2369 .as_ref()
2370 .map(|b| b.name.clone())
2371 .unwrap_or_default();
2372 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2373 .get(&backup_name)
2374 .await
2375 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2376 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2377 .get(&backup.spec.repository_ref.name)
2378 .await
2379 .map_err(|e| {
2380 Error::kube_error(
2381 "get",
2382 "KanidmBackupRepository",
2383 &ns,
2384 &backup.spec.repository_ref.name,
2385 e,
2386 )
2387 })?;
2388 let endpoint = repo
2389 .spec
2390 .s3
2391 .endpoint
2392 .as_deref()
2393 .unwrap_or("https://s3.amazonaws.com");
2394 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2395 let operation_doc = build_download_operation_doc(
2396 restore,
2397 target,
2398 &backup.spec.manifest_key,
2399 &backup.spec.backup_id,
2400 &repo.spec.s3.bucket,
2401 &repo.spec.s3.prefix,
2402 endpoint,
2403 region,
2404 repo.spec.s3.force_path_style,
2405 repo.spec.s3.insecure,
2406 repo.spec
2407 .s3
2408 .ca_bundle_ref
2409 .as_deref()
2410 .map(|_| ca_bundle_path())
2411 .as_deref(),
2412 repo.spec.encryption.as_ref(),
2413 );
2414 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2415 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2416 let auth_method = &repo.spec.authentication.reader;
2417 let data_mover_image = data_mover_image();
2418 let download_script = format!(
2419 r#"set -eu
2420/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2421RESULT_FILE="/run/kaniop-result/result.json"
2422TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2423if [ ! -f "$RESULT_FILE" ]; then
2424 echo "ERROR: result document not found at $RESULT_FILE"
2425 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2426 exit 1
2427fi
2428if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2429 echo "ERROR: source preparation download reported failure"
2430 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2431 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2432 exit 1
2433fi
2434dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2435echo "source preparation download completed successfully"
2436"#
2437 );
2438 let job = Job {
2439 metadata: ObjectMeta {
2440 name: Some(name.to_string()),
2441 namespace: Some(ns.clone()),
2442 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2443 ..Default::default()
2444 },
2445 spec: Some(JobSpec {
2446 backoff_limit: Some(0),
2447 template: PodTemplateSpec {
2448 metadata: Some(ObjectMeta::default()),
2449 spec: Some(PodSpec {
2450 automount_service_account_token: Some(false),
2451 restart_policy: Some("Never".to_string()),
2452 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2453 run_as_non_root: Some(true),
2454 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2455 type_: "RuntimeDefault".to_string(),
2456 ..Default::default()
2457 }),
2458 ..Default::default()
2459 }),
2460 containers: vec![Container {
2461 name: "source-prep".to_string(),
2462 image: Some(data_mover_image),
2463 command: Some(vec!["/bin/sh".to_string()]),
2464 args: Some(vec!["-c".to_string(), download_script]),
2465 security_context: Some(hardened_security_context()),
2466 resources: Some(job_resource_requirements()),
2467 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2468 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2469 env: {
2470 let mut env_vars = build_auth_env_vars(
2471 auth_method,
2472 &backup.spec.repository_ref.name,
2473 AuthRole::Reader,
2474 );
2475 env_vars.push(EnvVar {
2476 name: "RUST_LOG".to_string(),
2477 value: Some("info".to_string()),
2478 ..Default::default()
2479 });
2480 if repo.spec.s3.ca_bundle_ref.is_some() {
2481 env_vars.push(ca_bundle_env_var());
2482 }
2483 env_vars.extend(build_encryption_env_vars(
2484 repo.spec
2485 .encryption
2486 .as_ref()
2487 .and_then(|e| e.key_ref.as_ref()),
2488 ));
2489 Some(env_vars)
2490 },
2491 volume_mounts: {
2492 let mut mounts = vec![
2493 VolumeMount {
2494 name: STAGING_VOLUME.to_string(),
2495 mount_path: STAGING_PATH.to_string(),
2496 ..Default::default()
2497 },
2498 VolumeMount {
2499 name: "operation".to_string(),
2500 mount_path: "/run/kaniop".to_string(),
2501 read_only: Some(true),
2502 ..Default::default()
2503 },
2504 VolumeMount {
2505 name: "result".to_string(),
2506 mount_path: "/run/kaniop-result".to_string(),
2507 ..Default::default()
2508 },
2509 ];
2510 mounts.extend(build_auth_volume_mounts(auth_method));
2511 if repo.spec.s3.ca_bundle_ref.is_some() {
2512 mounts.push(build_ca_bundle_volume_mount());
2513 }
2514 Some(mounts)
2515 },
2516 ..Default::default()
2517 }],
2518 volumes: {
2519 let mut vols = vec![
2520 Volume {
2521 name: STAGING_VOLUME.to_string(),
2522 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2523 size_limit: Some(crate::controller::backup_job_volume_size()),
2524 ..Default::default()
2525 }),
2526 ..Default::default()
2527 },
2528 Volume {
2529 name: "operation".to_string(),
2530 config_map: Some(ConfigMapVolumeSource {
2531 name: operation_cm_name,
2532 ..Default::default()
2533 }),
2534 ..Default::default()
2535 },
2536 Volume {
2537 name: "result".to_string(),
2538 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2539 size_limit: Some(Quantity("16Mi".to_string())),
2540 ..Default::default()
2541 }),
2542 ..Default::default()
2543 },
2544 ];
2545 vols.extend(build_auth_volumes(auth_method));
2546 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2547 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2548 }
2549 Some(vols)
2550 },
2551 ..Default::default()
2552 }),
2553 },
2554 ..Default::default()
2555 }),
2556 ..Default::default()
2557 };
2558 api.create(&PostParams::default(), &job)
2559 .await
2560 .map(|_| ())
2561 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2562}
2563
2564#[allow(clippy::too_many_arguments)]
2565fn build_safety_upload_operation_doc(
2566 restore: &KanidmRestore,
2567 target: &Kanidm,
2568 backup_id: &str,
2569 manifest_key: &str,
2570 bucket: &str,
2571 prefix: &str,
2572 endpoint: &str,
2573 region: &str,
2574 force_path_style: bool,
2575 insecure: bool,
2576 ca_bundle_path: Option<&str>,
2577 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2578) -> Result<String> {
2579 let enc_mode =
2580 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2581 let enc_key_id = encryption
2582 .and_then(|e| e.key_id.as_ref())
2583 .map(|s| serde_json::Value::String(s.clone()));
2584 let op = UploadOperation {
2585 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2586 bucket: bucket.to_string(),
2587 prefix: prefix.to_string(),
2588 endpoint: endpoint.to_string(),
2589 region: region.to_string(),
2590 force_path_style,
2591 insecure,
2592 ca_bundle_path: ca_bundle_path.map(str::to_string),
2593 backup_id: backup_id.to_string(),
2594 namespace_uid: restore.namespace().unwrap_or_default(),
2595 kanidm_uid: restore.spec.target_ref.uid.clone(),
2596 kanidm_name: restore.spec.target_ref.name.clone(),
2597 domain: target.spec.domain.clone(),
2598 kanidm_version: target
2599 .status
2600 .as_ref()
2601 .and_then(|s| s.version.as_ref())
2602 .map(|v| v.image_tag.clone())
2603 .unwrap_or_default(),
2604 image_digest: None,
2605 consistency: "kanidm-offline".to_string(),
2606 reason: "restore-safety".to_string(),
2607 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2608 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2609 result_path: "/run/kaniop-result/result.json".to_string(),
2610 max_concurrent_parts: 4,
2611 max_retries: 3,
2612 };
2613 let _ = manifest_key;
2614 let doc = OperationDocument {
2615 api_version: OPERATION_DOC_VERSION.to_string(),
2616 kind: "OperationDocument".to_string(),
2617 spec: OperationSpec::Upload(op),
2618 };
2619 serde_json::to_string(&doc)
2620 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2621}
2622
2623async fn validate_backup_ref(
2624 restore: &KanidmRestore,
2625 target: &Kanidm,
2626 ctx: &RestoreContext,
2627) -> Result<()> {
2628 let backup_ref = restore
2629 .spec
2630 .source
2631 .backup_ref
2632 .as_ref()
2633 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2634 let ns = restore.namespace().unwrap();
2635 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2636 .get(&backup_ref.name)
2637 .await
2638 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2639 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2640 return Err(Error::MissingData(format!(
2641 "KanidmBackup '{}' is not Ready (phase: {:?})",
2642 backup_ref.name,
2643 backup.status.as_ref().map(|s| &s.phase)
2644 )));
2645 }
2646 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2647 return Err(Error::MissingData(format!(
2648 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2649 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2650 )));
2651 }
2652 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2653 return Err(Error::MissingData(format!(
2654 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2655 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2656 )));
2657 }
2658 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2659 .get(&backup.spec.repository_ref.name)
2660 .await
2661 .map_err(|e| {
2662 Error::kube_error(
2663 "get",
2664 "KanidmBackupRepository",
2665 &ns,
2666 &backup.spec.repository_ref.name,
2667 e,
2668 )
2669 })?;
2670 if !has_accepted_condition(
2671 &repo
2672 .status
2673 .as_ref()
2674 .map(|s| &s.conditions)
2675 .cloned()
2676 .unwrap_or_default(),
2677 ) {
2678 return Err(Error::MissingData(format!(
2679 "KanidmBackupRepository '{}' configuration has not been accepted",
2680 backup.spec.repository_ref.name
2681 )));
2682 }
2683 validate_backup_compatibility(restore, target, &backup)?;
2684 Ok(())
2685}
2686
2687fn has_accepted_condition(conditions: &[Condition]) -> bool {
2688 conditions
2689 .iter()
2690 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2691}
2692
2693fn validate_backup_compatibility(
2694 restore: &KanidmRestore,
2695 target: &Kanidm,
2696 backup: &KanidmBackup,
2697) -> Result<()> {
2698 let status = backup.status.as_ref();
2699 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2700 let target_version = target
2701 .status
2702 .as_ref()
2703 .and_then(|s| s.version.as_ref())
2704 .map(|v| v.image_tag.as_str());
2705 if let Some(tv) = target_version {
2706 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2707 return Err(Error::MissingData(format!(
2708 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2709 )));
2710 }
2711 }
2712 }
2713 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2714 if !backup_digest.is_empty()
2715 && restore.spec.restore_image.contains('@')
2716 && !restore.spec.restore_image.contains(backup_digest)
2717 {
2718 return Err(Error::MissingData(format!(
2719 "restore image digest does not match backup image digest '{backup_digest}'"
2720 )));
2721 }
2722 }
2723 Ok(())
2724}
2725
2726#[allow(clippy::too_many_arguments)]
2727fn build_download_operation_doc(
2728 restore: &KanidmRestore,
2729 target: &Kanidm,
2730 manifest_key: &str,
2731 expected_backup_id: &str,
2732 bucket: &str,
2733 prefix: &str,
2734 endpoint: &str,
2735 region: &str,
2736 force_path_style: bool,
2737 insecure: bool,
2738 ca_bundle_path: Option<&str>,
2739 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2740) -> String {
2741 let enc_mode =
2742 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2743 let enc_key_id = encryption
2744 .and_then(|e| e.key_id.as_ref())
2745 .map(|s| serde_json::Value::String(s.clone()));
2746 serde_json::json!({
2747 "apiVersion": "backup.kaniop.rs/v1alpha1",
2748 "kind": "OperationDocument",
2749 "operation": "download",
2750 "manifestKey": manifest_key,
2751 "bucket": bucket,
2752 "prefix": prefix,
2753 "endpoint": endpoint,
2754 "region": region,
2755 "forcePathStyle": force_path_style,
2756 "insecure": insecure,
2757 "caBundlePath": ca_bundle_path,
2758 "expectedBackupId": expected_backup_id,
2759 "expectedKanidmUid": restore.spec.target_ref.uid,
2760 "expectedDomain": target.spec.domain,
2761 "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2762 "resultPath": "/run/kaniop-result/result.json",
2763 "maxRetries": 3,
2764 "encryptionMode": enc_mode,
2765 "encryptionKeyId": enc_key_id,
2766 })
2767 .to_string()
2768}
2769
2770async fn ensure_operation_configmap(
2771 restore: &KanidmRestore,
2772 name: &str,
2773 data: &str,
2774 ns: &str,
2775 ctx: &RestoreContext,
2776) -> Result<()> {
2777 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2778 if api
2779 .get_opt(name)
2780 .await
2781 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2782 .is_some()
2783 {
2784 return Ok(());
2785 }
2786 let cm = ConfigMap {
2787 metadata: ObjectMeta {
2788 name: Some(name.to_string()),
2789 namespace: Some(ns.to_string()),
2790 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2791 ..Default::default()
2792 },
2793 data: Some(std::collections::BTreeMap::from([(
2794 "operation.json".to_string(),
2795 data.to_string(),
2796 )])),
2797 ..Default::default()
2798 };
2799 api.create(&PostParams::default(), &cm)
2800 .await
2801 .map(|_| ())
2802 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2803}
2804
2805#[cfg(test)]
2806mod tests {
2807 use super::{
2808 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2809 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2810 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2811 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2812 hardened_security_context, has_accepted_condition, is_remote_source,
2813 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2814 validate_safety_backup_config, validate_source,
2815 };
2816 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2817 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2818 use kube::api::ObjectMeta as ApiObjectMeta;
2819 use std::collections::BTreeMap;
2820
2821 fn make_restore(
2822 source: KanidmRestoreSource,
2823 safety: Option<SafetyBackupConfig>,
2824 ) -> KanidmRestore {
2825 KanidmRestore {
2826 metadata: ApiObjectMeta {
2827 name: Some("test-restore".to_string()),
2828 namespace: Some("default".to_string()),
2829 ..Default::default()
2830 },
2831 spec: KanidmRestoreSpec {
2832 target_ref: KanidmRestoreTargetRef {
2833 name: "corp-idm".to_string(),
2834 uid: "test-uid-123".to_string(),
2835 },
2836 source,
2837 restore_image: "kanidm/server@sha256:abc".to_string(),
2838 safety_backup: safety,
2839 },
2840 status: None,
2841 }
2842 }
2843
2844 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2845 super::super::crd::Kanidm {
2846 status: Some(super::super::crd::KanidmStatus {
2847 available_replicas: 1,
2848 replicas: 1,
2849 unavailable_replicas: 0,
2850 updated_replicas: 1,
2851 replica_statuses: vec![],
2852 replica_column: "1/1".to_string(),
2853 secret_name: None,
2854 version: Some(super::super::crd::KanidmVersionStatus {
2855 image_tag: image_tag.to_string(),
2856 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2857 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2858 }),
2859 domain_appearance_image: None,
2860 mail_sender: None,
2861 conditions: None,
2862 }),
2863 ..Default::default()
2864 }
2865 }
2866
2867 #[test]
2868 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2869 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2870 }
2871
2872 #[test]
2873 fn completed_restore_sets_ready_condition() {
2874 let mut status = KanidmRestoreStatus {
2875 phase: KanidmRestorePhase::Completed,
2876 ..Default::default()
2877 };
2878 super::update_restore_conditions(&mut status, Some(7));
2879 assert!(status.conditions.iter().any(|condition| {
2880 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2881 }));
2882 assert!(!status.conditions.iter().any(|condition| {
2883 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2884 }));
2885 }
2886
2887 #[test]
2888 fn failed_restore_sets_failed_condition() {
2889 let mut status = KanidmRestoreStatus {
2890 phase: KanidmRestorePhase::Failed,
2891 message: Some("verification failed".to_string()),
2892 ..Default::default()
2893 };
2894 super::update_restore_conditions(&mut status, Some(8));
2895 assert!(status.conditions.iter().any(|condition| {
2896 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2897 }));
2898 }
2899
2900 #[test]
2901 fn restore_filename_is_confined_to_backup_directory() {
2902 assert!(safe_basename("backup.json.gz"));
2903 assert!(!safe_basename("../backup.json.gz"));
2904 assert!(!safe_basename("a/b"));
2905 assert!(!safe_basename(""));
2906 }
2907
2908 #[test]
2909 fn restore_image_rejects_latest_and_unpinned_names() {
2910 assert!(mutable_image("kanidm/server:latest"));
2911 assert!(mutable_image("kanidm/server"));
2912 assert!(!mutable_image("kanidm/server:1.10.0"));
2913 assert!(!mutable_image("kanidm/server@sha256:abc"));
2914 }
2915
2916 #[test]
2917 fn source_validation_rejects_both_local_and_backup_ref() {
2918 let source = KanidmRestoreSource {
2919 local: Some(super::KanidmRestoreLocalSource {
2920 file_name: "backup.json".to_string(),
2921 }),
2922 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2923 name: "some-backup".to_string(),
2924 }),
2925 };
2926 let restore = make_restore(source, None);
2927 assert!(validate_source(&restore).is_err());
2928 }
2929
2930 #[test]
2931 fn source_validation_rejects_empty_source() {
2932 let source = KanidmRestoreSource {
2933 local: None,
2934 backup_ref: None,
2935 };
2936 let restore = make_restore(source, None);
2937 assert!(validate_source(&restore).is_err());
2938 }
2939
2940 #[test]
2941 fn source_validation_accepts_local_source() {
2942 let source = KanidmRestoreSource {
2943 local: Some(super::KanidmRestoreLocalSource {
2944 file_name: "backup.json".to_string(),
2945 }),
2946 backup_ref: None,
2947 };
2948 let restore = make_restore(source, None);
2949 assert!(validate_source(&restore).is_ok());
2950 }
2951
2952 #[test]
2953 fn source_validation_accepts_backup_ref_source() {
2954 let source = KanidmRestoreSource {
2955 local: None,
2956 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2957 name: "corp-idm-019c7c76".to_string(),
2958 }),
2959 };
2960 let restore = make_restore(source, None);
2961 assert!(validate_source(&restore).is_ok());
2962 }
2963
2964 #[test]
2965 fn source_validation_rejects_empty_backup_ref_name() {
2966 let source = KanidmRestoreSource {
2967 local: None,
2968 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2969 name: "".to_string(),
2970 }),
2971 };
2972 let restore = make_restore(source, None);
2973 assert!(validate_source(&restore).is_err());
2974 }
2975
2976 #[test]
2977 fn is_remote_source_true_for_backup_ref() {
2978 let source = KanidmRestoreSource {
2979 local: None,
2980 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2981 name: "backup-1".to_string(),
2982 }),
2983 };
2984 let restore = make_restore(source, None);
2985 assert!(is_remote_source(&restore));
2986 }
2987
2988 #[test]
2989 fn is_remote_source_false_for_local() {
2990 let source = KanidmRestoreSource {
2991 local: Some(super::KanidmRestoreLocalSource {
2992 file_name: "backup.json".to_string(),
2993 }),
2994 backup_ref: None,
2995 };
2996 let restore = make_restore(source, None);
2997 assert!(!is_remote_source(&restore));
2998 }
2999
3000 #[test]
3001 fn requires_safety_backup_default_true() {
3002 let source = KanidmRestoreSource {
3003 local: None,
3004 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3005 name: "backup-1".to_string(),
3006 }),
3007 };
3008 let restore = make_restore(source, None);
3009 assert!(requires_safety_backup(&restore));
3010 }
3011
3012 #[test]
3013 fn requires_safety_backup_false_when_skip() {
3014 let source = KanidmRestoreSource {
3015 local: None,
3016 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3017 name: "backup-1".to_string(),
3018 }),
3019 };
3020 let safety = SafetyBackupConfig {
3021 repository_ref: Some(SafetyBackupRepositoryRef {
3022 name: "offsite".to_string(),
3023 }),
3024 skip: true,
3025 };
3026 let restore = make_restore(source, Some(safety));
3027 assert!(!requires_safety_backup(&restore));
3028 }
3029
3030 #[test]
3031 fn break_glass_validation_rejects_skip_without_reason() {
3032 let source = KanidmRestoreSource {
3033 local: None,
3034 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3035 name: "backup-1".to_string(),
3036 }),
3037 };
3038 let safety = SafetyBackupConfig {
3039 repository_ref: Some(SafetyBackupRepositoryRef {
3040 name: "offsite".to_string(),
3041 }),
3042 skip: true,
3043 };
3044 let mut restore = make_restore(source, Some(safety));
3045 restore.metadata.annotations = Some(BTreeMap::from([(
3046 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3047 "commander@example.com".to_string(),
3048 )]));
3049 assert!(validate_safety_backup_config(&restore).is_err());
3050 }
3051
3052 #[test]
3053 fn break_glass_validation_rejects_skip_without_approver() {
3054 let source = KanidmRestoreSource {
3055 local: None,
3056 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3057 name: "backup-1".to_string(),
3058 }),
3059 };
3060 let safety = SafetyBackupConfig {
3061 repository_ref: Some(SafetyBackupRepositoryRef {
3062 name: "offsite".to_string(),
3063 }),
3064 skip: true,
3065 };
3066 let mut restore = make_restore(source, Some(safety));
3067 restore.metadata.annotations = Some(BTreeMap::from([(
3068 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3069 "PVC is unreadable".to_string(),
3070 )]));
3071 assert!(validate_safety_backup_config(&restore).is_err());
3072 }
3073
3074 #[test]
3075 fn break_glass_validation_accepts_valid_annotations() {
3076 let source = KanidmRestoreSource {
3077 local: None,
3078 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3079 name: "backup-1".to_string(),
3080 }),
3081 };
3082 let safety = SafetyBackupConfig {
3083 repository_ref: Some(SafetyBackupRepositoryRef {
3084 name: "offsite".to_string(),
3085 }),
3086 skip: true,
3087 };
3088 let mut restore = make_restore(source, Some(safety));
3089 restore.metadata.annotations = Some(BTreeMap::from([
3090 (
3091 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3092 "PVC is unreadable".to_string(),
3093 ),
3094 (
3095 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3096 "commander@example.com".to_string(),
3097 ),
3098 ]));
3099 assert!(validate_safety_backup_config(&restore).is_ok());
3100 }
3101
3102 #[test]
3103 fn remote_source_requires_repository_ref_when_not_skipping() {
3104 let source = KanidmRestoreSource {
3105 local: None,
3106 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3107 name: "backup-1".to_string(),
3108 }),
3109 };
3110 let safety = SafetyBackupConfig {
3111 repository_ref: None,
3112 skip: false,
3113 };
3114 let restore = make_restore(source, Some(safety));
3115 assert!(validate_safety_backup_config(&restore).is_err());
3116 }
3117
3118 #[test]
3119 fn remote_source_accepts_repository_ref() {
3120 let source = KanidmRestoreSource {
3121 local: None,
3122 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3123 name: "backup-1".to_string(),
3124 }),
3125 };
3126 let safety = SafetyBackupConfig {
3127 repository_ref: Some(SafetyBackupRepositoryRef {
3128 name: "offsite".to_string(),
3129 }),
3130 skip: false,
3131 };
3132 let restore = make_restore(source, Some(safety));
3133 assert!(validate_safety_backup_config(&restore).is_ok());
3134 }
3135
3136 #[test]
3137 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3138 let phases = vec![
3139 KanidmRestorePhase::Pending,
3140 KanidmRestorePhase::Validating,
3141 KanidmRestorePhase::Quiescing,
3142 KanidmRestorePhase::SafetyBackup,
3143 KanidmRestorePhase::PreparingSource,
3144 KanidmRestorePhase::RestoringPrimary,
3145 KanidmRestorePhase::Verifying,
3146 KanidmRestorePhase::RebuildingReplicas,
3147 KanidmRestorePhase::Resuming,
3148 KanidmRestorePhase::Completed,
3149 ];
3150 assert_eq!(phases.len(), 10);
3151 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3152 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3153 }
3154
3155 #[test]
3156 fn status_default_has_empty_phase_timestamps() {
3157 let status = KanidmRestoreStatus::default();
3158 assert!(status.phase_timestamps.is_empty());
3159 }
3160
3161 #[test]
3162 fn status_default_has_empty_original_replicas() {
3163 let status = KanidmRestoreStatus::default();
3164 assert!(status.original_replicas.is_empty());
3165 }
3166
3167 #[test]
3168 fn hardened_security_context_drops_all_capabilities() {
3169 let ctx = hardened_security_context();
3170 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3171 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3172 assert_eq!(ctx.run_as_non_root, Some(true));
3173 let caps = ctx.capabilities.unwrap();
3174 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3175 assert_eq!(
3176 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3177 Some("RuntimeDefault")
3178 );
3179 }
3180
3181 #[test]
3182 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3183 let ctx = kanidm_job_security_context();
3184 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3185 assert!(ctx.run_as_non_root.is_none());
3186 assert!(ctx.read_only_root_filesystem.is_none());
3187 let caps = ctx.capabilities.unwrap();
3188 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3189 assert_eq!(
3190 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3191 Some("RuntimeDefault")
3192 );
3193 }
3194
3195 #[test]
3196 fn safety_backup_job_name_is_deterministic() {
3197 let source = KanidmRestoreSource {
3198 local: Some(super::KanidmRestoreLocalSource {
3199 file_name: "backup.json".to_string(),
3200 }),
3201 backup_ref: None,
3202 };
3203 let restore = make_restore(source, None);
3204 let name1 = super::safety_backup_job_name(&restore);
3205 let name2 = super::safety_backup_job_name(&restore);
3206 assert_eq!(name1, name2);
3207 assert_eq!(name1, "test-restore-safety-backup");
3208 }
3209
3210 #[test]
3211 fn source_prep_job_name_is_deterministic() {
3212 let source = KanidmRestoreSource {
3213 local: None,
3214 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3215 name: "backup-1".to_string(),
3216 }),
3217 };
3218 let restore = make_restore(source, None);
3219 let name = super::source_prep_job_name(&restore);
3220 assert_eq!(name, "test-restore-source-prep");
3221 }
3222
3223 #[test]
3224 fn break_glass_condition_is_recorded_in_status() {
3225 let mut status = KanidmRestoreStatus::default();
3226 let condition = super::restore_condition(
3227 &status.conditions,
3228 CONDITION_BREAK_GLASS,
3229 CONDITION_TRUE,
3230 "BreakGlassOverride",
3231 "Safety backup skipped",
3232 Some(1),
3233 );
3234 status.conditions.push(condition);
3235 assert!(status.conditions.iter().any(|c| {
3236 c.type_ == CONDITION_BREAK_GLASS
3237 && c.status == CONDITION_TRUE
3238 && c.reason == "BreakGlassOverride"
3239 }));
3240 }
3241
3242 #[test]
3243 fn replica_count_entry_serialization() {
3244 let entry = ReplicaCountEntry {
3245 group: "primary".to_string(),
3246 replicas: 3,
3247 };
3248 let json = serde_json::to_string(&entry).unwrap();
3249 assert!(json.contains("\"group\":\"primary\""));
3250 assert!(json.contains("\"replicas\":3"));
3251 }
3252
3253 #[test]
3254 fn local_source_does_not_require_repository_ref() {
3255 let source = KanidmRestoreSource {
3256 local: Some(super::KanidmRestoreLocalSource {
3257 file_name: "backup.json".to_string(),
3258 }),
3259 backup_ref: None,
3260 };
3261 let safety = SafetyBackupConfig {
3262 repository_ref: None,
3263 skip: false,
3264 };
3265 let restore = make_restore(source, Some(safety));
3266 assert!(validate_safety_backup_config(&restore).is_ok());
3267 }
3268
3269 #[test]
3270 fn skip_without_break_glass_annotations_is_rejected() {
3271 let source = KanidmRestoreSource {
3272 local: Some(super::KanidmRestoreLocalSource {
3273 file_name: "backup.json".to_string(),
3274 }),
3275 backup_ref: None,
3276 };
3277 let safety = SafetyBackupConfig {
3278 repository_ref: None,
3279 skip: true,
3280 };
3281 let restore = make_restore(source, Some(safety));
3282 assert!(validate_safety_backup_config(&restore).is_err());
3283 }
3284
3285 #[test]
3286 fn has_accepted_condition_true() {
3287 let conditions = vec![Condition {
3288 type_: "Ready".to_string(),
3289 status: "True".to_string(),
3290 reason: "Accepted".to_string(),
3291 message: String::new(),
3292 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3293 k8s_openapi::jiff::Timestamp::now(),
3294 ),
3295 observed_generation: None,
3296 }];
3297 assert!(has_accepted_condition(&conditions));
3298 }
3299
3300 #[test]
3301 fn has_accepted_condition_false_when_not_accepted_reason() {
3302 let conditions = vec![Condition {
3303 type_: "Ready".to_string(),
3304 status: "True".to_string(),
3305 reason: "SomeOtherReason".to_string(),
3306 message: String::new(),
3307 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3308 k8s_openapi::jiff::Timestamp::now(),
3309 ),
3310 observed_generation: None,
3311 }];
3312 assert!(!has_accepted_condition(&conditions));
3313 }
3314
3315 #[test]
3316 fn has_accepted_condition_false_when_status_false() {
3317 let conditions = vec![Condition {
3318 type_: "Ready".to_string(),
3319 status: "False".to_string(),
3320 reason: "Accepted".to_string(),
3321 message: String::new(),
3322 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3323 k8s_openapi::jiff::Timestamp::now(),
3324 ),
3325 observed_generation: None,
3326 }];
3327 assert!(!has_accepted_condition(&conditions));
3328 }
3329
3330 #[test]
3331 fn has_accepted_condition_empty() {
3332 assert!(!has_accepted_condition(&[]));
3333 }
3334
3335 #[test]
3336 fn safety_backup_upload_operation_doc_is_valid() {
3337 let source = KanidmRestoreSource {
3338 local: None,
3339 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3340 name: "backup-1".to_string(),
3341 }),
3342 };
3343 let mut restore = make_restore(source, None);
3344 restore.metadata.namespace = Some("test-ns".to_string());
3345 let target = super::super::crd::Kanidm::default();
3346 let doc_str = super::build_safety_upload_operation_doc(
3347 &restore,
3348 &target,
3349 "test-backup-id",
3350 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3351 "my-bucket",
3352 "prod",
3353 "https://s3.example.com",
3354 "us-east-1",
3355 false,
3356 false,
3357 None,
3358 None,
3359 )
3360 .unwrap();
3361 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3362 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3363 assert_eq!(parsed["kind"], "OperationDocument");
3364 assert_eq!(parsed["operation"], "upload");
3365 assert_eq!(parsed["bucket"], "my-bucket");
3366 assert_eq!(parsed["prefix"], "prod");
3367 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3368 assert_eq!(parsed["region"], "us-east-1");
3369 assert_eq!(parsed["backupId"], "test-backup-id");
3370 assert_eq!(parsed["consistency"], "kanidm-offline");
3371 assert_eq!(parsed["reason"], "restore-safety");
3372 assert_eq!(
3373 parsed["payloadPath"],
3374 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3375 );
3376 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3377 }
3378
3379 #[test]
3380 fn download_operation_doc_uses_real_repository_data() {
3381 let source = KanidmRestoreSource {
3382 local: None,
3383 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3384 name: "backup-1".to_string(),
3385 }),
3386 };
3387 let restore = make_restore(source, None);
3388 let target = super::super::crd::Kanidm::default();
3389 let doc_str = super::build_download_operation_doc(
3390 &restore,
3391 &target,
3392 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3393 "019c7c76-f423-7a12-8f41-2bea7588a303",
3394 "real-bucket",
3395 "real-prefix",
3396 "https://real-endpoint.com",
3397 "eu-west-1",
3398 true,
3399 false,
3400 None,
3401 None,
3402 );
3403 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3404 assert_eq!(parsed["bucket"], "real-bucket");
3405 assert_eq!(parsed["prefix"], "real-prefix");
3406 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3407 assert_eq!(parsed["region"], "eu-west-1");
3408 assert_eq!(parsed["forcePathStyle"], true);
3409 assert_eq!(
3410 parsed["manifestKey"],
3411 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3412 );
3413 assert_eq!(
3414 parsed["expectedBackupId"],
3415 "019c7c76-f423-7a12-8f41-2bea7588a303"
3416 );
3417 }
3418
3419 #[test]
3420 fn safety_backup_ref_is_set_after_verification() {
3421 let status = KanidmRestoreStatus {
3422 phase: KanidmRestorePhase::SafetyBackup,
3423 safety_backup_job_name: Some("test-safety-backup".to_string()),
3424 safety_backup_ref: Some("safety-test-restore".to_string()),
3425 safety_backup_expected_backup_id: Some(
3426 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3427 ),
3428 safety_backup_manifest_key: Some(
3429 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3430 ),
3431 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3432 ..Default::default()
3433 };
3434 assert!(status.safety_backup_ref.is_some());
3435 assert!(
3436 status
3437 .safety_backup_ref
3438 .as_ref()
3439 .unwrap()
3440 .starts_with("safety-")
3441 );
3442 assert!(status.safety_backup_expected_backup_id.is_some());
3443 assert!(status.safety_backup_manifest_key.is_some());
3444 assert!(status.safety_backup_payload_sha256.is_some());
3445 }
3446
3447 #[test]
3448 fn database_mutation_not_started_without_safety_backup_ref() {
3449 let status = KanidmRestoreStatus {
3450 phase: KanidmRestorePhase::PreparingSource,
3451 safety_backup_ref: None,
3452 database_mutation_started: false,
3453 ..Default::default()
3454 };
3455 assert!(!status.database_mutation_started);
3456 assert!(status.safety_backup_ref.is_none());
3457 }
3458
3459 #[test]
3460 fn pre_boundary_failure_does_not_set_mutation_started() {
3461 let status = KanidmRestoreStatus {
3462 phase: KanidmRestorePhase::Failed,
3463 message: Some(
3464 "safety backup job failed; target restored to original state".to_string(),
3465 ),
3466 database_mutation_started: false,
3467 ..Default::default()
3468 };
3469 assert!(!status.database_mutation_started);
3470 }
3471
3472 #[test]
3473 fn shared_volume_constants_are_distinct() {
3474 assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3475 assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3476 }
3477
3478 #[test]
3479 fn validate_backup_compatibility_matching_versions_passes() {
3480 let source = KanidmRestoreSource {
3481 local: None,
3482 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3483 name: "backup-1".to_string(),
3484 }),
3485 };
3486 let restore = make_restore(source, None);
3487 let target = kanidm_with_version("1.10.4");
3488 let backup = kaniop_backup_core::crd::KanidmBackup {
3489 metadata: kube::api::ObjectMeta::default(),
3490 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3491 backup_id: "id".to_string(),
3492 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3493 name: "k".to_string(),
3494 uid: "uid".to_string(),
3495 },
3496 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3497 name: "repo".to_string(),
3498 },
3499 manifest_key: "key".to_string(),
3500 },
3501 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3502 kanidm_version: Some("1.10.4".to_string()),
3503 ..Default::default()
3504 }),
3505 };
3506 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3507 }
3508
3509 #[test]
3510 fn validate_backup_compatibility_version_mismatch_fails() {
3511 let source = KanidmRestoreSource {
3512 local: None,
3513 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3514 name: "backup-1".to_string(),
3515 }),
3516 };
3517 let restore = make_restore(source, None);
3518 let target = kanidm_with_version("1.11.0");
3519 let backup = kaniop_backup_core::crd::KanidmBackup {
3520 metadata: kube::api::ObjectMeta::default(),
3521 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3522 backup_id: "id".to_string(),
3523 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3524 name: "k".to_string(),
3525 uid: "uid".to_string(),
3526 },
3527 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3528 name: "repo".to_string(),
3529 },
3530 manifest_key: "key".to_string(),
3531 },
3532 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3533 kanidm_version: Some("1.10.4".to_string()),
3534 ..Default::default()
3535 }),
3536 };
3537 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3538 }
3539
3540 #[test]
3541 fn validate_backup_compatibility_no_backup_version_passes() {
3542 let source = KanidmRestoreSource {
3543 local: None,
3544 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3545 name: "backup-1".to_string(),
3546 }),
3547 };
3548 let restore = make_restore(source, None);
3549 let target = super::super::crd::Kanidm::default();
3550 let backup = kaniop_backup_core::crd::KanidmBackup {
3551 metadata: kube::api::ObjectMeta::default(),
3552 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3553 backup_id: "id".to_string(),
3554 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3555 name: "k".to_string(),
3556 uid: "uid".to_string(),
3557 },
3558 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3559 name: "repo".to_string(),
3560 },
3561 manifest_key: "key".to_string(),
3562 },
3563 status: None,
3564 };
3565 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3566 }
3567
3568 #[test]
3569 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3570 let source = KanidmRestoreSource {
3571 local: None,
3572 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3573 name: "backup-1".to_string(),
3574 }),
3575 };
3576 let mut restore = make_restore(source, None);
3577 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3578 let target = super::super::crd::Kanidm::default();
3579 let backup = kaniop_backup_core::crd::KanidmBackup {
3580 metadata: kube::api::ObjectMeta::default(),
3581 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3582 backup_id: "id".to_string(),
3583 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3584 name: "k".to_string(),
3585 uid: "uid".to_string(),
3586 },
3587 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3588 name: "repo".to_string(),
3589 },
3590 manifest_key: "key".to_string(),
3591 },
3592 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3593 image_digest: Some("sha256:originaldigest".to_string()),
3594 ..Default::default()
3595 }),
3596 };
3597 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3598 }
3599
3600 #[test]
3601 fn validate_backup_compatibility_image_digest_match_passes() {
3602 let source = KanidmRestoreSource {
3603 local: None,
3604 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3605 name: "backup-1".to_string(),
3606 }),
3607 };
3608 let mut restore = make_restore(source, None);
3609 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3610 let target = super::super::crd::Kanidm::default();
3611 let backup = kaniop_backup_core::crd::KanidmBackup {
3612 metadata: kube::api::ObjectMeta::default(),
3613 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3614 backup_id: "id".to_string(),
3615 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3616 name: "k".to_string(),
3617 uid: "uid".to_string(),
3618 },
3619 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3620 name: "repo".to_string(),
3621 },
3622 manifest_key: "key".to_string(),
3623 },
3624 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3625 image_digest: Some("sha256:abc123".to_string()),
3626 ..Default::default()
3627 }),
3628 };
3629 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3630 }
3631
3632 #[test]
3633 fn safety_backup_id_is_deterministic() {
3634 let source = KanidmRestoreSource {
3635 local: None,
3636 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3637 name: "backup-1".to_string(),
3638 }),
3639 };
3640 let restore = make_restore(source, None);
3641 let id1 = super::compute_safety_backup_id(&restore);
3642 let id2 = super::compute_safety_backup_id(&restore);
3643 assert_eq!(id1, id2);
3644 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3645 }
3646
3647 #[test]
3648 fn safety_backup_id_differs_for_different_restores() {
3649 let source1 = KanidmRestoreSource {
3650 local: None,
3651 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3652 name: "backup-1".to_string(),
3653 }),
3654 };
3655 let restore1 = make_restore(source1, None);
3656 let mut restore2 = make_restore(
3657 KanidmRestoreSource {
3658 local: None,
3659 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3660 name: "backup-2".to_string(),
3661 }),
3662 },
3663 None,
3664 );
3665 restore2.metadata.name = Some("different-restore".to_string());
3666 let id1 = super::compute_safety_backup_id(&restore1);
3667 let id2 = super::compute_safety_backup_id(&restore2);
3668 assert_ne!(id1, id2);
3669 }
3670
3671 #[test]
3672 fn safety_manifest_key_is_deterministic() {
3673 let source = KanidmRestoreSource {
3674 local: None,
3675 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3676 name: "backup-1".to_string(),
3677 }),
3678 };
3679 let restore = make_restore(source, None);
3680 let key1 = super::compute_safety_manifest_key(&restore);
3681 let key2 = super::compute_safety_manifest_key(&restore);
3682 assert_eq!(key1, key2);
3683 assert!(key1.contains("manifest.json"));
3684 assert!(key1.starts_with("v1/tenants/"));
3685 }
3686
3687 #[test]
3688 fn safety_backup_id_survives_restart() {
3689 let source = KanidmRestoreSource {
3690 local: None,
3691 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3692 name: "backup-1".to_string(),
3693 }),
3694 };
3695 let mut restore = make_restore(source, None);
3696 let original_id = super::compute_safety_backup_id(&restore);
3697 restore.status = Some(KanidmRestoreStatus {
3698 phase: KanidmRestorePhase::SafetyBackup,
3699 safety_backup_expected_backup_id: Some(original_id.clone()),
3700 ..Default::default()
3701 });
3702 let recomputed_id = super::compute_safety_backup_id(&restore);
3703 assert_eq!(original_id, recomputed_id);
3704 }
3705
3706 #[test]
3707 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3708 let source = KanidmRestoreSource {
3709 local: None,
3710 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3711 name: "backup-1".to_string(),
3712 }),
3713 };
3714 let restore = make_restore(source, None);
3715 assert!(super::requires_safety_backup(&restore));
3716 let status_without_ref = KanidmRestoreStatus {
3717 phase: KanidmRestorePhase::RestoringPrimary,
3718 safety_backup_ref: None,
3719 database_mutation_started: false,
3720 ..Default::default()
3721 };
3722 assert!(status_without_ref.safety_backup_ref.is_none());
3723 assert!(!status_without_ref.database_mutation_started);
3724 }
3725
3726 #[test]
3727 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3728 let status_with_ref = KanidmRestoreStatus {
3729 phase: KanidmRestorePhase::RestoringPrimary,
3730 safety_backup_ref: Some("safety-test-restore".to_string()),
3731 safety_backup_manifest_key: Some(
3732 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3733 ),
3734 safety_backup_payload_sha256: Some("abc123".to_string()),
3735 database_mutation_started: false,
3736 ..Default::default()
3737 };
3738 assert!(status_with_ref.safety_backup_ref.is_some());
3739 assert!(!status_with_ref.database_mutation_started);
3740 }
3741
3742 #[test]
3743 fn result_document_verification_rejects_missing_backup_id() {
3744 let result_json = r#"{
3745 "apiVersion": "backup.kaniop.rs/v1alpha1",
3746 "kind": "ResultDocument",
3747 "operation": "upload",
3748 "success": true,
3749 "exitCode": "success",
3750 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3751 "payloadSha256": "abc123"
3752 }"#;
3753 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3754 assert!(result.backup_id.is_none());
3755 }
3756
3757 #[test]
3758 fn result_document_verification_rejects_missing_manifest_key() {
3759 let result_json = r#"{
3760 "apiVersion": "backup.kaniop.rs/v1alpha1",
3761 "kind": "ResultDocument",
3762 "operation": "upload",
3763 "success": true,
3764 "exitCode": "success",
3765 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3766 "payloadSha256": "abc123"
3767 }"#;
3768 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3769 assert!(result.manifest_key.is_none());
3770 }
3771
3772 #[test]
3773 fn result_document_verification_rejects_missing_payload_sha256() {
3774 let result_json = r#"{
3775 "apiVersion": "backup.kaniop.rs/v1alpha1",
3776 "kind": "ResultDocument",
3777 "operation": "upload",
3778 "success": true,
3779 "exitCode": "success",
3780 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3781 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3782 }"#;
3783 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3784 assert!(result.payload_sha256.is_none());
3785 }
3786
3787 #[test]
3788 fn result_document_verification_rejects_failure_exit_code() {
3789 let result_json = r#"{
3790 "apiVersion": "backup.kaniop.rs/v1alpha1",
3791 "kind": "ResultDocument",
3792 "operation": "upload",
3793 "success": false,
3794 "exitCode": "integrity",
3795 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3796 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3797 "payloadSha256": "abc123"
3798 }"#;
3799 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3800 assert!(!result.success);
3801 assert_eq!(
3802 result.exit_code,
3803 kaniop_backup_core::result::ExitCode::Integrity
3804 );
3805 }
3806
3807 #[test]
3808 fn result_document_verification_rejects_wrong_operation() {
3809 let result_json = r#"{
3810 "apiVersion": "backup.kaniop.rs/v1alpha1",
3811 "kind": "ResultDocument",
3812 "operation": "download",
3813 "success": true,
3814 "exitCode": "success",
3815 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3816 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3817 "payloadSha256": "abc123"
3818 }"#;
3819 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3820 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3821 }
3822
3823 #[test]
3824 fn result_document_verification_accepts_valid_document() {
3825 let result_json = r#"{
3826 "apiVersion": "backup.kaniop.rs/v1alpha1",
3827 "kind": "ResultDocument",
3828 "operation": "upload",
3829 "success": true,
3830 "exitCode": "success",
3831 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3832 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3833 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3834 }"#;
3835 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3836 assert!(result.success);
3837 assert_eq!(
3838 result.exit_code,
3839 kaniop_backup_core::result::ExitCode::Success
3840 );
3841 assert_eq!(result.operation, "upload");
3842 assert!(result.backup_id.is_some());
3843 assert!(result.manifest_key.is_some());
3844 assert!(result.payload_sha256.is_some());
3845 }
3846
3847 #[test]
3848 fn termination_message_path_constant_is_valid() {
3849 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3850 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3851 }
3852
3853 #[test]
3854 fn status_tracks_safety_backup_evidence() {
3855 let status = KanidmRestoreStatus {
3856 phase: KanidmRestorePhase::PreparingSource,
3857 safety_backup_ref: Some("safety-test-restore".to_string()),
3858 safety_backup_expected_backup_id: Some(
3859 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3860 ),
3861 safety_backup_manifest_key: Some(
3862 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3863 ),
3864 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3865 ..Default::default()
3866 };
3867 assert!(status.safety_backup_ref.is_some());
3868 assert!(status.safety_backup_expected_backup_id.is_some());
3869 assert!(status.safety_backup_manifest_key.is_some());
3870 assert!(status.safety_backup_payload_sha256.is_some());
3871 }
3872
3873 #[test]
3874 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3875 let invalid_json = "not valid json";
3876 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3877 assert!(result.is_err());
3878 }
3879
3880 #[test]
3881 fn fail_closed_semantics_preserved_on_oversized_result() {
3882 let oversized_json = format!(
3883 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3884 kaniop_backup_core::result::RESULT_DOC_VERSION,
3885 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3886 );
3887 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3888 assert!(result.is_err());
3889 }
3890
3891 #[test]
3892 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3893 let source = KanidmRestoreSource {
3894 local: None,
3895 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3896 name: "backup-1".to_string(),
3897 }),
3898 };
3899 let mut restore = make_restore(source, None);
3900 restore.metadata.namespace = Some("test-ns".to_string());
3901 let target = super::super::crd::Kanidm::default();
3902 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3903 let doc_str = super::build_safety_upload_operation_doc(
3904 &restore,
3905 &target,
3906 "test-backup-id",
3907 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3908 "my-bucket",
3909 "prod",
3910 "https://s3.example.com",
3911 "us-east-1",
3912 false,
3913 false,
3914 Some(&ca_path),
3915 None,
3916 )
3917 .unwrap();
3918 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3919 assert_eq!(parsed["caBundlePath"], ca_path);
3920 }
3921
3922 #[test]
3923 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3924 let source = KanidmRestoreSource {
3925 local: None,
3926 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3927 name: "backup-1".to_string(),
3928 }),
3929 };
3930 let mut restore = make_restore(source, None);
3931 restore.metadata.namespace = Some("test-ns".to_string());
3932 let target = super::super::crd::Kanidm::default();
3933 let doc_str = super::build_safety_upload_operation_doc(
3934 &restore,
3935 &target,
3936 "test-backup-id",
3937 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3938 "my-bucket",
3939 "prod",
3940 "https://s3.example.com",
3941 "us-east-1",
3942 false,
3943 false,
3944 None,
3945 None,
3946 )
3947 .unwrap();
3948 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3949 assert!(parsed["caBundlePath"].is_null());
3950 }
3951
3952 #[test]
3953 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3954 let source = KanidmRestoreSource {
3955 local: None,
3956 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3957 name: "backup-1".to_string(),
3958 }),
3959 };
3960 let restore = make_restore(source, None);
3961 let target = super::super::crd::Kanidm::default();
3962 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3963 let doc_str = super::build_download_operation_doc(
3964 &restore,
3965 &target,
3966 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3967 "019c7c76-f423-7a12-8f41-2bea7588a303",
3968 "real-bucket",
3969 "real-prefix",
3970 "https://real-endpoint.com",
3971 "eu-west-1",
3972 true,
3973 false,
3974 Some(&ca_path),
3975 None,
3976 );
3977 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3978 assert_eq!(parsed["caBundlePath"], ca_path);
3979 }
3980
3981 #[test]
3982 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3983 let source = KanidmRestoreSource {
3984 local: None,
3985 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3986 name: "backup-1".to_string(),
3987 }),
3988 };
3989 let restore = make_restore(source, None);
3990 let target = super::super::crd::Kanidm::default();
3991 let doc_str = super::build_download_operation_doc(
3992 &restore,
3993 &target,
3994 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3995 "019c7c76-f423-7a12-8f41-2bea7588a303",
3996 "real-bucket",
3997 "real-prefix",
3998 "https://real-endpoint.com",
3999 "eu-west-1",
4000 true,
4001 false,
4002 None,
4003 None,
4004 );
4005 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4006 assert!(parsed["caBundlePath"].is_null());
4007 }
4008
4009 #[test]
4010 fn source_prep_result_operation_is_download() {
4011 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4012 assert_ne!(
4013 super::SOURCE_PREP_RESULT_OPERATION,
4014 super::SAFETY_BACKUP_RESULT_OPERATION
4015 );
4016 }
4017
4018 #[test]
4019 fn download_result_document_verification_accepts_valid_document() {
4020 let result_json = r#"{
4021 "apiVersion": "backup.kaniop.rs/v1alpha1",
4022 "kind": "ResultDocument",
4023 "operation": "download",
4024 "success": true,
4025 "exitCode": "success",
4026 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4027 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4028 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4029 }"#;
4030 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4031 assert!(result.success);
4032 assert_eq!(
4033 result.exit_code,
4034 kaniop_backup_core::result::ExitCode::Success
4035 );
4036 assert_eq!(result.operation, "download");
4037 assert_eq!(
4038 result.backup_id.as_deref(),
4039 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4040 );
4041 assert!(result.manifest_key.is_some());
4042 assert!(result.payload_sha256.is_some());
4043 }
4044
4045 #[test]
4046 fn download_result_document_rejects_wrong_operation() {
4047 let result_json = r#"{
4048 "apiVersion": "backup.kaniop.rs/v1alpha1",
4049 "kind": "ResultDocument",
4050 "operation": "upload",
4051 "success": true,
4052 "exitCode": "success",
4053 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4054 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4055 "payloadSha256": "abc123"
4056 }"#;
4057 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4058 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4059 }
4060
4061 #[test]
4062 fn download_result_document_rejects_failure_success_field() {
4063 let result_json = r#"{
4064 "apiVersion": "backup.kaniop.rs/v1alpha1",
4065 "kind": "ResultDocument",
4066 "operation": "download",
4067 "success": false,
4068 "exitCode": "retryable",
4069 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4070 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4071 "payloadSha256": "abc123"
4072 }"#;
4073 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4074 assert!(!result.success);
4075 assert_ne!(
4076 result.exit_code,
4077 kaniop_backup_core::result::ExitCode::Success
4078 );
4079 }
4080
4081 #[test]
4082 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4083 let status = KanidmRestoreStatus {
4084 phase: KanidmRestorePhase::PreparingSource,
4085 database_mutation_started: false,
4086 ..Default::default()
4087 };
4088 assert!(!status.database_mutation_started);
4089 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4090 }
4091
4092 #[test]
4093 fn backup_job_volume_size_default_is_10gi() {
4094 let qty = crate::controller::backup_job_volume_size();
4095 assert_eq!(qty, Quantity("10Gi".to_string()));
4096 }
4097
4098 #[test]
4099 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4100 use kaniop_backup_core::auth::build_encryption_env_vars;
4101 use kaniop_backup_core::crd::SecretRef;
4102
4103 let key_ref = SecretRef {
4104 name: "kek-secret".to_string(),
4105 };
4106 let env_vars = build_encryption_env_vars(Some(&key_ref));
4107 assert_eq!(env_vars.len(), 1);
4108 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4109 let secret_ref = env_vars[0]
4110 .value_from
4111 .as_ref()
4112 .unwrap()
4113 .secret_key_ref
4114 .as_ref()
4115 .unwrap();
4116 assert_eq!(secret_ref.name, "kek-secret");
4117 assert_eq!(secret_ref.key, "encryption-key");
4118 }
4119
4120 #[test]
4121 fn encryption_env_vars_absent_when_no_key_ref() {
4122 use kaniop_backup_core::auth::build_encryption_env_vars;
4123
4124 let env_vars = build_encryption_env_vars(None);
4125 assert!(
4126 env_vars.is_empty(),
4127 "encryption env vars must be empty when key_ref is None"
4128 );
4129 }
4130
4131 #[test]
4132 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4133 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4134
4135 let source = KanidmRestoreSource {
4136 local: None,
4137 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4138 name: "backup-1".to_string(),
4139 }),
4140 };
4141 let mut restore = make_restore(source, None);
4142 restore.metadata.namespace = Some("test-ns".to_string());
4143 let target = super::super::crd::Kanidm::default();
4144 let encryption = RepositoryEncryption {
4145 mode: EncryptionMode::ClientSide,
4146 key_id: None,
4147 key_ref: Some(SecretRef {
4148 name: "kek-secret".to_string(),
4149 }),
4150 };
4151 let doc_str = super::build_safety_upload_operation_doc(
4152 &restore,
4153 &target,
4154 "test-backup-id",
4155 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4156 "my-bucket",
4157 "prod",
4158 "https://s3.example.com",
4159 "us-east-1",
4160 false,
4161 false,
4162 None,
4163 Some(&encryption),
4164 )
4165 .unwrap();
4166 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4167 assert_eq!(parsed["encryptionMode"], "clientSide");
4168 assert!(parsed["encryptionKeyId"].is_null());
4169 }
4170
4171 #[test]
4172 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4173 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4174
4175 let source = KanidmRestoreSource {
4176 local: None,
4177 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4178 name: "backup-1".to_string(),
4179 }),
4180 };
4181 let restore = make_restore(source, None);
4182 let target = super::super::crd::Kanidm::default();
4183 let encryption = RepositoryEncryption {
4184 mode: EncryptionMode::ProviderKms,
4185 key_id: Some("alias/kaniop-backups".to_string()),
4186 key_ref: None,
4187 };
4188 let doc_str = super::build_download_operation_doc(
4189 &restore,
4190 &target,
4191 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4192 "019c7c76-f423-7a12-8f41-2bea7588a303",
4193 "real-bucket",
4194 "real-prefix",
4195 "https://real-endpoint.com",
4196 "eu-west-1",
4197 true,
4198 false,
4199 None,
4200 Some(&encryption),
4201 );
4202 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4203 assert_eq!(parsed["encryptionMode"], "providerKms");
4204 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4205 }
4206}