1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, ca_bundle_env_var, ca_bundle_path,
9};
10use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
11use kaniop_backup_core::image::data_mover_image;
12use kaniop_backup_core::operation::{
13 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
14};
15use kaniop_backup_core::result::{ExitCode, parse_result_document};
16use kaniop_k8s_util::error::{Error, Result};
17
18use std::collections::BTreeSet;
19use std::sync::Arc;
20use std::time::Duration;
21
22use futures::StreamExt;
23use k8s_openapi::api::apps::v1::StatefulSet;
24use k8s_openapi::api::batch::v1::{Job, JobSpec};
25use k8s_openapi::api::core::v1::{
26 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
27 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
28 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
29};
30use k8s_openapi::api::storage::v1::VolumeAttachment;
31use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
32use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
33use k8s_openapi::jiff::Timestamp;
34use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
35use kube::runtime::controller::{Action, Controller};
36use kube::runtime::events::{Event, EventType, Recorder, Reporter};
37use kube::runtime::finalizer::{Event as Finalizer, finalizer};
38use kube::runtime::watcher;
39use kube::{Api, Client, CustomResource, Resource, ResourceExt};
40use opentelemetry::metrics::{Counter, Histogram};
41use opentelemetry::{KeyValue, global};
42#[cfg(feature = "schemars")]
43use schemars::JsonSchema;
44use serde::{Deserialize, Serialize};
45use serde_json::json;
46use tracing::{debug, error, info, warn};
47
48pub const CONTROLLER_ID: &str = "kanidm-restore";
49pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
50const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
51const DATA_VOLUME: &str = "kanidm-data";
52const STAGING_VOLUME: &str = "restore-staging";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const STAGING_PATH: &str = "/staging";
59const SHARED_VOL_PATH: &str = "/shared";
60const REQUEUE: Duration = Duration::from_secs(2);
61const CONDITION_TRUE: &str = "True";
62const CONDITION_FALSE: &str = "False";
63const CONDITION_PROGRESSING: &str = "Progressing";
64const CONDITION_READY: &str = "Ready";
65const CONDITION_FAILED: &str = "Failed";
66
67pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
68pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
69const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
70const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
71const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
72const SOURCE_PREP_RESULT_OPERATION: &str = "download";
73
74#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
75#[cfg_attr(feature = "schemars", derive(JsonSchema))]
76#[cfg_attr(
77 feature = "schemars",
78 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
79)]
80#[kube(
81 category = "kaniop",
82 group = "kaniop.rs",
83 version = "v1beta1",
84 kind = "KanidmRestore",
85 plural = "kanidmrestores",
86 singular = "kanidmrestore",
87 shortname = "idmrestore",
88 namespaced,
89 status = "KanidmRestoreStatus",
90 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
91 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
92 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
93)]
94#[serde(rename_all = "camelCase")]
95pub struct KanidmRestoreSpec {
96 pub target_ref: KanidmRestoreTargetRef,
97 pub source: KanidmRestoreSource,
98 pub restore_image: String,
99 #[serde(skip_serializing_if = "Option::is_none")]
100 pub safety_backup: Option<SafetyBackupConfig>,
101}
102
103#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
104#[cfg_attr(feature = "schemars", derive(JsonSchema))]
105#[serde(rename_all = "camelCase")]
106pub struct KanidmRestoreTargetRef {
107 pub name: String,
108 pub uid: String,
109}
110
111#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
112#[cfg_attr(feature = "schemars", derive(JsonSchema))]
113#[cfg_attr(
114 feature = "schemars",
115 schemars(extend("x-kubernetes-validations" = [
116 {
117 "message": "exactly one of local or backupRef must be set",
118 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
119 }
120 ]))
121)]
122#[serde(rename_all = "camelCase")]
123pub struct KanidmRestoreSource {
124 #[serde(skip_serializing_if = "Option::is_none")]
126 pub local: Option<KanidmRestoreLocalSource>,
127 #[serde(skip_serializing_if = "Option::is_none")]
129 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
130}
131
132#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
133#[cfg_attr(feature = "schemars", derive(JsonSchema))]
134#[serde(rename_all = "camelCase")]
135pub struct KanidmRestoreLocalSource {
136 pub file_name: String,
138}
139
140#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
141#[cfg_attr(feature = "schemars", derive(JsonSchema))]
142#[serde(rename_all = "camelCase")]
143pub struct KanidmRestoreBackupRefSource {
144 pub name: String,
146}
147
148#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
149#[cfg_attr(feature = "schemars", derive(JsonSchema))]
150#[serde(rename_all = "camelCase")]
151pub struct SafetyBackupConfig {
152 #[serde(skip_serializing_if = "Option::is_none")]
154 pub repository_ref: Option<SafetyBackupRepositoryRef>,
155 #[serde(default)]
157 pub skip: bool,
158}
159
160#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
161#[cfg_attr(feature = "schemars", derive(JsonSchema))]
162#[serde(rename_all = "camelCase")]
163pub struct SafetyBackupRepositoryRef {
164 pub name: String,
165}
166
167#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
168#[cfg_attr(feature = "schemars", derive(JsonSchema))]
169pub enum KanidmRestorePhase {
170 #[default]
171 Pending,
172 Validating,
173 Quiescing,
174 SafetyBackup,
175 PreparingSource,
176 RestoringPrimary,
177 Verifying,
178 RebuildingReplicas,
179 Resuming,
180 Completed,
181 Failed,
182}
183
184#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
185#[cfg_attr(feature = "schemars", derive(JsonSchema))]
186#[serde(rename_all = "camelCase")]
187pub struct KanidmRestoreStatus {
188 #[serde(skip_serializing_if = "Option::is_none")]
189 pub observed_generation: Option<i64>,
190 #[serde(skip_serializing_if = "Option::is_none")]
191 pub observed_target_uid: Option<String>,
192 #[serde(default)]
193 pub phase: KanidmRestorePhase,
194 #[serde(skip_serializing_if = "Option::is_none")]
195 pub restore_job_name: Option<String>,
196 #[serde(skip_serializing_if = "Option::is_none")]
197 pub verify_job_name: Option<String>,
198 #[serde(skip_serializing_if = "Option::is_none")]
199 pub safety_backup_job_name: Option<String>,
200 #[serde(skip_serializing_if = "Option::is_none")]
201 pub source_prep_job_name: Option<String>,
202 #[serde(skip_serializing_if = "Option::is_none")]
203 pub safety_backup_ref: Option<String>,
204 #[serde(skip_serializing_if = "Option::is_none")]
205 pub safety_backup_expected_backup_id: Option<String>,
206 #[serde(skip_serializing_if = "Option::is_none")]
207 pub safety_backup_manifest_key: Option<String>,
208 #[serde(skip_serializing_if = "Option::is_none")]
209 pub safety_backup_payload_sha256: Option<String>,
210 #[serde(default)]
211 pub replicas_cleared: bool,
212 #[serde(default)]
213 pub certificates_cleared: bool,
214 #[serde(default)]
215 pub database_mutation_started: bool,
216 #[serde(default, skip_serializing_if = "Vec::is_empty")]
217 pub original_replicas: Vec<ReplicaCountEntry>,
218 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
219 pub phase_timestamps: std::collections::BTreeMap<String, String>,
220 #[serde(default, skip_serializing_if = "Vec::is_empty")]
221 #[cfg_attr(
222 feature = "schemars",
223 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
224 )]
225 pub conditions: Vec<Condition>,
226 #[serde(skip_serializing_if = "Option::is_none")]
227 pub message: Option<String>,
228}
229
230#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
231#[cfg_attr(feature = "schemars", derive(JsonSchema))]
232#[serde(rename_all = "camelCase")]
233pub struct ReplicaCountEntry {
234 pub group: String,
235 pub replicas: i32,
236}
237
238#[derive(Clone)]
239struct RestoreMetrics {
240 attempts: Counter<u64>,
241 outcomes: Counter<u64>,
242 duration_seconds: Histogram<f64>,
243 break_glass_total: Counter<u64>,
244 safety_backup_duration_seconds: Histogram<f64>,
245}
246
247impl RestoreMetrics {
248 fn new() -> Self {
249 let meter = global::meter("kaniop");
250 Self {
251 attempts: meter
252 .u64_counter("kaniop_restore_attempts_total")
253 .with_description("Number of Kanidm restore attempts started")
254 .build(),
255 outcomes: meter
256 .u64_counter("kaniop_restore_outcomes_total")
257 .with_description("Number of terminal Kanidm restore outcomes")
258 .build(),
259 duration_seconds: meter
260 .f64_histogram("kaniop_restore_duration_seconds")
261 .with_description("Kanidm restore duration from object creation to terminal phase")
262 .with_unit("s")
263 .build(),
264 break_glass_total: meter
265 .u64_counter("kaniop_restore_break_glass_total")
266 .with_description("Number of break-glass safety backup overrides")
267 .build(),
268 safety_backup_duration_seconds: meter
269 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
270 .with_description("Duration of pre-restore safety backup creation")
271 .with_unit("s")
272 .build(),
273 }
274 }
275}
276
277#[derive(Clone)]
278struct RestoreContext {
279 client: Client,
280 recorder: Recorder,
281 metrics: RestoreMetrics,
282}
283
284pub async fn run(client: Client) {
285 let api = Api::<KanidmRestore>::all(client.clone());
286 let recorder = Recorder::new(
287 client.clone(),
288 Reporter {
289 controller: CONTROLLER_ID.into(),
290 instance: None,
291 },
292 );
293 let ctx = Arc::new(RestoreContext {
294 client,
295 recorder,
296 metrics: RestoreMetrics::new(),
297 });
298 info!("starting {CONTROLLER_ID} controller");
299 Controller::new(api, watcher::Config::default().any_semantic())
300 .shutdown_on_signal()
301 .run(reconcile_restore, error_policy, ctx)
302 .for_each(|result| async move {
303 if let Err(error) = result {
304 error!(%error, "KanidmRestore reconciliation failed");
305 }
306 })
307 .await;
308}
309
310fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
311 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
312 Action::requeue(Duration::from_secs(5))
313}
314
315async fn reconcile_restore(
316 restore: Arc<KanidmRestore>,
317 ctx: Arc<RestoreContext>,
318) -> Result<Action> {
319 let namespace = restore
320 .namespace()
321 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
322 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
323 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
324 let ctx = ctx.clone();
325 async move {
326 match event {
327 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
328 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
329 }
330 }
331 })
332 .await
333 .map_err(|error| {
334 Error::FinalizerError(
335 "failed on KanidmRestore finalizer".to_string(),
336 Box::new(error),
337 )
338 })
339}
340
341async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
342 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
343 match phase {
344 KanidmRestorePhase::Pending => {
345 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
346 Ok(Action::requeue(REQUEUE))
347 }
348 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
349 Ok(target) => {
350 mark_restoring(&restore, &target, &ctx).await?;
351 let mut status = restore.status.clone().unwrap_or_default();
352 status.observed_target_uid = target.uid();
353 status.original_replicas = capture_original_replicas(&target);
354 status.phase = KanidmRestorePhase::Quiescing;
355 status.message = None;
356 patch_status(&restore, &ctx, status).await?;
357 Ok(Action::requeue(REQUEUE))
358 }
359 Err(error) => {
360 set_phase(
361 &restore,
362 &ctx,
363 KanidmRestorePhase::Failed,
364 Some(error.to_string()),
365 )
366 .await?;
367 Ok(Action::requeue(Duration::from_secs(300)))
368 }
369 },
370 KanidmRestorePhase::Quiescing => {
371 let target = get_target(&restore, &ctx).await?;
372 scale_all(&target, &ctx, 0).await?;
373 if target_pods_stopped(&target, &ctx).await?
374 && target_volumes_detached(&target, &ctx).await?
375 {
376 let next_phase = if requires_safety_backup(&restore) {
377 KanidmRestorePhase::SafetyBackup
378 } else {
379 record_break_glass(&restore, &ctx).await?;
380 KanidmRestorePhase::PreparingSource
381 };
382 if next_phase == KanidmRestorePhase::PreparingSource {
383 let refreshed = Api::<KanidmRestore>::namespaced(
384 ctx.client.clone(),
385 restore.namespace().as_deref().unwrap_or_default(),
386 )
387 .get(&restore.name_any())
388 .await
389 .map_err(|error| {
390 Error::kube_error(
391 "get",
392 "KanidmRestore",
393 restore.namespace().as_deref().unwrap_or_default(),
394 restore.name_any(),
395 error,
396 )
397 })?;
398 set_phase(&refreshed, &ctx, next_phase, None).await?;
399 } else {
400 set_phase(&restore, &ctx, next_phase, None).await?;
401 }
402 }
403 Ok(Action::requeue(REQUEUE))
404 }
405 KanidmRestorePhase::SafetyBackup => {
406 let target = get_target(&restore, &ctx).await?;
407 let name = safety_backup_job_name(&restore);
408 let expected_backup_id = compute_safety_backup_id(&restore);
409 let expected_manifest_key = compute_safety_manifest_key(&restore);
410 ensure_safety_backup_job(
411 &restore,
412 &target,
413 &ctx,
414 &name,
415 &expected_backup_id,
416 &expected_manifest_key,
417 )
418 .await?;
419 let mut status = restore.status.clone().unwrap_or_default();
420 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
421 match job_state(&restore, &ctx, &name).await? {
422 JobState::Complete => {
423 let result =
424 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
425 match result {
426 Ok(verified) => {
427 status.safety_backup_job_name = Some(name);
428 status.safety_backup_ref =
429 Some(format!("safety-{}", restore.name_any()));
430 status.safety_backup_manifest_key = Some(verified.manifest_key);
431 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
432 record_safety_backup_duration(&restore, &ctx).await;
433 status.phase = KanidmRestorePhase::PreparingSource;
434 status.message = None;
435 patch_status(&restore, &ctx, status).await?;
436 }
437 Err(error) => {
438 resume_before_mutation(&restore, &ctx).await?;
439 set_phase(
440 &restore,
441 &ctx,
442 KanidmRestorePhase::Failed,
443 Some(format!("safety backup result verification failed: {error}")),
444 )
445 .await?;
446 }
447 }
448 }
449 JobState::Failed => {
450 resume_before_mutation(&restore, &ctx).await?;
451 set_phase(
452 &restore,
453 &ctx,
454 KanidmRestorePhase::Failed,
455 Some(
456 "safety backup job failed; target restored to original state"
457 .to_string(),
458 ),
459 )
460 .await?;
461 }
462 JobState::Running => {
463 patch_status(&restore, &ctx, status).await?;
464 }
465 }
466 Ok(Action::requeue(REQUEUE))
467 }
468 KanidmRestorePhase::PreparingSource => {
469 if is_remote_source(&restore) {
470 let target = get_target(&restore, &ctx).await?;
471 let name = source_prep_job_name(&restore);
472 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
473 match job_state(&restore, &ctx, &name).await? {
474 JobState::Complete => {
475 let backup_id = restore
476 .spec
477 .source
478 .backup_ref
479 .as_ref()
480 .map(|b| b.name.clone())
481 .unwrap_or_default();
482 let backup = {
483 let ns = restore.namespace().unwrap();
484 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
485 .get(&backup_id)
486 .await
487 .map_err(|e| {
488 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
489 })?
490 };
491 let expected_backup_id = &backup.spec.backup_id;
492 let result =
493 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
494 .await;
495 match result {
496 Ok(_verified) => {
497 let mut status = restore.status.clone().unwrap_or_default();
498 status.source_prep_job_name = Some(name);
499 status.phase = KanidmRestorePhase::RestoringPrimary;
500 status.message = None;
501 patch_status(&restore, &ctx, status).await?;
502 }
503 Err(error) => {
504 resume_before_mutation(&restore, &ctx).await?;
505 set_phase(
506 &restore,
507 &ctx,
508 KanidmRestorePhase::Failed,
509 Some(format!(
510 "source preparation result verification failed: {error}"
511 )),
512 )
513 .await?;
514 }
515 }
516 }
517 JobState::Failed => {
518 resume_before_mutation(&restore, &ctx).await?;
519 set_phase(
520 &restore,
521 &ctx,
522 KanidmRestorePhase::Failed,
523 Some(
524 "source preparation job failed; target restored to original state"
525 .to_string(),
526 ),
527 )
528 .await?;
529 }
530 JobState::Running => {}
531 }
532 } else {
533 let target = get_target(&restore, &ctx).await?;
534 let name = source_check_job_name(&restore);
535 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
536 match job_state(&restore, &ctx, &name).await? {
537 JobState::Complete => {
538 let mut status = restore.status.clone().unwrap_or_default();
539 status.source_prep_job_name = Some(name);
540 status.phase = KanidmRestorePhase::RestoringPrimary;
541 status.message = None;
542 patch_status(&restore, &ctx, status).await?;
543 }
544 JobState::Failed => {
545 resume_before_mutation(&restore, &ctx).await?;
546 set_phase(
547 &restore,
548 &ctx,
549 KanidmRestorePhase::Failed,
550 Some(
551 "local backup file check failed; target restored to original state"
552 .to_string(),
553 ),
554 )
555 .await?;
556 }
557 JobState::Running => {}
558 }
559 }
560 Ok(Action::requeue(REQUEUE))
561 }
562 KanidmRestorePhase::RestoringPrimary => {
563 let target = get_target(&restore, &ctx).await?;
564 let status = restore.status.as_ref().cloned().unwrap_or_default();
565 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
566 return Err(Error::MissingData(
567 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
568 ));
569 }
570 if !status.database_mutation_started {
571 let mut status = status;
572 status.database_mutation_started = true;
573 status.message = Some(
574 "database mutation boundary persisted; starting primary restore".to_string(),
575 );
576 patch_status(&restore, &ctx, status).await?;
577 return Ok(Action::requeue(REQUEUE));
578 }
579 let name = restore_job_name(&restore);
580 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
581 match job_state(&restore, &ctx, &name).await? {
582 JobState::Complete => {
583 let mut status = restore.status.clone().unwrap_or_default();
584 status.restore_job_name = Some(name);
585 status.phase = KanidmRestorePhase::Verifying;
586 status.message = None;
587 patch_status(&restore, &ctx, status).await?;
588 }
589 JobState::Failed => {
590 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
591 }
592 JobState::Running => {}
593 }
594 Ok(Action::requeue(REQUEUE))
595 }
596 KanidmRestorePhase::Verifying => {
597 let target = get_target(&restore, &ctx).await?;
598 let name = verify_job_name(&restore);
599 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
600 match job_state(&restore, &ctx, &name).await? {
601 JobState::Complete => {
602 let mut status = restore.status.clone().unwrap_or_default();
603 status.verify_job_name = Some(name);
604 status.phase = KanidmRestorePhase::RebuildingReplicas;
605 status.message = None;
606 patch_status(&restore, &ctx, status).await?;
607 }
608 JobState::Failed => {
609 fail_after_mutation(&restore, &ctx, "database verification failed").await?
610 }
611 JobState::Running => {}
612 }
613 Ok(Action::requeue(REQUEUE))
614 }
615 KanidmRestorePhase::RebuildingReplicas => {
616 let target = get_target(&restore, &ctx).await?;
617 let mut status = restore.status.clone().unwrap_or_default();
618 if !status.replicas_cleared {
619 if delete_secondary_pvcs(&target, &ctx).await? {
620 status.replicas_cleared = true;
621 status.message = Some("secondary database state cleared".to_string());
622 patch_status(&restore, &ctx, status).await?;
623 }
624 return Ok(Action::requeue(REQUEUE));
625 }
626
627 scale_primary(&target, &ctx, 1).await?;
628 if !primary_ready(&target, &ctx).await? {
629 return Ok(Action::requeue(REQUEUE));
630 }
631 if !status.certificates_cleared {
632 delete_replica_cert_secrets(&target, &ctx).await?;
633 delete_admin_secret(&target, &ctx).await?;
634 status.certificates_cleared = true;
635 status.message = Some(
636 "replica certificates and admin secret cleared for regeneration".to_string(),
637 );
638 patch_status(&restore, &ctx, status).await?;
639 return Ok(Action::requeue(REQUEUE));
640 }
641 status.phase = KanidmRestorePhase::Resuming;
642 status.message = None;
643 patch_status(&restore, &ctx, status).await?;
644 Ok(Action::requeue(REQUEUE))
645 }
646 KanidmRestorePhase::Resuming => {
647 let target = get_target(&restore, &ctx).await?;
648 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
649 clear_restoring(&restore, &target, &ctx).await?;
650 Ok(Action::requeue(Duration::from_secs(3600)))
651 }
652 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
653 Ok(Action::requeue(Duration::from_secs(3600)))
654 }
655 }
656}
657
658async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
659 let status = restore.status.clone().unwrap_or_default();
660 if status.database_mutation_started
661 && status.phase != KanidmRestorePhase::Completed
662 && status.phase != KanidmRestorePhase::Failed
663 {
664 reconcile_apply(restore.clone(), ctx.clone()).await?;
665 return Ok(Action::requeue(REQUEUE));
666 }
667 if let Ok(target) = get_target(&restore, &ctx).await {
668 let owns_maintenance =
669 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
670 if owns_maintenance && !status.database_mutation_started {
671 scale_desired(&target, &ctx).await?;
672 }
673 clear_restoring(&restore, &target, &ctx).await?;
674 }
675 Ok(Action::await_change())
676}
677
678async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
679 let target = get_target(restore, ctx).await?;
680 let actual_uid = target
681 .uid()
682 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
683 if actual_uid != restore.spec.target_ref.uid {
684 return Err(Error::MissingData(format!(
685 "target UID mismatch: expected {}, got {}",
686 restore.spec.target_ref.uid, actual_uid
687 )));
688 }
689 validate_source(restore)?;
690 if is_remote_source(restore) {
691 validate_backup_ref(restore, &target, ctx).await?;
692 }
693 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
694 {
695 return Err(Error::MissingData(format!(
696 "restoreImage must be the target's pinned Kanidm image (target image is {})",
697 target.spec.image
698 )));
699 }
700 let storage = target
701 .spec
702 .storage
703 .as_ref()
704 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
705 if storage.empty_dir.is_some()
706 || storage.ephemeral.is_some()
707 || storage.volume_claim_template.is_none()
708 {
709 return Err(Error::MissingData(
710 "restore requires PVC-backed Kanidm storage".to_string(),
711 ));
712 }
713 let primaries = target
714 .spec
715 .replica_groups
716 .iter()
717 .filter(|rg| rg.primary_node)
718 .count();
719 if primaries != 1 {
720 return Err(Error::MissingData(
721 "backup/restore requires exactly one primary replica group".to_string(),
722 ));
723 }
724 let ns = restore.namespace().unwrap();
725 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
726 .list(&ListParams::default())
727 .await
728 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
729 if restores.items.iter().any(|other| {
730 other.name_any() != restore.name_any()
731 && other.spec.target_ref.name == restore.spec.target_ref.name
732 && !matches!(
733 other.status.as_ref().map(|s| s.phase),
734 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
735 )
736 }) {
737 return Err(Error::MissingData(
738 "another active restore targets this Kanidm".to_string(),
739 ));
740 }
741 validate_safety_backup_config(restore)?;
742 Ok(target)
743}
744
745fn safe_basename(name: &str) -> bool {
746 !name.is_empty()
747 && name != "."
748 && name != ".."
749 && !name.contains('/')
750 && !name.contains('\\')
751 && !name.contains('\0')
752}
753
754fn mutable_image(image: &str) -> bool {
755 image == "kanidm/server:latest"
756 || image.ends_with(":latest")
757 || (!image.contains('@')
758 && !image
759 .rsplit('/')
760 .next()
761 .is_some_and(|part| part.contains(':')))
762}
763
764async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
765 let ns = restore
766 .namespace()
767 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
768 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
769 .get(&restore.spec.target_ref.name)
770 .await
771 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
772}
773
774async fn mark_restoring(
775 restore: &KanidmRestore,
776 target: &Kanidm,
777 ctx: &RestoreContext,
778) -> Result<()> {
779 let ns = target.namespace().unwrap();
780 let uid = restore
781 .uid()
782 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
783 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
784 .patch(
785 &target.name_any(),
786 &PatchParams::default(),
787 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
788 )
789 .await
790 .map(|_| ())
791 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
792}
793
794async fn clear_restoring(
795 restore: &KanidmRestore,
796 target: &Kanidm,
797 ctx: &RestoreContext,
798) -> Result<()> {
799 let ns = target.namespace().unwrap();
800 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
801 return Ok(());
802 }
803 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
804 .patch(
805 &target.name_any(),
806 &PatchParams::default(),
807 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
808 )
809 .await
810 .map(|_| ())
811 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
812}
813
814async fn patch_status(
815 restore: &KanidmRestore,
816 ctx: &RestoreContext,
817 mut status: KanidmRestoreStatus,
818) -> Result<()> {
819 let ns = restore.namespace().unwrap();
820 let previous_phase = restore.status.as_ref().map(|current| current.phase);
821 let next_phase = status.phase;
822 status.observed_generation = restore.metadata.generation;
823 update_restore_conditions(&mut status, restore.metadata.generation);
824
825 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
826 .patch_status(
827 &restore.name_any(),
828 &PatchParams::default(),
829 &Patch::Merge(json!({"status": &status})),
830 )
831 .await
832 .map_err(|e| {
833 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
834 })?;
835
836 if previous_phase != Some(next_phase) {
837 record_restore_transition(
838 restore,
839 ctx,
840 previous_phase,
841 next_phase,
842 status.message.as_deref(),
843 )
844 .await;
845 }
846 Ok(())
847}
848
849fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
850 let previous = status.conditions.clone();
851 let phase = status.phase;
852 let phase_reason = format!("{phase:?}");
853 let message = status
854 .message
855 .clone()
856 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
857 let terminal_success = phase == KanidmRestorePhase::Completed;
858 let terminal_failure = phase == KanidmRestorePhase::Failed;
859 let progressing = !terminal_success && !terminal_failure;
860
861 let break_glass = previous
862 .iter()
863 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
864 .cloned();
865 status.conditions = vec![
866 restore_condition(
867 &previous,
868 CONDITION_PROGRESSING,
869 if progressing {
870 CONDITION_TRUE
871 } else {
872 CONDITION_FALSE
873 },
874 &phase_reason,
875 &message,
876 generation,
877 ),
878 restore_condition(
879 &previous,
880 CONDITION_READY,
881 if terminal_success {
882 CONDITION_TRUE
883 } else {
884 CONDITION_FALSE
885 },
886 if terminal_success {
887 "RestoreCompleted"
888 } else {
889 "RestoreNotCompleted"
890 },
891 if terminal_success {
892 "Kanidm restore completed successfully."
893 } else {
894 "Kanidm restore has not completed successfully."
895 },
896 generation,
897 ),
898 restore_condition(
899 &previous,
900 CONDITION_FAILED,
901 if terminal_failure {
902 CONDITION_TRUE
903 } else {
904 CONDITION_FALSE
905 },
906 if terminal_failure {
907 "RestoreFailed"
908 } else {
909 "NoRestoreFailure"
910 },
911 if terminal_failure {
912 status
913 .message
914 .as_deref()
915 .unwrap_or("Kanidm restore failed.")
916 } else {
917 "No terminal restore failure has been recorded."
918 },
919 generation,
920 ),
921 ];
922 if let Some(condition) = break_glass {
923 status.conditions.push(condition);
924 }
925}
926
927fn restore_condition(
928 previous: &[Condition],
929 condition_type: &str,
930 condition_status: &str,
931 reason: &str,
932 message: &str,
933 generation: Option<i64>,
934) -> Condition {
935 let last_transition_time = previous
936 .iter()
937 .find(|condition| {
938 condition.type_ == condition_type
939 && condition.status == condition_status
940 && condition.reason == reason
941 })
942 .map(|condition| condition.last_transition_time.clone())
943 .unwrap_or_else(|| Time(Timestamp::now()));
944 Condition {
945 type_: condition_type.to_string(),
946 status: condition_status.to_string(),
947 reason: reason.to_string(),
948 message: message.to_string(),
949 last_transition_time,
950 observed_generation: generation,
951 }
952}
953
954async fn record_restore_transition(
955 restore: &KanidmRestore,
956 ctx: &RestoreContext,
957 previous_phase: Option<KanidmRestorePhase>,
958 phase: KanidmRestorePhase,
959 message: Option<&str>,
960) {
961 if phase == KanidmRestorePhase::Validating {
962 ctx.metrics.attempts.add(1, &[]);
963 }
964
965 let result = match phase {
966 KanidmRestorePhase::Completed => Some("success"),
967 KanidmRestorePhase::Failed => Some("failure"),
968 _ => None,
969 };
970 if let Some(result) = result {
971 let attributes = [KeyValue::new("result", result)];
972 ctx.metrics.outcomes.add(1, &attributes);
973 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
974 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
975 ctx.metrics.duration_seconds.record(elapsed, &attributes);
976 }
977 }
978
979 let reason = if phase == KanidmRestorePhase::Failed {
980 "RestoreFailed"
981 } else {
982 "RestorePhaseChanged"
983 };
984 let note = message.map(str::to_string).or_else(|| {
985 Some(format!(
986 "Kanidm restore phase changed from {} to {phase:?}.",
987 previous_phase
988 .map(|previous| format!("{previous:?}"))
989 .unwrap_or_else(|| "None".to_string())
990 ))
991 });
992 if let Err(error) = ctx
993 .recorder
994 .publish(
995 &Event {
996 type_: if phase == KanidmRestorePhase::Failed {
997 EventType::Warning
998 } else {
999 EventType::Normal
1000 },
1001 reason: reason.to_string(),
1002 note,
1003 action: "Restore".to_string(),
1004 secondary: None,
1005 },
1006 &restore.object_ref(&()),
1007 )
1008 .await
1009 {
1010 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1011 }
1012}
1013
1014async fn set_phase(
1015 restore: &KanidmRestore,
1016 ctx: &RestoreContext,
1017 phase: KanidmRestorePhase,
1018 message: Option<String>,
1019) -> Result<()> {
1020 let mut status = restore.status.clone().unwrap_or_default();
1021 status.phase = phase;
1022 status.message = message;
1023 patch_status(restore, ctx, status).await
1024}
1025
1026async fn fail_after_mutation(
1027 restore: &KanidmRestore,
1028 ctx: &RestoreContext,
1029 message: &str,
1030) -> Result<()> {
1031 set_phase(
1032 restore,
1033 ctx,
1034 KanidmRestorePhase::Failed,
1035 Some(message.to_string()),
1036 )
1037 .await
1038}
1039
1040fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1041 target
1042 .spec
1043 .replica_groups
1044 .iter()
1045 .find(|rg| rg.primary_node)
1046 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1047}
1048
1049async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1050 let ns = target.namespace().unwrap();
1051 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1052 for rg in &target.spec.replica_groups {
1053 let name = target.statefulset_name(&rg.name);
1054 api.patch(
1055 &name,
1056 &PatchParams::default(),
1057 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1058 )
1059 .await
1060 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1061 }
1062 Ok(())
1063}
1064
1065async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1066 let ns = target.namespace().unwrap();
1067 let rg = primary_group(target)?;
1068 let name = target.statefulset_name(&rg.name);
1069 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1070 .patch(
1071 &name,
1072 &PatchParams::default(),
1073 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1074 )
1075 .await
1076 .map(|_| ())
1077 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1078}
1079
1080async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1081 let ns = target.namespace().unwrap();
1082 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1083 for rg in &target.spec.replica_groups {
1084 let name = target.statefulset_name(&rg.name);
1085 api.patch(
1086 &name,
1087 &PatchParams::default(),
1088 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1089 )
1090 .await
1091 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1092 }
1093 Ok(())
1094}
1095
1096async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1097 let ns = target.namespace().unwrap();
1098 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1099 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1100 .await
1101 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1102 Ok(pods.items.is_empty())
1103}
1104
1105async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1106 let ns = target.namespace().unwrap();
1107 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1108 let mut pv_names = BTreeSet::new();
1109 let primary_name = primary_pvc_name(target)?;
1110 let primary_pvc = pvc_api
1111 .get(&primary_name)
1112 .await
1113 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1114 let primary_pv = primary_pvc
1115 .spec
1116 .as_ref()
1117 .and_then(|spec| spec.volume_name.clone())
1118 .ok_or_else(|| {
1119 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1120 })?;
1121 pv_names.insert(primary_pv);
1122
1123 for rg in &target.spec.replica_groups {
1124 let sts = target.statefulset_name(&rg.name);
1125 for ordinal in 0..rg.replicas {
1126 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1127 if name == primary_name {
1128 continue;
1129 }
1130 if let Some(pvc) = pvc_api
1131 .get_opt(&name)
1132 .await
1133 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1134 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1135 {
1136 pv_names.insert(pv_name);
1137 }
1138 }
1139 }
1140
1141 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1142 .list(&ListParams::default())
1143 .await
1144 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1145 Ok(!attachments.items.iter().any(|attachment| {
1146 attachment
1147 .spec
1148 .source
1149 .persistent_volume_name
1150 .as_ref()
1151 .is_some_and(|pv_name| pv_names.contains(pv_name))
1152 }))
1153}
1154
1155async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1156 let ns = target.namespace().unwrap();
1157 let name = target.statefulset_name(&primary_group(target)?.name);
1158 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1159 .get(&name)
1160 .await
1161 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1162 Ok(sts
1163 .status
1164 .as_ref()
1165 .and_then(|s| s.ready_replicas)
1166 .unwrap_or(0)
1167 >= 1)
1168}
1169
1170fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1171 let rg = primary_group(target)?;
1172 Ok(format!(
1173 "{DATA_VOLUME}-{}-0",
1174 target.statefulset_name(&rg.name)
1175 ))
1176}
1177
1178async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1179 let ns = target.namespace().unwrap();
1180 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1181 let mut all_absent = true;
1182 for rg in &target.spec.replica_groups {
1183 let sts = target.statefulset_name(&rg.name);
1184 for ordinal in 0..rg.replicas {
1185 if rg.primary_node && ordinal == 0 {
1186 continue;
1187 }
1188 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1189 match api
1190 .get_opt(&name)
1191 .await
1192 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1193 {
1194 None => {}
1195 Some(pvc) => {
1196 all_absent = false;
1197 if pvc.metadata.deletion_timestamp.is_none() {
1198 match api.delete(&name, &DeleteParams::default()).await {
1199 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1200 Err(kube::Error::Api(status)) if status.code == 404 => {}
1201 Err(error) => {
1202 return Err(Error::kube_error(
1203 "delete",
1204 "PersistentVolumeClaim",
1205 &ns,
1206 &name,
1207 error,
1208 ));
1209 }
1210 }
1211 }
1212 }
1213 }
1214 }
1215 }
1216 Ok(all_absent)
1217}
1218
1219async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1220 let ns = target.namespace().unwrap();
1221 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1222 let label_selector = format!(
1223 "{}={},{}={}",
1224 CLUSTER_LABEL,
1225 target.name_any(),
1226 SECRET_TYPE_LABEL,
1227 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1228 );
1229 let secrets = api
1230 .list(&ListParams::default().labels(&label_selector))
1231 .await
1232 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1233 let mut all_absent = secrets.items.is_empty();
1234 for secret in secrets.items {
1235 let name = secret.name_any();
1236 if secret.metadata.deletion_timestamp.is_none() {
1237 match api.delete(&name, &DeleteParams::default()).await {
1238 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1239 Err(kube::Error::Api(status)) if status.code == 404 => {}
1240 Err(error) => {
1241 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1242 }
1243 }
1244 all_absent = false;
1245 }
1246 }
1247 Ok(all_absent)
1248}
1249
1250async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1251 let ns = target.namespace().unwrap();
1252 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1253 let name = format!("{}-admin-passwords", target.name_any());
1254 match api
1255 .get_opt(&name)
1256 .await
1257 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1258 {
1259 None => Ok(true),
1260 Some(secret) => {
1261 if secret.metadata.deletion_timestamp.is_some() {
1262 return Ok(false);
1263 }
1264 match api.delete(&name, &DeleteParams::default()).await {
1265 Ok(_) => {
1266 debug!(secret = %name, "deleting stale admin secret");
1267 Ok(false)
1268 }
1269 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1270 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1271 }
1272 }
1273 }
1274}
1275
1276fn restore_job_name(restore: &KanidmRestore) -> String {
1277 format!("{}-restore", restore.name_any())
1278}
1279fn verify_job_name(restore: &KanidmRestore) -> String {
1280 format!("{}-verify", restore.name_any())
1281}
1282
1283fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1284 let origin = target
1285 .spec
1286 .origin
1287 .clone()
1288 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1289 vec![
1290 EnvVar {
1291 name: "KANIDM_DB_PATH".to_string(),
1292 value: Some("/data/kanidm.db".to_string()),
1293 ..Default::default()
1294 },
1295 EnvVar {
1296 name: "KANIDM_DOMAIN".to_string(),
1297 value: Some(target.spec.domain.clone()),
1298 ..Default::default()
1299 },
1300 EnvVar {
1301 name: "KANIDM_ORIGIN".to_string(),
1302 value: Some(origin),
1303 ..Default::default()
1304 },
1305 EnvVar {
1306 name: "KANIDM_TLS_CHAIN".to_string(),
1307 value: Some(format!("{TLS_PATH}/tls.crt")),
1308 ..Default::default()
1309 },
1310 EnvVar {
1311 name: "KANIDM_TLS_KEY".to_string(),
1312 value: Some(format!("{TLS_PATH}/tls.key")),
1313 ..Default::default()
1314 },
1315 ]
1316}
1317
1318async fn ensure_database_job(
1319 restore: &KanidmRestore,
1320 target: &Kanidm,
1321 ctx: &RestoreContext,
1322 name: &str,
1323 verify: bool,
1324) -> Result<()> {
1325 let ns = restore.namespace().unwrap();
1326 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1327 if api
1328 .get_opt(name)
1329 .await
1330 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1331 .is_some()
1332 {
1333 return Ok(());
1334 }
1335 let mut command = vec![
1336 "kanidmd".to_string(),
1337 "database".to_string(),
1338 if verify { "backup" } else { "restore" }.to_string(),
1339 ];
1340 if verify {
1341 command.push("/verify/verification.json.gz".to_string());
1342 } else {
1343 if is_remote_source(restore) {
1344 command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1345 } else {
1346 let file_name = restore
1347 .spec
1348 .source
1349 .local
1350 .as_ref()
1351 .map(|l| l.file_name.clone())
1352 .unwrap_or_default();
1353 command.push(format!("{BACKUP_PATH}/{file_name}"));
1354 }
1355 }
1356 let job = Job {
1357 metadata: ObjectMeta {
1358 name: Some(name.to_string()),
1359 namespace: Some(ns.clone()),
1360 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1361 ..Default::default()
1362 },
1363 spec: Some(JobSpec {
1364 backoff_limit: Some(0),
1365 template: PodTemplateSpec {
1366 metadata: Some(ObjectMeta::default()),
1367 spec: Some(PodSpec {
1368 automount_service_account_token: Some(false),
1369 restart_policy: Some("Never".to_string()),
1370 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1371 seccomp_profile: Some(SeccompProfile {
1372 type_: "RuntimeDefault".to_string(),
1373 ..Default::default()
1374 }),
1375 ..Default::default()
1376 }),
1377 containers: vec![Container {
1378 name: if verify { "verify" } else { "restore" }.to_string(),
1379 image: Some(restore.spec.restore_image.clone()),
1380 command: Some(command),
1381 env: Some(build_kanidm_db_env_vars(target)),
1382 security_context: Some(kanidm_job_security_context()),
1383 resources: Some(job_resource_requirements()),
1384 volume_mounts: Some(vec![
1385 VolumeMount {
1386 name: DATA_VOLUME.to_string(),
1387 mount_path: DATA_PATH.to_string(),
1388 ..Default::default()
1389 },
1390 VolumeMount {
1391 name: TLS_VOLUME.to_string(),
1392 mount_path: TLS_PATH.to_string(),
1393 read_only: Some(true),
1394 ..Default::default()
1395 },
1396 VolumeMount {
1397 name: "verification-output".to_string(),
1398 mount_path: "/verify".to_string(),
1399 ..Default::default()
1400 },
1401 ]),
1402 ..Default::default()
1403 }],
1404 volumes: Some(vec![
1405 Volume {
1406 name: DATA_VOLUME.to_string(),
1407 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1408 claim_name: primary_pvc_name(target)?,
1409 read_only: Some(false),
1410 }),
1411 ..Default::default()
1412 },
1413 Volume {
1414 name: TLS_VOLUME.to_string(),
1415 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1416 secret_name: Some(target.effective_tls_secret_name()),
1417 ..Default::default()
1418 }),
1419 ..Default::default()
1420 },
1421 Volume {
1422 name: "verification-output".to_string(),
1423 empty_dir: Some(Default::default()),
1424 ..Default::default()
1425 },
1426 ]),
1427 ..Default::default()
1428 }),
1429 },
1430 ..Default::default()
1431 }),
1432 ..Default::default()
1433 };
1434 api.create(&PostParams::default(), &job)
1435 .await
1436 .map(|_| ())
1437 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1438}
1439
1440enum JobState {
1441 Running,
1442 Complete,
1443 Failed,
1444}
1445
1446async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1447 let ns = restore.namespace().unwrap();
1448 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1449 .get(name)
1450 .await
1451 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1452 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1453 return Ok(JobState::Failed);
1454 }
1455 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1456 return Ok(JobState::Complete);
1457 }
1458 Ok(JobState::Running)
1459}
1460
1461fn is_remote_source(restore: &KanidmRestore) -> bool {
1462 restore.spec.source.backup_ref.is_some()
1463}
1464
1465fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1466 let safety = restore.spec.safety_backup.as_ref();
1467 let skip = safety.is_some_and(|s| s.skip);
1468 !skip
1469}
1470
1471fn validate_source(restore: &KanidmRestore) -> Result<()> {
1472 let has_local = restore.spec.source.local.is_some();
1473 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1474 if has_local && has_backup_ref {
1475 return Err(Error::MissingData(
1476 "source.local and source.backupRef are mutually exclusive".to_string(),
1477 ));
1478 }
1479 if !has_local && !has_backup_ref {
1480 return Err(Error::MissingData(
1481 "source must specify either local or backupRef".to_string(),
1482 ));
1483 }
1484 if let Some(local) = &restore.spec.source.local {
1485 if !safe_basename(&local.file_name) {
1486 return Err(Error::MissingData(
1487 "restore source fileName must be a safe basename".to_string(),
1488 ));
1489 }
1490 }
1491 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1492 if backup_ref.name.is_empty() {
1493 return Err(Error::MissingData(
1494 "source.backupRef.name must not be empty".to_string(),
1495 ));
1496 }
1497 }
1498 Ok(())
1499}
1500
1501fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1502 let safety = restore.spec.safety_backup.as_ref();
1503 let skip = safety.is_some_and(|s| s.skip);
1504 if skip {
1505 let annotations = restore.metadata.annotations.as_ref();
1506 let reason = annotations
1507 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1508 .map(|s| s.trim())
1509 .unwrap_or("");
1510 let approver = annotations
1511 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1512 .map(|s| s.trim())
1513 .unwrap_or("");
1514 if reason.is_empty() {
1515 return Err(Error::MissingData(format!(
1516 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1517 )));
1518 }
1519 if approver.is_empty() {
1520 return Err(Error::MissingData(format!(
1521 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1522 )));
1523 }
1524 }
1525 if is_remote_source(restore) && !skip {
1526 let has_repo = safety
1527 .and_then(|s| s.repository_ref.as_ref())
1528 .is_some_and(|r| !r.name.is_empty());
1529 if !has_repo {
1530 return Err(Error::MissingData(
1531 "remote restore requires safetyBackup.repositoryRef".to_string(),
1532 ));
1533 }
1534 }
1535 Ok(())
1536}
1537
1538fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1539 target
1540 .spec
1541 .replica_groups
1542 .iter()
1543 .map(|rg| ReplicaCountEntry {
1544 group: rg.name.clone(),
1545 replicas: rg.replicas,
1546 })
1547 .collect()
1548}
1549
1550async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1551 if let Ok(target) = get_target(restore, ctx).await {
1552 let owns_maintenance =
1553 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1554 if owns_maintenance {
1555 scale_desired(&target, ctx).await?;
1556 clear_restoring(restore, &target, ctx).await?;
1557 }
1558 }
1559 Ok(())
1560}
1561
1562async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1563 ctx.metrics.break_glass_total.add(1, &[]);
1564 let reason = restore
1565 .metadata
1566 .annotations
1567 .as_ref()
1568 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1569 .cloned()
1570 .unwrap_or_default();
1571 let approver = restore
1572 .metadata
1573 .annotations
1574 .as_ref()
1575 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1576 .cloned()
1577 .unwrap_or_default();
1578 let mut status = restore.status.clone().unwrap_or_default();
1579 let break_glass_condition = restore_condition(
1580 &status.conditions,
1581 CONDITION_BREAK_GLASS,
1582 CONDITION_TRUE,
1583 "BreakGlassOverride",
1584 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1585 restore.metadata.generation,
1586 );
1587 status
1588 .conditions
1589 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1590 status.conditions.push(break_glass_condition);
1591 patch_status(restore, ctx, status).await?;
1592 if let Err(error) = ctx
1593 .recorder
1594 .publish(
1595 &Event {
1596 type_: EventType::Warning,
1597 reason: "SafetyBackupSkipped".to_string(),
1598 note: Some(format!(
1599 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1600 )),
1601 action: "Restore".to_string(),
1602 secondary: None,
1603 },
1604 &restore.object_ref(&()),
1605 )
1606 .await
1607 {
1608 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1609 }
1610 warn!(
1611 restore = %restore.name_any(),
1612 reason = %reason,
1613 approver = %approver,
1614 "break-glass safety backup override activated"
1615 );
1616 Ok(())
1617}
1618
1619async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1620 let status = restore.status.as_ref().cloned().unwrap_or_default();
1621 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1622 if let Ok(start_ts) = start.parse::<Timestamp>() {
1623 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1624 ctx.metrics
1625 .safety_backup_duration_seconds
1626 .record(elapsed, &[]);
1627 }
1628 }
1629}
1630
1631fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1632 format!("{}-safety-backup", restore.name_any())
1633}
1634
1635fn source_prep_job_name(restore: &KanidmRestore) -> String {
1636 format!("{}-source-prep", restore.name_any())
1637}
1638
1639fn source_check_job_name(restore: &KanidmRestore) -> String {
1640 format!("{}-source-check", restore.name_any())
1641}
1642
1643async fn ensure_source_check_job(
1644 restore: &KanidmRestore,
1645 target: &Kanidm,
1646 ctx: &RestoreContext,
1647 name: &str,
1648) -> Result<()> {
1649 let ns = restore.namespace().unwrap();
1650 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1651 if api
1652 .get_opt(name)
1653 .await
1654 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1655 .is_some()
1656 {
1657 return Ok(());
1658 }
1659 let file_name = restore
1660 .spec
1661 .source
1662 .local
1663 .as_ref()
1664 .map(|l| l.file_name.clone())
1665 .unwrap_or_default();
1666 let operation_doc = serde_json::json!({
1667 "apiVersion": "backup.kaniop.rs/v1alpha1",
1668 "kind": "OperationDocument",
1669 "operation": "check",
1670 "path": format!("{BACKUP_PATH}/{file_name}"),
1671 "resultPath": "/run/kaniop-result/result.json",
1672 })
1673 .to_string();
1674 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1675 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1676 let data_mover_image = data_mover_image();
1677 let job = Job {
1678 metadata: ObjectMeta {
1679 name: Some(name.to_string()),
1680 namespace: Some(ns.clone()),
1681 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1682 ..Default::default()
1683 },
1684 spec: Some(JobSpec {
1685 backoff_limit: Some(0),
1686 template: PodTemplateSpec {
1687 metadata: Some(ObjectMeta::default()),
1688 spec: Some(PodSpec {
1689 automount_service_account_token: Some(false),
1690 restart_policy: Some("Never".to_string()),
1691 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1692 run_as_non_root: Some(true),
1693 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1694 type_: "RuntimeDefault".to_string(),
1695 ..Default::default()
1696 }),
1697 ..Default::default()
1698 }),
1699 containers: vec![Container {
1700 name: "source-check".to_string(),
1701 image: Some(data_mover_image),
1702 command: Some(vec![
1703 "/bin/kaniop-data-mover".to_string(),
1704 "check".to_string(),
1705 "--operation-doc".to_string(),
1706 "/run/kaniop/operation.json".to_string(),
1707 ]),
1708 security_context: Some(hardened_security_context()),
1709 resources: Some(job_resource_requirements()),
1710 env: Some(vec![EnvVar {
1711 name: "RUST_LOG".to_string(),
1712 value: Some("info".to_string()),
1713 ..Default::default()
1714 }]),
1715 volume_mounts: Some(vec![
1716 VolumeMount {
1717 name: DATA_VOLUME.to_string(),
1718 mount_path: DATA_PATH.to_string(),
1719 read_only: Some(true),
1720 ..Default::default()
1721 },
1722 VolumeMount {
1723 name: "operation".to_string(),
1724 mount_path: "/run/kaniop".to_string(),
1725 read_only: Some(true),
1726 ..Default::default()
1727 },
1728 VolumeMount {
1729 name: "result".to_string(),
1730 mount_path: "/run/kaniop-result".to_string(),
1731 ..Default::default()
1732 },
1733 ]),
1734 ..Default::default()
1735 }],
1736 volumes: Some(vec![
1737 Volume {
1738 name: DATA_VOLUME.to_string(),
1739 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1740 claim_name: primary_pvc_name(target)?,
1741 read_only: Some(true),
1742 }),
1743 ..Default::default()
1744 },
1745 Volume {
1746 name: "operation".to_string(),
1747 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1748 name: operation_cm_name,
1749 ..Default::default()
1750 }),
1751 ..Default::default()
1752 },
1753 Volume {
1754 name: "result".to_string(),
1755 empty_dir: Some(Default::default()),
1756 ..Default::default()
1757 },
1758 ]),
1759 ..Default::default()
1760 }),
1761 },
1762 ..Default::default()
1763 }),
1764 ..Default::default()
1765 };
1766 api.create(&PostParams::default(), &job)
1767 .await
1768 .map(|_| ())
1769 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1770}
1771
1772const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1773
1774fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1775 let ns = restore.namespace().unwrap_or_default();
1776 let name = restore.name_any();
1777 let namespace_uuid =
1778 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1779 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1780 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1781}
1782
1783fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1784 let ns = restore.namespace().unwrap_or_default();
1785 let kanidm_uid = &restore.spec.target_ref.uid;
1786 let backup_id = compute_safety_backup_id(restore);
1787 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1788}
1789
1790struct VerifiedSafetyBackup {
1791 manifest_key: String,
1792 payload_sha256: String,
1793}
1794
1795async fn read_safety_backup_result(
1796 restore: &KanidmRestore,
1797 ctx: &RestoreContext,
1798 job_name: &str,
1799 expected_backup_id: &str,
1800) -> Result<VerifiedSafetyBackup> {
1801 let ns = restore.namespace().unwrap();
1802 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1803 let pod_list = pods
1804 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1805 .await
1806 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1807 let pod = pod_list.items.first().ok_or_else(|| {
1808 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1809 })?;
1810 let pod_name = pod.name_any();
1811 let container_status = pod
1812 .status
1813 .as_ref()
1814 .and_then(|s| s.container_statuses.as_ref())
1815 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1816 .ok_or_else(|| {
1817 Error::MissingData(format!(
1818 "safety-upload container status not found in pod {pod_name}"
1819 ))
1820 })?;
1821 let termination_message = container_status
1822 .state
1823 .as_ref()
1824 .and_then(|state| state.terminated.as_ref())
1825 .and_then(|t| t.message.as_ref())
1826 .ok_or_else(|| {
1827 Error::MissingData(format!(
1828 "termination message not found in pod {pod_name}; result transport failed"
1829 ))
1830 })?;
1831 if termination_message.is_empty() {
1832 return Err(Error::MissingData(format!(
1833 "termination message is empty in pod {pod_name}; result document was not written"
1834 )));
1835 }
1836 let result_doc = parse_result_document(termination_message).map_err(|e| {
1837 Error::ParseError(format!(
1838 "failed to parse result document from pod {pod_name}: {e}"
1839 ))
1840 })?;
1841 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1842 return Err(Error::ParseError(format!(
1843 "unsupported result document version: {}",
1844 result_doc.api_version
1845 )));
1846 }
1847 if result_doc.kind != "ResultDocument" {
1848 return Err(Error::ParseError(format!(
1849 "invalid result document kind: {}",
1850 result_doc.kind
1851 )));
1852 }
1853 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1854 return Err(Error::ParseError(format!(
1855 "result document operation mismatch: expected '{}', got '{}'",
1856 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1857 )));
1858 }
1859 if !result_doc.success {
1860 return Err(Error::ParseError(format!(
1861 "safety backup reported failure with exit code {:?}",
1862 result_doc.exit_code
1863 )));
1864 }
1865 if result_doc.exit_code != ExitCode::Success {
1866 return Err(Error::ParseError(format!(
1867 "safety backup exit code is not Success: {:?}",
1868 result_doc.exit_code
1869 )));
1870 }
1871 let doc_backup_id = result_doc
1872 .backup_id
1873 .as_deref()
1874 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1875 if doc_backup_id != expected_backup_id {
1876 return Err(Error::ParseError(format!(
1877 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1878 )));
1879 }
1880 let manifest_key = result_doc
1881 .manifest_key
1882 .clone()
1883 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1884 if manifest_key.is_empty() {
1885 return Err(Error::ParseError(
1886 "result document manifestKey is empty".to_string(),
1887 ));
1888 }
1889 let payload_sha256 = result_doc
1890 .payload_sha256
1891 .clone()
1892 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1893 if payload_sha256.is_empty() {
1894 return Err(Error::ParseError(
1895 "result document payloadSha256 is empty".to_string(),
1896 ));
1897 }
1898 info!(
1899 restore = %restore.name_any(),
1900 backup_id = %expected_backup_id,
1901 manifest_key = %manifest_key,
1902 "safety backup result verified"
1903 );
1904 Ok(VerifiedSafetyBackup {
1905 manifest_key,
1906 payload_sha256,
1907 })
1908}
1909
1910struct VerifiedSourcePrepResult {
1911 #[allow(dead_code)]
1912 manifest_key: String,
1913 #[allow(dead_code)]
1914 payload_sha256: String,
1915}
1916
1917async fn read_source_prep_result(
1918 restore: &KanidmRestore,
1919 ctx: &RestoreContext,
1920 job_name: &str,
1921 expected_backup_id: &str,
1922) -> Result<VerifiedSourcePrepResult> {
1923 let ns = restore.namespace().unwrap();
1924 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1925 let pod_list = pods
1926 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1927 .await
1928 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1929 let pod = pod_list.items.first().ok_or_else(|| {
1930 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1931 })?;
1932 let pod_name = pod.name_any();
1933 let container_status = pod
1934 .status
1935 .as_ref()
1936 .and_then(|s| s.container_statuses.as_ref())
1937 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1938 .ok_or_else(|| {
1939 Error::MissingData(format!(
1940 "source-prep container status not found in pod {pod_name}"
1941 ))
1942 })?;
1943 let termination_message = container_status
1944 .state
1945 .as_ref()
1946 .and_then(|state| state.terminated.as_ref())
1947 .and_then(|t| t.message.as_ref())
1948 .ok_or_else(|| {
1949 Error::MissingData(format!(
1950 "termination message not found in pod {pod_name}; result transport failed"
1951 ))
1952 })?;
1953 if termination_message.is_empty() {
1954 return Err(Error::MissingData(format!(
1955 "termination message is empty in pod {pod_name}; result document was not written"
1956 )));
1957 }
1958 let result_doc = parse_result_document(termination_message).map_err(|e| {
1959 Error::ParseError(format!(
1960 "failed to parse result document from pod {pod_name}: {e}"
1961 ))
1962 })?;
1963 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1964 return Err(Error::ParseError(format!(
1965 "unsupported result document version: {}",
1966 result_doc.api_version
1967 )));
1968 }
1969 if result_doc.kind != "ResultDocument" {
1970 return Err(Error::ParseError(format!(
1971 "invalid result document kind: {}",
1972 result_doc.kind
1973 )));
1974 }
1975 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1976 return Err(Error::ParseError(format!(
1977 "result document operation mismatch: expected '{}', got '{}'",
1978 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1979 )));
1980 }
1981 if !result_doc.success {
1982 return Err(Error::ParseError(format!(
1983 "source prep reported failure with exit code {:?}",
1984 result_doc.exit_code
1985 )));
1986 }
1987 if result_doc.exit_code != ExitCode::Success {
1988 return Err(Error::ParseError(format!(
1989 "source prep exit code is not Success: {:?}",
1990 result_doc.exit_code
1991 )));
1992 }
1993 let doc_backup_id = result_doc
1994 .backup_id
1995 .as_deref()
1996 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1997 if doc_backup_id != expected_backup_id {
1998 return Err(Error::ParseError(format!(
1999 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
2000 )));
2001 }
2002 let manifest_key = result_doc
2003 .manifest_key
2004 .clone()
2005 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2006 if manifest_key.is_empty() {
2007 return Err(Error::ParseError(
2008 "result document manifestKey is empty".to_string(),
2009 ));
2010 }
2011 let payload_sha256 = result_doc
2012 .payload_sha256
2013 .clone()
2014 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2015 if payload_sha256.is_empty() {
2016 return Err(Error::ParseError(
2017 "result document payloadSha256 is empty".to_string(),
2018 ));
2019 }
2020 info!(
2021 restore = %restore.name_any(),
2022 backup_id = %expected_backup_id,
2023 manifest_key = %manifest_key,
2024 payload_sha256 = %payload_sha256,
2025 "source prep result verified"
2026 );
2027 Ok(VerifiedSourcePrepResult {
2028 manifest_key,
2029 payload_sha256,
2030 })
2031}
2032
2033fn hardened_security_context() -> SecurityContext {
2034 SecurityContext {
2035 allow_privilege_escalation: Some(false),
2036 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2037 drop: Some(vec!["ALL".to_string()]),
2038 ..Default::default()
2039 }),
2040 read_only_root_filesystem: Some(true),
2041 run_as_non_root: Some(true),
2042 seccomp_profile: Some(SeccompProfile {
2043 type_: "RuntimeDefault".to_string(),
2044 ..Default::default()
2045 }),
2046 ..Default::default()
2047 }
2048}
2049
2050fn kanidm_job_security_context() -> SecurityContext {
2051 SecurityContext {
2052 allow_privilege_escalation: Some(false),
2053 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2054 drop: Some(vec!["ALL".to_string()]),
2055 ..Default::default()
2056 }),
2057 seccomp_profile: Some(SeccompProfile {
2058 type_: "RuntimeDefault".to_string(),
2059 ..Default::default()
2060 }),
2061 ..Default::default()
2062 }
2063}
2064
2065fn job_resource_requirements() -> ResourceRequirements {
2066 ResourceRequirements {
2067 requests: Some(std::collections::BTreeMap::from([
2068 ("cpu".to_string(), Quantity("100m".to_string())),
2069 ("memory".to_string(), Quantity("128Mi".to_string())),
2070 ])),
2071 limits: Some(std::collections::BTreeMap::from([
2072 ("cpu".to_string(), Quantity("2".to_string())),
2073 ("memory".to_string(), Quantity("2Gi".to_string())),
2074 ])),
2075 ..Default::default()
2076 }
2077}
2078
2079async fn ensure_safety_backup_job(
2080 restore: &KanidmRestore,
2081 target: &Kanidm,
2082 ctx: &RestoreContext,
2083 name: &str,
2084 backup_id: &str,
2085 manifest_key: &str,
2086) -> Result<()> {
2087 let ns = restore.namespace().unwrap();
2088 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2089 if api
2090 .get_opt(name)
2091 .await
2092 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2093 .is_some()
2094 {
2095 return Ok(());
2096 }
2097 let safety = restore
2098 .spec
2099 .safety_backup
2100 .as_ref()
2101 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2102 let repo_ref = safety
2103 .repository_ref
2104 .as_ref()
2105 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2106 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2107 .get(&repo_ref.name)
2108 .await
2109 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2110 let _namespace_uid = &ns;
2111 let _kanidm_uid = &restore.spec.target_ref.uid;
2112 let endpoint = repo
2113 .spec
2114 .s3
2115 .endpoint
2116 .as_deref()
2117 .unwrap_or("https://s3.amazonaws.com");
2118 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2119 let operation_doc = build_safety_upload_operation_doc(
2120 restore,
2121 target,
2122 backup_id,
2123 manifest_key,
2124 &repo.spec.s3.bucket,
2125 &repo.spec.s3.prefix,
2126 endpoint,
2127 region,
2128 repo.spec.s3.force_path_style,
2129 repo.spec.s3.insecure,
2130 repo.spec
2131 .s3
2132 .ca_bundle_ref
2133 .as_deref()
2134 .map(|_| ca_bundle_path())
2135 .as_deref(),
2136 )?;
2137 let operation_cm_name = format!("{name}-op");
2138 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2139 let auth_method = &repo.spec.authentication.writer;
2140 let data_mover_image = data_mover_image();
2141 let init_command = vec![
2142 "kanidmd".to_string(),
2143 "database".to_string(),
2144 "backup".to_string(),
2145 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2146 ];
2147 let upload_script = format!(
2148 r#"set -eu
2149/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2150RESULT_FILE="/run/kaniop-result/result.json"
2151TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2152if [ ! -f "$RESULT_FILE" ]; then
2153 echo "ERROR: result document not found at $RESULT_FILE"
2154 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2155 exit 1
2156fi
2157if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2158 echo "ERROR: safety backup upload reported failure"
2159 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2160 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2161 exit 1
2162fi
2163dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2164echo "safety backup upload completed successfully"
2165"#
2166 );
2167 let job = Job {
2168 metadata: ObjectMeta {
2169 name: Some(name.to_string()),
2170 namespace: Some(ns.clone()),
2171 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2172 ..Default::default()
2173 },
2174 spec: Some(JobSpec {
2175 backoff_limit: Some(0),
2176 template: PodTemplateSpec {
2177 metadata: Some(ObjectMeta::default()),
2178 spec: Some(PodSpec {
2179 automount_service_account_token: Some(false),
2180 restart_policy: Some("Never".to_string()),
2181 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2182 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2183 type_: "RuntimeDefault".to_string(),
2184 ..Default::default()
2185 }),
2186 ..Default::default()
2187 }),
2188 init_containers: Some(vec![Container {
2189 name: "safety-backup".to_string(),
2190 image: Some(restore.spec.restore_image.clone()),
2191 command: Some(init_command),
2192 security_context: Some(SecurityContext {
2193 allow_privilege_escalation: Some(false),
2194 capabilities: Some(Capabilities {
2195 drop: Some(vec!["ALL".to_string()]),
2196 ..Default::default()
2197 }),
2198 read_only_root_filesystem: Some(true),
2199 seccomp_profile: Some(SeccompProfile {
2200 type_: "RuntimeDefault".to_string(),
2201 ..Default::default()
2202 }),
2203 ..Default::default()
2204 }),
2205 resources: Some(job_resource_requirements()),
2206 env: Some(build_kanidm_db_env_vars(target)),
2207 volume_mounts: Some(vec![
2208 VolumeMount {
2209 name: DATA_VOLUME.to_string(),
2210 mount_path: DATA_PATH.to_string(),
2211 read_only: Some(false),
2212 ..Default::default()
2213 },
2214 VolumeMount {
2215 name: TLS_VOLUME.to_string(),
2216 mount_path: TLS_PATH.to_string(),
2217 read_only: Some(true),
2218 ..Default::default()
2219 },
2220 VolumeMount {
2221 name: SHARED_VOLUME.to_string(),
2222 mount_path: SHARED_VOL_PATH.to_string(),
2223 ..Default::default()
2224 },
2225 ]),
2226 ..Default::default()
2227 }]),
2228 containers: vec![Container {
2229 name: "safety-upload".to_string(),
2230 image: Some(data_mover_image),
2231 command: Some(vec!["/bin/sh".to_string()]),
2232 args: Some(vec!["-c".to_string(), upload_script]),
2233 security_context: Some(hardened_security_context()),
2234 resources: Some(job_resource_requirements()),
2235 env: {
2236 let mut env_vars =
2237 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2238 env_vars.push(EnvVar {
2239 name: "RUST_LOG".to_string(),
2240 value: Some("info".to_string()),
2241 ..Default::default()
2242 });
2243 if repo.spec.s3.ca_bundle_ref.is_some() {
2244 env_vars.push(ca_bundle_env_var());
2245 }
2246 Some(env_vars)
2247 },
2248 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2249 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2250 volume_mounts: {
2251 let mut mounts = vec![
2252 VolumeMount {
2253 name: SHARED_VOLUME.to_string(),
2254 mount_path: SHARED_VOL_PATH.to_string(),
2255 read_only: Some(true),
2256 ..Default::default()
2257 },
2258 VolumeMount {
2259 name: "operation".to_string(),
2260 mount_path: "/run/kaniop".to_string(),
2261 read_only: Some(true),
2262 ..Default::default()
2263 },
2264 VolumeMount {
2265 name: "result".to_string(),
2266 mount_path: "/run/kaniop-result".to_string(),
2267 ..Default::default()
2268 },
2269 ];
2270 mounts.extend(build_auth_volume_mounts(auth_method));
2271 if repo.spec.s3.ca_bundle_ref.is_some() {
2272 mounts.push(build_ca_bundle_volume_mount());
2273 }
2274 Some(mounts)
2275 },
2276 ..Default::default()
2277 }],
2278 volumes: {
2279 let mut vols = vec![
2280 Volume {
2281 name: DATA_VOLUME.to_string(),
2282 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2283 claim_name: primary_pvc_name(target)?,
2284 read_only: Some(false),
2285 }),
2286 ..Default::default()
2287 },
2288 Volume {
2289 name: TLS_VOLUME.to_string(),
2290 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2291 secret_name: Some(target.effective_tls_secret_name()),
2292 ..Default::default()
2293 }),
2294 ..Default::default()
2295 },
2296 Volume {
2297 name: SHARED_VOLUME.to_string(),
2298 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2299 size_limit: Some(Quantity("10Gi".to_string())),
2300 ..Default::default()
2301 }),
2302 ..Default::default()
2303 },
2304 Volume {
2305 name: "operation".to_string(),
2306 config_map: Some(ConfigMapVolumeSource {
2307 name: operation_cm_name,
2308 ..Default::default()
2309 }),
2310 ..Default::default()
2311 },
2312 Volume {
2313 name: "result".to_string(),
2314 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2315 size_limit: Some(Quantity("16Mi".to_string())),
2316 ..Default::default()
2317 }),
2318 ..Default::default()
2319 },
2320 ];
2321 vols.extend(build_auth_volumes(auth_method));
2322 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2323 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2324 }
2325 Some(vols)
2326 },
2327 ..Default::default()
2328 }),
2329 },
2330 ..Default::default()
2331 }),
2332 ..Default::default()
2333 };
2334 api.create(&PostParams::default(), &job)
2335 .await
2336 .map(|_| ())
2337 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2338}
2339
2340async fn ensure_source_prep_job(
2341 restore: &KanidmRestore,
2342 target: &Kanidm,
2343 ctx: &RestoreContext,
2344 name: &str,
2345) -> Result<()> {
2346 let ns = restore.namespace().unwrap();
2347 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2348 if api
2349 .get_opt(name)
2350 .await
2351 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2352 .is_some()
2353 {
2354 return Ok(());
2355 }
2356 let backup_name = restore
2357 .spec
2358 .source
2359 .backup_ref
2360 .as_ref()
2361 .map(|b| b.name.clone())
2362 .unwrap_or_default();
2363 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2364 .get(&backup_name)
2365 .await
2366 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2367 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2368 .get(&backup.spec.repository_ref.name)
2369 .await
2370 .map_err(|e| {
2371 Error::kube_error(
2372 "get",
2373 "KanidmBackupRepository",
2374 &ns,
2375 &backup.spec.repository_ref.name,
2376 e,
2377 )
2378 })?;
2379 let endpoint = repo
2380 .spec
2381 .s3
2382 .endpoint
2383 .as_deref()
2384 .unwrap_or("https://s3.amazonaws.com");
2385 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2386 let operation_doc = build_download_operation_doc(
2387 restore,
2388 target,
2389 &backup.spec.manifest_key,
2390 &backup.spec.backup_id,
2391 &repo.spec.s3.bucket,
2392 &repo.spec.s3.prefix,
2393 endpoint,
2394 region,
2395 repo.spec.s3.force_path_style,
2396 repo.spec.s3.insecure,
2397 repo.spec
2398 .s3
2399 .ca_bundle_ref
2400 .as_deref()
2401 .map(|_| ca_bundle_path())
2402 .as_deref(),
2403 );
2404 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2405 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2406 let auth_method = &repo.spec.authentication.reader;
2407 let data_mover_image = data_mover_image();
2408 let download_script = format!(
2409 r#"set -eu
2410/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2411RESULT_FILE="/run/kaniop-result/result.json"
2412TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2413if [ ! -f "$RESULT_FILE" ]; then
2414 echo "ERROR: result document not found at $RESULT_FILE"
2415 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2416 exit 1
2417fi
2418if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2419 echo "ERROR: source preparation download reported failure"
2420 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2421 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2422 exit 1
2423fi
2424dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2425echo "source preparation download completed successfully"
2426"#
2427 );
2428 let job = Job {
2429 metadata: ObjectMeta {
2430 name: Some(name.to_string()),
2431 namespace: Some(ns.clone()),
2432 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2433 ..Default::default()
2434 },
2435 spec: Some(JobSpec {
2436 backoff_limit: Some(0),
2437 template: PodTemplateSpec {
2438 metadata: Some(ObjectMeta::default()),
2439 spec: Some(PodSpec {
2440 automount_service_account_token: Some(false),
2441 restart_policy: Some("Never".to_string()),
2442 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2443 run_as_non_root: Some(true),
2444 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2445 type_: "RuntimeDefault".to_string(),
2446 ..Default::default()
2447 }),
2448 ..Default::default()
2449 }),
2450 containers: vec![Container {
2451 name: "source-prep".to_string(),
2452 image: Some(data_mover_image),
2453 command: Some(vec!["/bin/sh".to_string()]),
2454 args: Some(vec!["-c".to_string(), download_script]),
2455 security_context: Some(hardened_security_context()),
2456 resources: Some(job_resource_requirements()),
2457 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2458 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2459 env: {
2460 let mut env_vars = build_auth_env_vars(
2461 auth_method,
2462 &backup.spec.repository_ref.name,
2463 AuthRole::Reader,
2464 );
2465 env_vars.push(EnvVar {
2466 name: "RUST_LOG".to_string(),
2467 value: Some("info".to_string()),
2468 ..Default::default()
2469 });
2470 if repo.spec.s3.ca_bundle_ref.is_some() {
2471 env_vars.push(ca_bundle_env_var());
2472 }
2473 Some(env_vars)
2474 },
2475 volume_mounts: {
2476 let mut mounts = vec![
2477 VolumeMount {
2478 name: STAGING_VOLUME.to_string(),
2479 mount_path: STAGING_PATH.to_string(),
2480 ..Default::default()
2481 },
2482 VolumeMount {
2483 name: "operation".to_string(),
2484 mount_path: "/run/kaniop".to_string(),
2485 read_only: Some(true),
2486 ..Default::default()
2487 },
2488 VolumeMount {
2489 name: "result".to_string(),
2490 mount_path: "/run/kaniop-result".to_string(),
2491 ..Default::default()
2492 },
2493 ];
2494 mounts.extend(build_auth_volume_mounts(auth_method));
2495 if repo.spec.s3.ca_bundle_ref.is_some() {
2496 mounts.push(build_ca_bundle_volume_mount());
2497 }
2498 Some(mounts)
2499 },
2500 ..Default::default()
2501 }],
2502 volumes: {
2503 let mut vols = vec![
2504 Volume {
2505 name: STAGING_VOLUME.to_string(),
2506 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2507 size_limit: Some(Quantity("10Gi".to_string())),
2508 ..Default::default()
2509 }),
2510 ..Default::default()
2511 },
2512 Volume {
2513 name: "operation".to_string(),
2514 config_map: Some(ConfigMapVolumeSource {
2515 name: operation_cm_name,
2516 ..Default::default()
2517 }),
2518 ..Default::default()
2519 },
2520 Volume {
2521 name: "result".to_string(),
2522 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2523 size_limit: Some(Quantity("16Mi".to_string())),
2524 ..Default::default()
2525 }),
2526 ..Default::default()
2527 },
2528 ];
2529 vols.extend(build_auth_volumes(auth_method));
2530 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2531 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2532 }
2533 Some(vols)
2534 },
2535 ..Default::default()
2536 }),
2537 },
2538 ..Default::default()
2539 }),
2540 ..Default::default()
2541 };
2542 api.create(&PostParams::default(), &job)
2543 .await
2544 .map(|_| ())
2545 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2546}
2547
2548#[allow(clippy::too_many_arguments)]
2549fn build_safety_upload_operation_doc(
2550 restore: &KanidmRestore,
2551 target: &Kanidm,
2552 backup_id: &str,
2553 manifest_key: &str,
2554 bucket: &str,
2555 prefix: &str,
2556 endpoint: &str,
2557 region: &str,
2558 force_path_style: bool,
2559 insecure: bool,
2560 ca_bundle_path: Option<&str>,
2561) -> Result<String> {
2562 let op = UploadOperation {
2563 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2564 bucket: bucket.to_string(),
2565 prefix: prefix.to_string(),
2566 endpoint: endpoint.to_string(),
2567 region: region.to_string(),
2568 force_path_style,
2569 insecure,
2570 ca_bundle_path: ca_bundle_path.map(str::to_string),
2571 backup_id: backup_id.to_string(),
2572 namespace_uid: restore.namespace().unwrap_or_default(),
2573 kanidm_uid: restore.spec.target_ref.uid.clone(),
2574 kanidm_name: restore.spec.target_ref.name.clone(),
2575 domain: target.spec.domain.clone(),
2576 kanidm_version: target
2577 .status
2578 .as_ref()
2579 .and_then(|s| s.version.as_ref())
2580 .map(|v| v.image_tag.clone())
2581 .unwrap_or_default(),
2582 image_digest: None,
2583 consistency: "kanidm-offline".to_string(),
2584 reason: "restore-safety".to_string(),
2585 encryption_mode: None,
2586 encryption_key_id: None,
2587 result_path: "/run/kaniop-result/result.json".to_string(),
2588 max_concurrent_parts: 4,
2589 max_retries: 3,
2590 };
2591 let _ = manifest_key;
2592 let doc = OperationDocument {
2593 api_version: OPERATION_DOC_VERSION.to_string(),
2594 kind: "OperationDocument".to_string(),
2595 spec: OperationSpec::Upload(op),
2596 };
2597 serde_json::to_string(&doc)
2598 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2599}
2600
2601async fn validate_backup_ref(
2602 restore: &KanidmRestore,
2603 target: &Kanidm,
2604 ctx: &RestoreContext,
2605) -> Result<()> {
2606 let backup_ref = restore
2607 .spec
2608 .source
2609 .backup_ref
2610 .as_ref()
2611 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2612 let ns = restore.namespace().unwrap();
2613 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2614 .get(&backup_ref.name)
2615 .await
2616 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2617 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2618 return Err(Error::MissingData(format!(
2619 "KanidmBackup '{}' is not Ready (phase: {:?})",
2620 backup_ref.name,
2621 backup.status.as_ref().map(|s| &s.phase)
2622 )));
2623 }
2624 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2625 return Err(Error::MissingData(format!(
2626 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2627 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2628 )));
2629 }
2630 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2631 return Err(Error::MissingData(format!(
2632 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2633 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2634 )));
2635 }
2636 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2637 .get(&backup.spec.repository_ref.name)
2638 .await
2639 .map_err(|e| {
2640 Error::kube_error(
2641 "get",
2642 "KanidmBackupRepository",
2643 &ns,
2644 &backup.spec.repository_ref.name,
2645 e,
2646 )
2647 })?;
2648 if !has_accepted_condition(
2649 &repo
2650 .status
2651 .as_ref()
2652 .map(|s| &s.conditions)
2653 .cloned()
2654 .unwrap_or_default(),
2655 ) {
2656 return Err(Error::MissingData(format!(
2657 "KanidmBackupRepository '{}' configuration has not been accepted",
2658 backup.spec.repository_ref.name
2659 )));
2660 }
2661 validate_backup_compatibility(restore, target, &backup)?;
2662 Ok(())
2663}
2664
2665fn has_accepted_condition(conditions: &[Condition]) -> bool {
2666 conditions
2667 .iter()
2668 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2669}
2670
2671fn validate_backup_compatibility(
2672 restore: &KanidmRestore,
2673 target: &Kanidm,
2674 backup: &KanidmBackup,
2675) -> Result<()> {
2676 let status = backup.status.as_ref();
2677 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2678 let target_version = target
2679 .status
2680 .as_ref()
2681 .and_then(|s| s.version.as_ref())
2682 .map(|v| v.image_tag.as_str());
2683 if let Some(tv) = target_version {
2684 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2685 return Err(Error::MissingData(format!(
2686 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2687 )));
2688 }
2689 }
2690 }
2691 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2692 if !backup_digest.is_empty()
2693 && restore.spec.restore_image.contains('@')
2694 && !restore.spec.restore_image.contains(backup_digest)
2695 {
2696 return Err(Error::MissingData(format!(
2697 "restore image digest does not match backup image digest '{backup_digest}'"
2698 )));
2699 }
2700 }
2701 Ok(())
2702}
2703
2704#[allow(clippy::too_many_arguments)]
2705fn build_download_operation_doc(
2706 restore: &KanidmRestore,
2707 target: &Kanidm,
2708 manifest_key: &str,
2709 expected_backup_id: &str,
2710 bucket: &str,
2711 prefix: &str,
2712 endpoint: &str,
2713 region: &str,
2714 force_path_style: bool,
2715 insecure: bool,
2716 ca_bundle_path: Option<&str>,
2717) -> String {
2718 serde_json::json!({
2719 "apiVersion": "backup.kaniop.rs/v1alpha1",
2720 "kind": "OperationDocument",
2721 "operation": "download",
2722 "manifestKey": manifest_key,
2723 "bucket": bucket,
2724 "prefix": prefix,
2725 "endpoint": endpoint,
2726 "region": region,
2727 "forcePathStyle": force_path_style,
2728 "insecure": insecure,
2729 "caBundlePath": ca_bundle_path,
2730 "expectedBackupId": expected_backup_id,
2731 "expectedKanidmUid": restore.spec.target_ref.uid,
2732 "expectedDomain": target.spec.domain,
2733 "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2734 "resultPath": "/run/kaniop-result/result.json",
2735 "maxRetries": 3,
2736 })
2737 .to_string()
2738}
2739
2740async fn ensure_operation_configmap(
2741 restore: &KanidmRestore,
2742 name: &str,
2743 data: &str,
2744 ns: &str,
2745 ctx: &RestoreContext,
2746) -> Result<()> {
2747 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2748 if api
2749 .get_opt(name)
2750 .await
2751 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2752 .is_some()
2753 {
2754 return Ok(());
2755 }
2756 let cm = ConfigMap {
2757 metadata: ObjectMeta {
2758 name: Some(name.to_string()),
2759 namespace: Some(ns.to_string()),
2760 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2761 ..Default::default()
2762 },
2763 data: Some(std::collections::BTreeMap::from([(
2764 "operation.json".to_string(),
2765 data.to_string(),
2766 )])),
2767 ..Default::default()
2768 };
2769 api.create(&PostParams::default(), &cm)
2770 .await
2771 .map(|_| ())
2772 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2773}
2774
2775#[cfg(test)]
2776mod tests {
2777 use super::{
2778 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2779 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2780 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2781 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2782 hardened_security_context, has_accepted_condition, is_remote_source,
2783 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2784 validate_safety_backup_config, validate_source,
2785 };
2786 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2787 use kube::api::ObjectMeta as ApiObjectMeta;
2788 use std::collections::BTreeMap;
2789
2790 fn make_restore(
2791 source: KanidmRestoreSource,
2792 safety: Option<SafetyBackupConfig>,
2793 ) -> KanidmRestore {
2794 KanidmRestore {
2795 metadata: ApiObjectMeta {
2796 name: Some("test-restore".to_string()),
2797 namespace: Some("default".to_string()),
2798 ..Default::default()
2799 },
2800 spec: KanidmRestoreSpec {
2801 target_ref: KanidmRestoreTargetRef {
2802 name: "corp-idm".to_string(),
2803 uid: "test-uid-123".to_string(),
2804 },
2805 source,
2806 restore_image: "kanidm/server@sha256:abc".to_string(),
2807 safety_backup: safety,
2808 },
2809 status: None,
2810 }
2811 }
2812
2813 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2814 super::super::crd::Kanidm {
2815 status: Some(super::super::crd::KanidmStatus {
2816 available_replicas: 1,
2817 replicas: 1,
2818 unavailable_replicas: 0,
2819 updated_replicas: 1,
2820 replica_statuses: vec![],
2821 replica_column: "1/1".to_string(),
2822 secret_name: None,
2823 version: Some(super::super::crd::KanidmVersionStatus {
2824 image_tag: image_tag.to_string(),
2825 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2826 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2827 }),
2828 domain_appearance_image: None,
2829 mail_sender: None,
2830 conditions: None,
2831 }),
2832 ..Default::default()
2833 }
2834 }
2835
2836 #[test]
2837 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2838 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2839 }
2840
2841 #[test]
2842 fn completed_restore_sets_ready_condition() {
2843 let mut status = KanidmRestoreStatus {
2844 phase: KanidmRestorePhase::Completed,
2845 ..Default::default()
2846 };
2847 super::update_restore_conditions(&mut status, Some(7));
2848 assert!(status.conditions.iter().any(|condition| {
2849 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2850 }));
2851 assert!(!status.conditions.iter().any(|condition| {
2852 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2853 }));
2854 }
2855
2856 #[test]
2857 fn failed_restore_sets_failed_condition() {
2858 let mut status = KanidmRestoreStatus {
2859 phase: KanidmRestorePhase::Failed,
2860 message: Some("verification failed".to_string()),
2861 ..Default::default()
2862 };
2863 super::update_restore_conditions(&mut status, Some(8));
2864 assert!(status.conditions.iter().any(|condition| {
2865 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2866 }));
2867 }
2868
2869 #[test]
2870 fn restore_filename_is_confined_to_backup_directory() {
2871 assert!(safe_basename("backup.json.gz"));
2872 assert!(!safe_basename("../backup.json.gz"));
2873 assert!(!safe_basename("a/b"));
2874 assert!(!safe_basename(""));
2875 }
2876
2877 #[test]
2878 fn restore_image_rejects_latest_and_unpinned_names() {
2879 assert!(mutable_image("kanidm/server:latest"));
2880 assert!(mutable_image("kanidm/server"));
2881 assert!(!mutable_image("kanidm/server:1.10.0"));
2882 assert!(!mutable_image("kanidm/server@sha256:abc"));
2883 }
2884
2885 #[test]
2886 fn source_validation_rejects_both_local_and_backup_ref() {
2887 let source = KanidmRestoreSource {
2888 local: Some(super::KanidmRestoreLocalSource {
2889 file_name: "backup.json".to_string(),
2890 }),
2891 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2892 name: "some-backup".to_string(),
2893 }),
2894 };
2895 let restore = make_restore(source, None);
2896 assert!(validate_source(&restore).is_err());
2897 }
2898
2899 #[test]
2900 fn source_validation_rejects_empty_source() {
2901 let source = KanidmRestoreSource {
2902 local: None,
2903 backup_ref: None,
2904 };
2905 let restore = make_restore(source, None);
2906 assert!(validate_source(&restore).is_err());
2907 }
2908
2909 #[test]
2910 fn source_validation_accepts_local_source() {
2911 let source = KanidmRestoreSource {
2912 local: Some(super::KanidmRestoreLocalSource {
2913 file_name: "backup.json".to_string(),
2914 }),
2915 backup_ref: None,
2916 };
2917 let restore = make_restore(source, None);
2918 assert!(validate_source(&restore).is_ok());
2919 }
2920
2921 #[test]
2922 fn source_validation_accepts_backup_ref_source() {
2923 let source = KanidmRestoreSource {
2924 local: None,
2925 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2926 name: "corp-idm-019c7c76".to_string(),
2927 }),
2928 };
2929 let restore = make_restore(source, None);
2930 assert!(validate_source(&restore).is_ok());
2931 }
2932
2933 #[test]
2934 fn source_validation_rejects_empty_backup_ref_name() {
2935 let source = KanidmRestoreSource {
2936 local: None,
2937 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2938 name: "".to_string(),
2939 }),
2940 };
2941 let restore = make_restore(source, None);
2942 assert!(validate_source(&restore).is_err());
2943 }
2944
2945 #[test]
2946 fn is_remote_source_true_for_backup_ref() {
2947 let source = KanidmRestoreSource {
2948 local: None,
2949 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2950 name: "backup-1".to_string(),
2951 }),
2952 };
2953 let restore = make_restore(source, None);
2954 assert!(is_remote_source(&restore));
2955 }
2956
2957 #[test]
2958 fn is_remote_source_false_for_local() {
2959 let source = KanidmRestoreSource {
2960 local: Some(super::KanidmRestoreLocalSource {
2961 file_name: "backup.json".to_string(),
2962 }),
2963 backup_ref: None,
2964 };
2965 let restore = make_restore(source, None);
2966 assert!(!is_remote_source(&restore));
2967 }
2968
2969 #[test]
2970 fn requires_safety_backup_default_true() {
2971 let source = KanidmRestoreSource {
2972 local: None,
2973 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2974 name: "backup-1".to_string(),
2975 }),
2976 };
2977 let restore = make_restore(source, None);
2978 assert!(requires_safety_backup(&restore));
2979 }
2980
2981 #[test]
2982 fn requires_safety_backup_false_when_skip() {
2983 let source = KanidmRestoreSource {
2984 local: None,
2985 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2986 name: "backup-1".to_string(),
2987 }),
2988 };
2989 let safety = SafetyBackupConfig {
2990 repository_ref: Some(SafetyBackupRepositoryRef {
2991 name: "offsite".to_string(),
2992 }),
2993 skip: true,
2994 };
2995 let restore = make_restore(source, Some(safety));
2996 assert!(!requires_safety_backup(&restore));
2997 }
2998
2999 #[test]
3000 fn break_glass_validation_rejects_skip_without_reason() {
3001 let source = KanidmRestoreSource {
3002 local: None,
3003 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3004 name: "backup-1".to_string(),
3005 }),
3006 };
3007 let safety = SafetyBackupConfig {
3008 repository_ref: Some(SafetyBackupRepositoryRef {
3009 name: "offsite".to_string(),
3010 }),
3011 skip: true,
3012 };
3013 let mut restore = make_restore(source, Some(safety));
3014 restore.metadata.annotations = Some(BTreeMap::from([(
3015 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3016 "commander@example.com".to_string(),
3017 )]));
3018 assert!(validate_safety_backup_config(&restore).is_err());
3019 }
3020
3021 #[test]
3022 fn break_glass_validation_rejects_skip_without_approver() {
3023 let source = KanidmRestoreSource {
3024 local: None,
3025 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3026 name: "backup-1".to_string(),
3027 }),
3028 };
3029 let safety = SafetyBackupConfig {
3030 repository_ref: Some(SafetyBackupRepositoryRef {
3031 name: "offsite".to_string(),
3032 }),
3033 skip: true,
3034 };
3035 let mut restore = make_restore(source, Some(safety));
3036 restore.metadata.annotations = Some(BTreeMap::from([(
3037 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3038 "PVC is unreadable".to_string(),
3039 )]));
3040 assert!(validate_safety_backup_config(&restore).is_err());
3041 }
3042
3043 #[test]
3044 fn break_glass_validation_accepts_valid_annotations() {
3045 let source = KanidmRestoreSource {
3046 local: None,
3047 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3048 name: "backup-1".to_string(),
3049 }),
3050 };
3051 let safety = SafetyBackupConfig {
3052 repository_ref: Some(SafetyBackupRepositoryRef {
3053 name: "offsite".to_string(),
3054 }),
3055 skip: true,
3056 };
3057 let mut restore = make_restore(source, Some(safety));
3058 restore.metadata.annotations = Some(BTreeMap::from([
3059 (
3060 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3061 "PVC is unreadable".to_string(),
3062 ),
3063 (
3064 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3065 "commander@example.com".to_string(),
3066 ),
3067 ]));
3068 assert!(validate_safety_backup_config(&restore).is_ok());
3069 }
3070
3071 #[test]
3072 fn remote_source_requires_repository_ref_when_not_skipping() {
3073 let source = KanidmRestoreSource {
3074 local: None,
3075 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3076 name: "backup-1".to_string(),
3077 }),
3078 };
3079 let safety = SafetyBackupConfig {
3080 repository_ref: None,
3081 skip: false,
3082 };
3083 let restore = make_restore(source, Some(safety));
3084 assert!(validate_safety_backup_config(&restore).is_err());
3085 }
3086
3087 #[test]
3088 fn remote_source_accepts_repository_ref() {
3089 let source = KanidmRestoreSource {
3090 local: None,
3091 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3092 name: "backup-1".to_string(),
3093 }),
3094 };
3095 let safety = SafetyBackupConfig {
3096 repository_ref: Some(SafetyBackupRepositoryRef {
3097 name: "offsite".to_string(),
3098 }),
3099 skip: false,
3100 };
3101 let restore = make_restore(source, Some(safety));
3102 assert!(validate_safety_backup_config(&restore).is_ok());
3103 }
3104
3105 #[test]
3106 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3107 let phases = vec![
3108 KanidmRestorePhase::Pending,
3109 KanidmRestorePhase::Validating,
3110 KanidmRestorePhase::Quiescing,
3111 KanidmRestorePhase::SafetyBackup,
3112 KanidmRestorePhase::PreparingSource,
3113 KanidmRestorePhase::RestoringPrimary,
3114 KanidmRestorePhase::Verifying,
3115 KanidmRestorePhase::RebuildingReplicas,
3116 KanidmRestorePhase::Resuming,
3117 KanidmRestorePhase::Completed,
3118 ];
3119 assert_eq!(phases.len(), 10);
3120 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3121 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3122 }
3123
3124 #[test]
3125 fn status_default_has_empty_phase_timestamps() {
3126 let status = KanidmRestoreStatus::default();
3127 assert!(status.phase_timestamps.is_empty());
3128 }
3129
3130 #[test]
3131 fn status_default_has_empty_original_replicas() {
3132 let status = KanidmRestoreStatus::default();
3133 assert!(status.original_replicas.is_empty());
3134 }
3135
3136 #[test]
3137 fn hardened_security_context_drops_all_capabilities() {
3138 let ctx = hardened_security_context();
3139 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3140 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3141 assert_eq!(ctx.run_as_non_root, Some(true));
3142 let caps = ctx.capabilities.unwrap();
3143 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3144 assert_eq!(
3145 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3146 Some("RuntimeDefault")
3147 );
3148 }
3149
3150 #[test]
3151 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3152 let ctx = kanidm_job_security_context();
3153 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3154 assert!(ctx.run_as_non_root.is_none());
3155 assert!(ctx.read_only_root_filesystem.is_none());
3156 let caps = ctx.capabilities.unwrap();
3157 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3158 assert_eq!(
3159 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3160 Some("RuntimeDefault")
3161 );
3162 }
3163
3164 #[test]
3165 fn safety_backup_job_name_is_deterministic() {
3166 let source = KanidmRestoreSource {
3167 local: Some(super::KanidmRestoreLocalSource {
3168 file_name: "backup.json".to_string(),
3169 }),
3170 backup_ref: None,
3171 };
3172 let restore = make_restore(source, None);
3173 let name1 = super::safety_backup_job_name(&restore);
3174 let name2 = super::safety_backup_job_name(&restore);
3175 assert_eq!(name1, name2);
3176 assert_eq!(name1, "test-restore-safety-backup");
3177 }
3178
3179 #[test]
3180 fn source_prep_job_name_is_deterministic() {
3181 let source = KanidmRestoreSource {
3182 local: None,
3183 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3184 name: "backup-1".to_string(),
3185 }),
3186 };
3187 let restore = make_restore(source, None);
3188 let name = super::source_prep_job_name(&restore);
3189 assert_eq!(name, "test-restore-source-prep");
3190 }
3191
3192 #[test]
3193 fn break_glass_condition_is_recorded_in_status() {
3194 let mut status = KanidmRestoreStatus::default();
3195 let condition = super::restore_condition(
3196 &status.conditions,
3197 CONDITION_BREAK_GLASS,
3198 CONDITION_TRUE,
3199 "BreakGlassOverride",
3200 "Safety backup skipped",
3201 Some(1),
3202 );
3203 status.conditions.push(condition);
3204 assert!(status.conditions.iter().any(|c| {
3205 c.type_ == CONDITION_BREAK_GLASS
3206 && c.status == CONDITION_TRUE
3207 && c.reason == "BreakGlassOverride"
3208 }));
3209 }
3210
3211 #[test]
3212 fn replica_count_entry_serialization() {
3213 let entry = ReplicaCountEntry {
3214 group: "primary".to_string(),
3215 replicas: 3,
3216 };
3217 let json = serde_json::to_string(&entry).unwrap();
3218 assert!(json.contains("\"group\":\"primary\""));
3219 assert!(json.contains("\"replicas\":3"));
3220 }
3221
3222 #[test]
3223 fn local_source_does_not_require_repository_ref() {
3224 let source = KanidmRestoreSource {
3225 local: Some(super::KanidmRestoreLocalSource {
3226 file_name: "backup.json".to_string(),
3227 }),
3228 backup_ref: None,
3229 };
3230 let safety = SafetyBackupConfig {
3231 repository_ref: None,
3232 skip: false,
3233 };
3234 let restore = make_restore(source, Some(safety));
3235 assert!(validate_safety_backup_config(&restore).is_ok());
3236 }
3237
3238 #[test]
3239 fn skip_without_break_glass_annotations_is_rejected() {
3240 let source = KanidmRestoreSource {
3241 local: Some(super::KanidmRestoreLocalSource {
3242 file_name: "backup.json".to_string(),
3243 }),
3244 backup_ref: None,
3245 };
3246 let safety = SafetyBackupConfig {
3247 repository_ref: None,
3248 skip: true,
3249 };
3250 let restore = make_restore(source, Some(safety));
3251 assert!(validate_safety_backup_config(&restore).is_err());
3252 }
3253
3254 #[test]
3255 fn has_accepted_condition_true() {
3256 let conditions = vec![Condition {
3257 type_: "Ready".to_string(),
3258 status: "True".to_string(),
3259 reason: "Accepted".to_string(),
3260 message: String::new(),
3261 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3262 k8s_openapi::jiff::Timestamp::now(),
3263 ),
3264 observed_generation: None,
3265 }];
3266 assert!(has_accepted_condition(&conditions));
3267 }
3268
3269 #[test]
3270 fn has_accepted_condition_false_when_not_accepted_reason() {
3271 let conditions = vec![Condition {
3272 type_: "Ready".to_string(),
3273 status: "True".to_string(),
3274 reason: "SomeOtherReason".to_string(),
3275 message: String::new(),
3276 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3277 k8s_openapi::jiff::Timestamp::now(),
3278 ),
3279 observed_generation: None,
3280 }];
3281 assert!(!has_accepted_condition(&conditions));
3282 }
3283
3284 #[test]
3285 fn has_accepted_condition_false_when_status_false() {
3286 let conditions = vec![Condition {
3287 type_: "Ready".to_string(),
3288 status: "False".to_string(),
3289 reason: "Accepted".to_string(),
3290 message: String::new(),
3291 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3292 k8s_openapi::jiff::Timestamp::now(),
3293 ),
3294 observed_generation: None,
3295 }];
3296 assert!(!has_accepted_condition(&conditions));
3297 }
3298
3299 #[test]
3300 fn has_accepted_condition_empty() {
3301 assert!(!has_accepted_condition(&[]));
3302 }
3303
3304 #[test]
3305 fn safety_backup_upload_operation_doc_is_valid() {
3306 let source = KanidmRestoreSource {
3307 local: None,
3308 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3309 name: "backup-1".to_string(),
3310 }),
3311 };
3312 let mut restore = make_restore(source, None);
3313 restore.metadata.namespace = Some("test-ns".to_string());
3314 let target = super::super::crd::Kanidm::default();
3315 let doc_str = super::build_safety_upload_operation_doc(
3316 &restore,
3317 &target,
3318 "test-backup-id",
3319 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3320 "my-bucket",
3321 "prod",
3322 "https://s3.example.com",
3323 "us-east-1",
3324 false,
3325 false,
3326 None,
3327 )
3328 .unwrap();
3329 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3330 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3331 assert_eq!(parsed["kind"], "OperationDocument");
3332 assert_eq!(parsed["operation"], "upload");
3333 assert_eq!(parsed["bucket"], "my-bucket");
3334 assert_eq!(parsed["prefix"], "prod");
3335 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3336 assert_eq!(parsed["region"], "us-east-1");
3337 assert_eq!(parsed["backupId"], "test-backup-id");
3338 assert_eq!(parsed["consistency"], "kanidm-offline");
3339 assert_eq!(parsed["reason"], "restore-safety");
3340 assert_eq!(
3341 parsed["payloadPath"],
3342 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3343 );
3344 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3345 }
3346
3347 #[test]
3348 fn download_operation_doc_uses_real_repository_data() {
3349 let source = KanidmRestoreSource {
3350 local: None,
3351 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3352 name: "backup-1".to_string(),
3353 }),
3354 };
3355 let restore = make_restore(source, None);
3356 let target = super::super::crd::Kanidm::default();
3357 let doc_str = super::build_download_operation_doc(
3358 &restore,
3359 &target,
3360 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3361 "019c7c76-f423-7a12-8f41-2bea7588a303",
3362 "real-bucket",
3363 "real-prefix",
3364 "https://real-endpoint.com",
3365 "eu-west-1",
3366 true,
3367 false,
3368 None,
3369 );
3370 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3371 assert_eq!(parsed["bucket"], "real-bucket");
3372 assert_eq!(parsed["prefix"], "real-prefix");
3373 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3374 assert_eq!(parsed["region"], "eu-west-1");
3375 assert_eq!(parsed["forcePathStyle"], true);
3376 assert_eq!(
3377 parsed["manifestKey"],
3378 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3379 );
3380 assert_eq!(
3381 parsed["expectedBackupId"],
3382 "019c7c76-f423-7a12-8f41-2bea7588a303"
3383 );
3384 }
3385
3386 #[test]
3387 fn safety_backup_ref_is_set_after_verification() {
3388 let status = KanidmRestoreStatus {
3389 phase: KanidmRestorePhase::SafetyBackup,
3390 safety_backup_job_name: Some("test-safety-backup".to_string()),
3391 safety_backup_ref: Some("safety-test-restore".to_string()),
3392 safety_backup_expected_backup_id: Some(
3393 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3394 ),
3395 safety_backup_manifest_key: Some(
3396 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3397 ),
3398 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3399 ..Default::default()
3400 };
3401 assert!(status.safety_backup_ref.is_some());
3402 assert!(
3403 status
3404 .safety_backup_ref
3405 .as_ref()
3406 .unwrap()
3407 .starts_with("safety-")
3408 );
3409 assert!(status.safety_backup_expected_backup_id.is_some());
3410 assert!(status.safety_backup_manifest_key.is_some());
3411 assert!(status.safety_backup_payload_sha256.is_some());
3412 }
3413
3414 #[test]
3415 fn database_mutation_not_started_without_safety_backup_ref() {
3416 let status = KanidmRestoreStatus {
3417 phase: KanidmRestorePhase::PreparingSource,
3418 safety_backup_ref: None,
3419 database_mutation_started: false,
3420 ..Default::default()
3421 };
3422 assert!(!status.database_mutation_started);
3423 assert!(status.safety_backup_ref.is_none());
3424 }
3425
3426 #[test]
3427 fn pre_boundary_failure_does_not_set_mutation_started() {
3428 let status = KanidmRestoreStatus {
3429 phase: KanidmRestorePhase::Failed,
3430 message: Some(
3431 "safety backup job failed; target restored to original state".to_string(),
3432 ),
3433 database_mutation_started: false,
3434 ..Default::default()
3435 };
3436 assert!(!status.database_mutation_started);
3437 }
3438
3439 #[test]
3440 fn shared_volume_constants_are_distinct() {
3441 assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3442 assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3443 }
3444
3445 #[test]
3446 fn validate_backup_compatibility_matching_versions_passes() {
3447 let source = KanidmRestoreSource {
3448 local: None,
3449 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3450 name: "backup-1".to_string(),
3451 }),
3452 };
3453 let restore = make_restore(source, None);
3454 let target = kanidm_with_version("1.10.4");
3455 let backup = kaniop_backup_core::crd::KanidmBackup {
3456 metadata: kube::api::ObjectMeta::default(),
3457 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3458 backup_id: "id".to_string(),
3459 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3460 name: "k".to_string(),
3461 uid: "uid".to_string(),
3462 },
3463 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3464 name: "repo".to_string(),
3465 },
3466 manifest_key: "key".to_string(),
3467 },
3468 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3469 kanidm_version: Some("1.10.4".to_string()),
3470 ..Default::default()
3471 }),
3472 };
3473 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3474 }
3475
3476 #[test]
3477 fn validate_backup_compatibility_version_mismatch_fails() {
3478 let source = KanidmRestoreSource {
3479 local: None,
3480 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3481 name: "backup-1".to_string(),
3482 }),
3483 };
3484 let restore = make_restore(source, None);
3485 let target = kanidm_with_version("1.11.0");
3486 let backup = kaniop_backup_core::crd::KanidmBackup {
3487 metadata: kube::api::ObjectMeta::default(),
3488 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3489 backup_id: "id".to_string(),
3490 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3491 name: "k".to_string(),
3492 uid: "uid".to_string(),
3493 },
3494 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3495 name: "repo".to_string(),
3496 },
3497 manifest_key: "key".to_string(),
3498 },
3499 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3500 kanidm_version: Some("1.10.4".to_string()),
3501 ..Default::default()
3502 }),
3503 };
3504 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3505 }
3506
3507 #[test]
3508 fn validate_backup_compatibility_no_backup_version_passes() {
3509 let source = KanidmRestoreSource {
3510 local: None,
3511 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3512 name: "backup-1".to_string(),
3513 }),
3514 };
3515 let restore = make_restore(source, None);
3516 let target = super::super::crd::Kanidm::default();
3517 let backup = kaniop_backup_core::crd::KanidmBackup {
3518 metadata: kube::api::ObjectMeta::default(),
3519 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3520 backup_id: "id".to_string(),
3521 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3522 name: "k".to_string(),
3523 uid: "uid".to_string(),
3524 },
3525 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3526 name: "repo".to_string(),
3527 },
3528 manifest_key: "key".to_string(),
3529 },
3530 status: None,
3531 };
3532 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3533 }
3534
3535 #[test]
3536 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3537 let source = KanidmRestoreSource {
3538 local: None,
3539 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3540 name: "backup-1".to_string(),
3541 }),
3542 };
3543 let mut restore = make_restore(source, None);
3544 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3545 let target = super::super::crd::Kanidm::default();
3546 let backup = kaniop_backup_core::crd::KanidmBackup {
3547 metadata: kube::api::ObjectMeta::default(),
3548 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3549 backup_id: "id".to_string(),
3550 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3551 name: "k".to_string(),
3552 uid: "uid".to_string(),
3553 },
3554 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3555 name: "repo".to_string(),
3556 },
3557 manifest_key: "key".to_string(),
3558 },
3559 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3560 image_digest: Some("sha256:originaldigest".to_string()),
3561 ..Default::default()
3562 }),
3563 };
3564 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3565 }
3566
3567 #[test]
3568 fn validate_backup_compatibility_image_digest_match_passes() {
3569 let source = KanidmRestoreSource {
3570 local: None,
3571 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3572 name: "backup-1".to_string(),
3573 }),
3574 };
3575 let mut restore = make_restore(source, None);
3576 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3577 let target = super::super::crd::Kanidm::default();
3578 let backup = kaniop_backup_core::crd::KanidmBackup {
3579 metadata: kube::api::ObjectMeta::default(),
3580 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3581 backup_id: "id".to_string(),
3582 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3583 name: "k".to_string(),
3584 uid: "uid".to_string(),
3585 },
3586 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3587 name: "repo".to_string(),
3588 },
3589 manifest_key: "key".to_string(),
3590 },
3591 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3592 image_digest: Some("sha256:abc123".to_string()),
3593 ..Default::default()
3594 }),
3595 };
3596 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3597 }
3598
3599 #[test]
3600 fn safety_backup_id_is_deterministic() {
3601 let source = KanidmRestoreSource {
3602 local: None,
3603 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3604 name: "backup-1".to_string(),
3605 }),
3606 };
3607 let restore = make_restore(source, None);
3608 let id1 = super::compute_safety_backup_id(&restore);
3609 let id2 = super::compute_safety_backup_id(&restore);
3610 assert_eq!(id1, id2);
3611 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3612 }
3613
3614 #[test]
3615 fn safety_backup_id_differs_for_different_restores() {
3616 let source1 = KanidmRestoreSource {
3617 local: None,
3618 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3619 name: "backup-1".to_string(),
3620 }),
3621 };
3622 let restore1 = make_restore(source1, None);
3623 let mut restore2 = make_restore(
3624 KanidmRestoreSource {
3625 local: None,
3626 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3627 name: "backup-2".to_string(),
3628 }),
3629 },
3630 None,
3631 );
3632 restore2.metadata.name = Some("different-restore".to_string());
3633 let id1 = super::compute_safety_backup_id(&restore1);
3634 let id2 = super::compute_safety_backup_id(&restore2);
3635 assert_ne!(id1, id2);
3636 }
3637
3638 #[test]
3639 fn safety_manifest_key_is_deterministic() {
3640 let source = KanidmRestoreSource {
3641 local: None,
3642 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3643 name: "backup-1".to_string(),
3644 }),
3645 };
3646 let restore = make_restore(source, None);
3647 let key1 = super::compute_safety_manifest_key(&restore);
3648 let key2 = super::compute_safety_manifest_key(&restore);
3649 assert_eq!(key1, key2);
3650 assert!(key1.contains("manifest.json"));
3651 assert!(key1.starts_with("v1/tenants/"));
3652 }
3653
3654 #[test]
3655 fn safety_backup_id_survives_restart() {
3656 let source = KanidmRestoreSource {
3657 local: None,
3658 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3659 name: "backup-1".to_string(),
3660 }),
3661 };
3662 let mut restore = make_restore(source, None);
3663 let original_id = super::compute_safety_backup_id(&restore);
3664 restore.status = Some(KanidmRestoreStatus {
3665 phase: KanidmRestorePhase::SafetyBackup,
3666 safety_backup_expected_backup_id: Some(original_id.clone()),
3667 ..Default::default()
3668 });
3669 let recomputed_id = super::compute_safety_backup_id(&restore);
3670 assert_eq!(original_id, recomputed_id);
3671 }
3672
3673 #[test]
3674 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3675 let source = KanidmRestoreSource {
3676 local: None,
3677 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3678 name: "backup-1".to_string(),
3679 }),
3680 };
3681 let restore = make_restore(source, None);
3682 assert!(super::requires_safety_backup(&restore));
3683 let status_without_ref = KanidmRestoreStatus {
3684 phase: KanidmRestorePhase::RestoringPrimary,
3685 safety_backup_ref: None,
3686 database_mutation_started: false,
3687 ..Default::default()
3688 };
3689 assert!(status_without_ref.safety_backup_ref.is_none());
3690 assert!(!status_without_ref.database_mutation_started);
3691 }
3692
3693 #[test]
3694 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3695 let status_with_ref = KanidmRestoreStatus {
3696 phase: KanidmRestorePhase::RestoringPrimary,
3697 safety_backup_ref: Some("safety-test-restore".to_string()),
3698 safety_backup_manifest_key: Some(
3699 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3700 ),
3701 safety_backup_payload_sha256: Some("abc123".to_string()),
3702 database_mutation_started: false,
3703 ..Default::default()
3704 };
3705 assert!(status_with_ref.safety_backup_ref.is_some());
3706 assert!(!status_with_ref.database_mutation_started);
3707 }
3708
3709 #[test]
3710 fn result_document_verification_rejects_missing_backup_id() {
3711 let result_json = r#"{
3712 "apiVersion": "backup.kaniop.rs/v1alpha1",
3713 "kind": "ResultDocument",
3714 "operation": "upload",
3715 "success": true,
3716 "exitCode": "success",
3717 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3718 "payloadSha256": "abc123"
3719 }"#;
3720 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3721 assert!(result.backup_id.is_none());
3722 }
3723
3724 #[test]
3725 fn result_document_verification_rejects_missing_manifest_key() {
3726 let result_json = r#"{
3727 "apiVersion": "backup.kaniop.rs/v1alpha1",
3728 "kind": "ResultDocument",
3729 "operation": "upload",
3730 "success": true,
3731 "exitCode": "success",
3732 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3733 "payloadSha256": "abc123"
3734 }"#;
3735 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3736 assert!(result.manifest_key.is_none());
3737 }
3738
3739 #[test]
3740 fn result_document_verification_rejects_missing_payload_sha256() {
3741 let result_json = r#"{
3742 "apiVersion": "backup.kaniop.rs/v1alpha1",
3743 "kind": "ResultDocument",
3744 "operation": "upload",
3745 "success": true,
3746 "exitCode": "success",
3747 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3748 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3749 }"#;
3750 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3751 assert!(result.payload_sha256.is_none());
3752 }
3753
3754 #[test]
3755 fn result_document_verification_rejects_failure_exit_code() {
3756 let result_json = r#"{
3757 "apiVersion": "backup.kaniop.rs/v1alpha1",
3758 "kind": "ResultDocument",
3759 "operation": "upload",
3760 "success": false,
3761 "exitCode": "integrity",
3762 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3763 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3764 "payloadSha256": "abc123"
3765 }"#;
3766 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3767 assert!(!result.success);
3768 assert_eq!(
3769 result.exit_code,
3770 kaniop_backup_core::result::ExitCode::Integrity
3771 );
3772 }
3773
3774 #[test]
3775 fn result_document_verification_rejects_wrong_operation() {
3776 let result_json = r#"{
3777 "apiVersion": "backup.kaniop.rs/v1alpha1",
3778 "kind": "ResultDocument",
3779 "operation": "download",
3780 "success": true,
3781 "exitCode": "success",
3782 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3783 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3784 "payloadSha256": "abc123"
3785 }"#;
3786 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3787 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3788 }
3789
3790 #[test]
3791 fn result_document_verification_accepts_valid_document() {
3792 let result_json = r#"{
3793 "apiVersion": "backup.kaniop.rs/v1alpha1",
3794 "kind": "ResultDocument",
3795 "operation": "upload",
3796 "success": true,
3797 "exitCode": "success",
3798 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3799 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3800 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3801 }"#;
3802 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3803 assert!(result.success);
3804 assert_eq!(
3805 result.exit_code,
3806 kaniop_backup_core::result::ExitCode::Success
3807 );
3808 assert_eq!(result.operation, "upload");
3809 assert!(result.backup_id.is_some());
3810 assert!(result.manifest_key.is_some());
3811 assert!(result.payload_sha256.is_some());
3812 }
3813
3814 #[test]
3815 fn termination_message_path_constant_is_valid() {
3816 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3817 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3818 }
3819
3820 #[test]
3821 fn status_tracks_safety_backup_evidence() {
3822 let status = KanidmRestoreStatus {
3823 phase: KanidmRestorePhase::PreparingSource,
3824 safety_backup_ref: Some("safety-test-restore".to_string()),
3825 safety_backup_expected_backup_id: Some(
3826 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3827 ),
3828 safety_backup_manifest_key: Some(
3829 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3830 ),
3831 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3832 ..Default::default()
3833 };
3834 assert!(status.safety_backup_ref.is_some());
3835 assert!(status.safety_backup_expected_backup_id.is_some());
3836 assert!(status.safety_backup_manifest_key.is_some());
3837 assert!(status.safety_backup_payload_sha256.is_some());
3838 }
3839
3840 #[test]
3841 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3842 let invalid_json = "not valid json";
3843 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3844 assert!(result.is_err());
3845 }
3846
3847 #[test]
3848 fn fail_closed_semantics_preserved_on_oversized_result() {
3849 let oversized_json = format!(
3850 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3851 kaniop_backup_core::result::RESULT_DOC_VERSION,
3852 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3853 );
3854 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3855 assert!(result.is_err());
3856 }
3857
3858 #[test]
3859 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3860 let source = KanidmRestoreSource {
3861 local: None,
3862 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3863 name: "backup-1".to_string(),
3864 }),
3865 };
3866 let mut restore = make_restore(source, None);
3867 restore.metadata.namespace = Some("test-ns".to_string());
3868 let target = super::super::crd::Kanidm::default();
3869 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3870 let doc_str = super::build_safety_upload_operation_doc(
3871 &restore,
3872 &target,
3873 "test-backup-id",
3874 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3875 "my-bucket",
3876 "prod",
3877 "https://s3.example.com",
3878 "us-east-1",
3879 false,
3880 false,
3881 Some(&ca_path),
3882 )
3883 .unwrap();
3884 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3885 assert_eq!(parsed["caBundlePath"], ca_path);
3886 }
3887
3888 #[test]
3889 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3890 let source = KanidmRestoreSource {
3891 local: None,
3892 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3893 name: "backup-1".to_string(),
3894 }),
3895 };
3896 let mut restore = make_restore(source, None);
3897 restore.metadata.namespace = Some("test-ns".to_string());
3898 let target = super::super::crd::Kanidm::default();
3899 let doc_str = super::build_safety_upload_operation_doc(
3900 &restore,
3901 &target,
3902 "test-backup-id",
3903 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3904 "my-bucket",
3905 "prod",
3906 "https://s3.example.com",
3907 "us-east-1",
3908 false,
3909 false,
3910 None,
3911 )
3912 .unwrap();
3913 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3914 assert!(parsed["caBundlePath"].is_null());
3915 }
3916
3917 #[test]
3918 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3919 let source = KanidmRestoreSource {
3920 local: None,
3921 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3922 name: "backup-1".to_string(),
3923 }),
3924 };
3925 let restore = make_restore(source, None);
3926 let target = super::super::crd::Kanidm::default();
3927 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3928 let doc_str = super::build_download_operation_doc(
3929 &restore,
3930 &target,
3931 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3932 "019c7c76-f423-7a12-8f41-2bea7588a303",
3933 "real-bucket",
3934 "real-prefix",
3935 "https://real-endpoint.com",
3936 "eu-west-1",
3937 true,
3938 false,
3939 Some(&ca_path),
3940 );
3941 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3942 assert_eq!(parsed["caBundlePath"], ca_path);
3943 }
3944
3945 #[test]
3946 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3947 let source = KanidmRestoreSource {
3948 local: None,
3949 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3950 name: "backup-1".to_string(),
3951 }),
3952 };
3953 let restore = make_restore(source, None);
3954 let target = super::super::crd::Kanidm::default();
3955 let doc_str = super::build_download_operation_doc(
3956 &restore,
3957 &target,
3958 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3959 "019c7c76-f423-7a12-8f41-2bea7588a303",
3960 "real-bucket",
3961 "real-prefix",
3962 "https://real-endpoint.com",
3963 "eu-west-1",
3964 true,
3965 false,
3966 None,
3967 );
3968 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3969 assert!(parsed["caBundlePath"].is_null());
3970 }
3971
3972 #[test]
3973 fn source_prep_result_operation_is_download() {
3974 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3975 assert_ne!(
3976 super::SOURCE_PREP_RESULT_OPERATION,
3977 super::SAFETY_BACKUP_RESULT_OPERATION
3978 );
3979 }
3980
3981 #[test]
3982 fn download_result_document_verification_accepts_valid_document() {
3983 let result_json = r#"{
3984 "apiVersion": "backup.kaniop.rs/v1alpha1",
3985 "kind": "ResultDocument",
3986 "operation": "download",
3987 "success": true,
3988 "exitCode": "success",
3989 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3990 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3991 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3992 }"#;
3993 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3994 assert!(result.success);
3995 assert_eq!(
3996 result.exit_code,
3997 kaniop_backup_core::result::ExitCode::Success
3998 );
3999 assert_eq!(result.operation, "download");
4000 assert_eq!(
4001 result.backup_id.as_deref(),
4002 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4003 );
4004 assert!(result.manifest_key.is_some());
4005 assert!(result.payload_sha256.is_some());
4006 }
4007
4008 #[test]
4009 fn download_result_document_rejects_wrong_operation() {
4010 let result_json = r#"{
4011 "apiVersion": "backup.kaniop.rs/v1alpha1",
4012 "kind": "ResultDocument",
4013 "operation": "upload",
4014 "success": true,
4015 "exitCode": "success",
4016 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4017 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4018 "payloadSha256": "abc123"
4019 }"#;
4020 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4021 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4022 }
4023
4024 #[test]
4025 fn download_result_document_rejects_failure_success_field() {
4026 let result_json = r#"{
4027 "apiVersion": "backup.kaniop.rs/v1alpha1",
4028 "kind": "ResultDocument",
4029 "operation": "download",
4030 "success": false,
4031 "exitCode": "retryable",
4032 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4033 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4034 "payloadSha256": "abc123"
4035 }"#;
4036 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4037 assert!(!result.success);
4038 assert_ne!(
4039 result.exit_code,
4040 kaniop_backup_core::result::ExitCode::Success
4041 );
4042 }
4043
4044 #[test]
4045 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4046 let status = KanidmRestoreStatus {
4047 phase: KanidmRestorePhase::PreparingSource,
4048 database_mutation_started: false,
4049 ..Default::default()
4050 };
4051 assert!(!status.database_mutation_started);
4052 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4053 }
4054}