1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, ca_bundle_env_var, ca_bundle_path,
9};
10use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
11use kaniop_backup_core::operation::{
12 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
13};
14use kaniop_backup_core::result::{ExitCode, parse_result_document};
15use kaniop_k8s_util::error::{Error, Result};
16
17use std::collections::BTreeSet;
18use std::sync::Arc;
19use std::time::Duration;
20
21use futures::StreamExt;
22use k8s_openapi::api::apps::v1::StatefulSet;
23use k8s_openapi::api::batch::v1::{Job, JobSpec};
24use k8s_openapi::api::core::v1::{
25 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
26 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
27 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
28};
29use k8s_openapi::api::storage::v1::VolumeAttachment;
30use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
31use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
32use k8s_openapi::jiff::Timestamp;
33use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
34use kube::runtime::controller::{Action, Controller};
35use kube::runtime::events::{Event, EventType, Recorder, Reporter};
36use kube::runtime::finalizer::{Event as Finalizer, finalizer};
37use kube::runtime::watcher;
38use kube::{Api, Client, CustomResource, Resource, ResourceExt};
39use opentelemetry::metrics::{Counter, Histogram};
40use opentelemetry::{KeyValue, global};
41#[cfg(feature = "schemars")]
42use schemars::JsonSchema;
43use serde::{Deserialize, Serialize};
44use serde_json::json;
45use tracing::{debug, error, info, warn};
46
47pub const CONTROLLER_ID: &str = "kanidm-restore";
48pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
49const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
50const DATA_VOLUME: &str = "kanidm-data";
51const STAGING_VOLUME: &str = "restore-staging";
52const SHARED_VOLUME: &str = "safety-backup-shared";
53const DATA_PATH: &str = "/data";
54const TLS_VOLUME: &str = "kanidm-certs";
55const TLS_PATH: &str = "/etc/kanidm/tls";
56const BACKUP_PATH: &str = "/data";
57const STAGING_PATH: &str = "/staging";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76 feature = "schemars",
77 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80 category = "kaniop",
81 group = "kaniop.rs",
82 version = "v1beta1",
83 kind = "KanidmRestore",
84 plural = "kanidmrestores",
85 singular = "kanidmrestore",
86 shortname = "idmrestore",
87 namespaced,
88 status = "KanidmRestoreStatus",
89 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95 pub target_ref: KanidmRestoreTargetRef,
96 pub source: KanidmRestoreSource,
97 pub restore_image: String,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106 pub name: String,
107 pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113 feature = "schemars",
114 schemars(extend("x-kubernetes-validations" = [
115 {
116 "message": "exactly one of local or backupRef must be set",
117 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118 }
119 ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123 #[serde(skip_serializing_if = "Option::is_none")]
125 pub local: Option<KanidmRestoreLocalSource>,
126 #[serde(skip_serializing_if = "Option::is_none")]
128 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135 pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143 pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151 #[serde(skip_serializing_if = "Option::is_none")]
153 pub repository_ref: Option<SafetyBackupRepositoryRef>,
154 #[serde(default)]
156 pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163 pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169 #[default]
170 Pending,
171 Validating,
172 Quiescing,
173 SafetyBackup,
174 PreparingSource,
175 RestoringPrimary,
176 Verifying,
177 RebuildingReplicas,
178 Resuming,
179 Completed,
180 Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187 #[serde(skip_serializing_if = "Option::is_none")]
188 pub observed_generation: Option<i64>,
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_target_uid: Option<String>,
191 #[serde(default)]
192 pub phase: KanidmRestorePhase,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 pub restore_job_name: Option<String>,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub verify_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub safety_backup_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub source_prep_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub safety_backup_ref: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_expected_backup_id: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_manifest_key: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_payload_sha256: Option<String>,
209 #[serde(default)]
210 pub replicas_cleared: bool,
211 #[serde(default)]
212 pub certificates_cleared: bool,
213 #[serde(default)]
214 pub database_mutation_started: bool,
215 #[serde(default, skip_serializing_if = "Vec::is_empty")]
216 pub original_replicas: Vec<ReplicaCountEntry>,
217 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218 pub phase_timestamps: std::collections::BTreeMap<String, String>,
219 #[serde(default, skip_serializing_if = "Vec::is_empty")]
220 #[cfg_attr(
221 feature = "schemars",
222 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223 )]
224 pub conditions: Vec<Condition>,
225 #[serde(skip_serializing_if = "Option::is_none")]
226 pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233 pub group: String,
234 pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239 attempts: Counter<u64>,
240 outcomes: Counter<u64>,
241 duration_seconds: Histogram<f64>,
242 break_glass_total: Counter<u64>,
243 safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247 fn new() -> Self {
248 let meter = global::meter("kaniop");
249 Self {
250 attempts: meter
251 .u64_counter("kaniop_restore_attempts_total")
252 .with_description("Number of Kanidm restore attempts started")
253 .build(),
254 outcomes: meter
255 .u64_counter("kaniop_restore_outcomes_total")
256 .with_description("Number of terminal Kanidm restore outcomes")
257 .build(),
258 duration_seconds: meter
259 .f64_histogram("kaniop_restore_duration_seconds")
260 .with_description("Kanidm restore duration from object creation to terminal phase")
261 .with_unit("s")
262 .build(),
263 break_glass_total: meter
264 .u64_counter("kaniop_restore_break_glass_total")
265 .with_description("Number of break-glass safety backup overrides")
266 .build(),
267 safety_backup_duration_seconds: meter
268 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269 .with_description("Duration of pre-restore safety backup creation")
270 .with_unit("s")
271 .build(),
272 }
273 }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278 client: Client,
279 recorder: Recorder,
280 metrics: RestoreMetrics,
281}
282
283pub async fn run(client: Client) {
284 let api = Api::<KanidmRestore>::all(client.clone());
285 let recorder = Recorder::new(
286 client.clone(),
287 Reporter {
288 controller: CONTROLLER_ID.into(),
289 instance: None,
290 },
291 );
292 let ctx = Arc::new(RestoreContext {
293 client,
294 recorder,
295 metrics: RestoreMetrics::new(),
296 });
297 info!("starting {CONTROLLER_ID} controller");
298 Controller::new(api, watcher::Config::default().any_semantic())
299 .shutdown_on_signal()
300 .run(reconcile_restore, error_policy, ctx)
301 .for_each(|result| async move {
302 if let Err(error) = result {
303 error!(%error, "KanidmRestore reconciliation failed");
304 }
305 })
306 .await;
307}
308
309fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
310 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
311 Action::requeue(Duration::from_secs(5))
312}
313
314async fn reconcile_restore(
315 restore: Arc<KanidmRestore>,
316 ctx: Arc<RestoreContext>,
317) -> Result<Action> {
318 let namespace = restore
319 .namespace()
320 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
321 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
322 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
323 let ctx = ctx.clone();
324 async move {
325 match event {
326 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
327 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
328 }
329 }
330 })
331 .await
332 .map_err(|error| {
333 Error::FinalizerError(
334 "failed on KanidmRestore finalizer".to_string(),
335 Box::new(error),
336 )
337 })
338}
339
340async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
341 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
342 match phase {
343 KanidmRestorePhase::Pending => {
344 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
345 Ok(Action::requeue(REQUEUE))
346 }
347 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
348 Ok(target) => {
349 mark_restoring(&restore, &target, &ctx).await?;
350 let mut status = restore.status.clone().unwrap_or_default();
351 status.observed_target_uid = target.uid();
352 status.original_replicas = capture_original_replicas(&target);
353 status.phase = KanidmRestorePhase::Quiescing;
354 status.message = None;
355 patch_status(&restore, &ctx, status).await?;
356 Ok(Action::requeue(REQUEUE))
357 }
358 Err(error) => {
359 set_phase(
360 &restore,
361 &ctx,
362 KanidmRestorePhase::Failed,
363 Some(error.to_string()),
364 )
365 .await?;
366 Ok(Action::requeue(Duration::from_secs(300)))
367 }
368 },
369 KanidmRestorePhase::Quiescing => {
370 let target = get_target(&restore, &ctx).await?;
371 scale_all(&target, &ctx, 0).await?;
372 if target_pods_stopped(&target, &ctx).await?
373 && target_volumes_detached(&target, &ctx).await?
374 {
375 let next_phase = if requires_safety_backup(&restore) {
376 KanidmRestorePhase::SafetyBackup
377 } else {
378 record_break_glass(&restore, &ctx).await?;
379 KanidmRestorePhase::PreparingSource
380 };
381 if next_phase == KanidmRestorePhase::PreparingSource {
382 let refreshed = Api::<KanidmRestore>::namespaced(
383 ctx.client.clone(),
384 restore.namespace().as_deref().unwrap_or_default(),
385 )
386 .get(&restore.name_any())
387 .await
388 .map_err(|error| {
389 Error::kube_error(
390 "get",
391 "KanidmRestore",
392 restore.namespace().as_deref().unwrap_or_default(),
393 restore.name_any(),
394 error,
395 )
396 })?;
397 set_phase(&refreshed, &ctx, next_phase, None).await?;
398 } else {
399 set_phase(&restore, &ctx, next_phase, None).await?;
400 }
401 }
402 Ok(Action::requeue(REQUEUE))
403 }
404 KanidmRestorePhase::SafetyBackup => {
405 let target = get_target(&restore, &ctx).await?;
406 let name = safety_backup_job_name(&restore);
407 let expected_backup_id = compute_safety_backup_id(&restore);
408 let expected_manifest_key = compute_safety_manifest_key(&restore);
409 ensure_safety_backup_job(
410 &restore,
411 &target,
412 &ctx,
413 &name,
414 &expected_backup_id,
415 &expected_manifest_key,
416 )
417 .await?;
418 let mut status = restore.status.clone().unwrap_or_default();
419 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
420 match job_state(&restore, &ctx, &name).await? {
421 JobState::Complete => {
422 let result =
423 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
424 match result {
425 Ok(verified) => {
426 status.safety_backup_job_name = Some(name);
427 status.safety_backup_ref =
428 Some(format!("safety-{}", restore.name_any()));
429 status.safety_backup_manifest_key = Some(verified.manifest_key);
430 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
431 record_safety_backup_duration(&restore, &ctx).await;
432 status.phase = KanidmRestorePhase::PreparingSource;
433 status.message = None;
434 patch_status(&restore, &ctx, status).await?;
435 }
436 Err(error) => {
437 resume_before_mutation(&restore, &ctx).await?;
438 set_phase(
439 &restore,
440 &ctx,
441 KanidmRestorePhase::Failed,
442 Some(format!("safety backup result verification failed: {error}")),
443 )
444 .await?;
445 }
446 }
447 }
448 JobState::Failed => {
449 resume_before_mutation(&restore, &ctx).await?;
450 set_phase(
451 &restore,
452 &ctx,
453 KanidmRestorePhase::Failed,
454 Some(
455 "safety backup job failed; target restored to original state"
456 .to_string(),
457 ),
458 )
459 .await?;
460 }
461 JobState::Running => {
462 patch_status(&restore, &ctx, status).await?;
463 }
464 }
465 Ok(Action::requeue(REQUEUE))
466 }
467 KanidmRestorePhase::PreparingSource => {
468 if is_remote_source(&restore) {
469 let target = get_target(&restore, &ctx).await?;
470 let name = source_prep_job_name(&restore);
471 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
472 match job_state(&restore, &ctx, &name).await? {
473 JobState::Complete => {
474 let backup_id = restore
475 .spec
476 .source
477 .backup_ref
478 .as_ref()
479 .map(|b| b.name.clone())
480 .unwrap_or_default();
481 let backup = {
482 let ns = restore.namespace().unwrap();
483 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
484 .get(&backup_id)
485 .await
486 .map_err(|e| {
487 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
488 })?
489 };
490 let expected_backup_id = &backup.spec.backup_id;
491 let result =
492 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
493 .await;
494 match result {
495 Ok(_verified) => {
496 let mut status = restore.status.clone().unwrap_or_default();
497 status.source_prep_job_name = Some(name);
498 status.phase = KanidmRestorePhase::RestoringPrimary;
499 status.message = None;
500 patch_status(&restore, &ctx, status).await?;
501 }
502 Err(error) => {
503 resume_before_mutation(&restore, &ctx).await?;
504 set_phase(
505 &restore,
506 &ctx,
507 KanidmRestorePhase::Failed,
508 Some(format!(
509 "source preparation result verification failed: {error}"
510 )),
511 )
512 .await?;
513 }
514 }
515 }
516 JobState::Failed => {
517 resume_before_mutation(&restore, &ctx).await?;
518 set_phase(
519 &restore,
520 &ctx,
521 KanidmRestorePhase::Failed,
522 Some(
523 "source preparation job failed; target restored to original state"
524 .to_string(),
525 ),
526 )
527 .await?;
528 }
529 JobState::Running => {}
530 }
531 } else {
532 let target = get_target(&restore, &ctx).await?;
533 let name = source_check_job_name(&restore);
534 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
535 match job_state(&restore, &ctx, &name).await? {
536 JobState::Complete => {
537 let mut status = restore.status.clone().unwrap_or_default();
538 status.source_prep_job_name = Some(name);
539 status.phase = KanidmRestorePhase::RestoringPrimary;
540 status.message = None;
541 patch_status(&restore, &ctx, status).await?;
542 }
543 JobState::Failed => {
544 resume_before_mutation(&restore, &ctx).await?;
545 set_phase(
546 &restore,
547 &ctx,
548 KanidmRestorePhase::Failed,
549 Some(
550 "local backup file check failed; target restored to original state"
551 .to_string(),
552 ),
553 )
554 .await?;
555 }
556 JobState::Running => {}
557 }
558 }
559 Ok(Action::requeue(REQUEUE))
560 }
561 KanidmRestorePhase::RestoringPrimary => {
562 let target = get_target(&restore, &ctx).await?;
563 let status = restore.status.as_ref().cloned().unwrap_or_default();
564 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
565 return Err(Error::MissingData(
566 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
567 ));
568 }
569 if !status.database_mutation_started {
570 let mut status = status;
571 status.database_mutation_started = true;
572 status.message = Some(
573 "database mutation boundary persisted; starting primary restore".to_string(),
574 );
575 patch_status(&restore, &ctx, status).await?;
576 return Ok(Action::requeue(REQUEUE));
577 }
578 let name = restore_job_name(&restore);
579 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
580 match job_state(&restore, &ctx, &name).await? {
581 JobState::Complete => {
582 let mut status = restore.status.clone().unwrap_or_default();
583 status.restore_job_name = Some(name);
584 status.phase = KanidmRestorePhase::Verifying;
585 status.message = None;
586 patch_status(&restore, &ctx, status).await?;
587 }
588 JobState::Failed => {
589 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
590 }
591 JobState::Running => {}
592 }
593 Ok(Action::requeue(REQUEUE))
594 }
595 KanidmRestorePhase::Verifying => {
596 let target = get_target(&restore, &ctx).await?;
597 let name = verify_job_name(&restore);
598 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
599 match job_state(&restore, &ctx, &name).await? {
600 JobState::Complete => {
601 let mut status = restore.status.clone().unwrap_or_default();
602 status.verify_job_name = Some(name);
603 status.phase = KanidmRestorePhase::RebuildingReplicas;
604 status.message = None;
605 patch_status(&restore, &ctx, status).await?;
606 }
607 JobState::Failed => {
608 fail_after_mutation(&restore, &ctx, "database verification failed").await?
609 }
610 JobState::Running => {}
611 }
612 Ok(Action::requeue(REQUEUE))
613 }
614 KanidmRestorePhase::RebuildingReplicas => {
615 let target = get_target(&restore, &ctx).await?;
616 let mut status = restore.status.clone().unwrap_or_default();
617 if !status.replicas_cleared {
618 if delete_secondary_pvcs(&target, &ctx).await? {
619 status.replicas_cleared = true;
620 status.message = Some("secondary database state cleared".to_string());
621 patch_status(&restore, &ctx, status).await?;
622 }
623 return Ok(Action::requeue(REQUEUE));
624 }
625
626 scale_primary(&target, &ctx, 1).await?;
627 if !primary_ready(&target, &ctx).await? {
628 return Ok(Action::requeue(REQUEUE));
629 }
630 if !status.certificates_cleared {
631 delete_replica_cert_secrets(&target, &ctx).await?;
632 delete_admin_secret(&target, &ctx).await?;
633 status.certificates_cleared = true;
634 status.message = Some(
635 "replica certificates and admin secret cleared for regeneration".to_string(),
636 );
637 patch_status(&restore, &ctx, status).await?;
638 return Ok(Action::requeue(REQUEUE));
639 }
640 status.phase = KanidmRestorePhase::Resuming;
641 status.message = None;
642 patch_status(&restore, &ctx, status).await?;
643 Ok(Action::requeue(REQUEUE))
644 }
645 KanidmRestorePhase::Resuming => {
646 let target = get_target(&restore, &ctx).await?;
647 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
648 clear_restoring(&restore, &target, &ctx).await?;
649 Ok(Action::requeue(Duration::from_secs(3600)))
650 }
651 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
652 Ok(Action::requeue(Duration::from_secs(3600)))
653 }
654 }
655}
656
657async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
658 let status = restore.status.clone().unwrap_or_default();
659 if status.database_mutation_started
660 && status.phase != KanidmRestorePhase::Completed
661 && status.phase != KanidmRestorePhase::Failed
662 {
663 reconcile_apply(restore.clone(), ctx.clone()).await?;
664 return Ok(Action::requeue(REQUEUE));
665 }
666 if let Ok(target) = get_target(&restore, &ctx).await {
667 let owns_maintenance =
668 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
669 if owns_maintenance && !status.database_mutation_started {
670 scale_desired(&target, &ctx).await?;
671 }
672 clear_restoring(&restore, &target, &ctx).await?;
673 }
674 Ok(Action::await_change())
675}
676
677async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
678 let target = get_target(restore, ctx).await?;
679 let actual_uid = target
680 .uid()
681 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
682 if actual_uid != restore.spec.target_ref.uid {
683 return Err(Error::MissingData(format!(
684 "target UID mismatch: expected {}, got {}",
685 restore.spec.target_ref.uid, actual_uid
686 )));
687 }
688 validate_source(restore)?;
689 if is_remote_source(restore) {
690 validate_backup_ref(restore, &target, ctx).await?;
691 }
692 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
693 {
694 return Err(Error::MissingData(format!(
695 "restoreImage must be the target's pinned Kanidm image (target image is {})",
696 target.spec.image
697 )));
698 }
699 let storage = target
700 .spec
701 .storage
702 .as_ref()
703 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
704 if storage.empty_dir.is_some()
705 || storage.ephemeral.is_some()
706 || storage.volume_claim_template.is_none()
707 {
708 return Err(Error::MissingData(
709 "restore requires PVC-backed Kanidm storage".to_string(),
710 ));
711 }
712 let primaries = target
713 .spec
714 .replica_groups
715 .iter()
716 .filter(|rg| rg.primary_node)
717 .count();
718 if primaries != 1 {
719 return Err(Error::MissingData(
720 "backup/restore requires exactly one primary replica group".to_string(),
721 ));
722 }
723 let ns = restore.namespace().unwrap();
724 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
725 .list(&ListParams::default())
726 .await
727 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
728 if restores.items.iter().any(|other| {
729 other.name_any() != restore.name_any()
730 && other.spec.target_ref.name == restore.spec.target_ref.name
731 && !matches!(
732 other.status.as_ref().map(|s| s.phase),
733 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
734 )
735 }) {
736 return Err(Error::MissingData(
737 "another active restore targets this Kanidm".to_string(),
738 ));
739 }
740 validate_safety_backup_config(restore)?;
741 Ok(target)
742}
743
744fn safe_basename(name: &str) -> bool {
745 !name.is_empty()
746 && name != "."
747 && name != ".."
748 && !name.contains('/')
749 && !name.contains('\\')
750 && !name.contains('\0')
751}
752
753fn mutable_image(image: &str) -> bool {
754 image == "kanidm/server:latest"
755 || image.ends_with(":latest")
756 || (!image.contains('@')
757 && !image
758 .rsplit('/')
759 .next()
760 .is_some_and(|part| part.contains(':')))
761}
762
763async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
764 let ns = restore
765 .namespace()
766 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
767 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
768 .get(&restore.spec.target_ref.name)
769 .await
770 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
771}
772
773async fn mark_restoring(
774 restore: &KanidmRestore,
775 target: &Kanidm,
776 ctx: &RestoreContext,
777) -> Result<()> {
778 let ns = target.namespace().unwrap();
779 let uid = restore
780 .uid()
781 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
782 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
783 .patch(
784 &target.name_any(),
785 &PatchParams::default(),
786 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
787 )
788 .await
789 .map(|_| ())
790 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
791}
792
793async fn clear_restoring(
794 restore: &KanidmRestore,
795 target: &Kanidm,
796 ctx: &RestoreContext,
797) -> Result<()> {
798 let ns = target.namespace().unwrap();
799 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
800 return Ok(());
801 }
802 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
803 .patch(
804 &target.name_any(),
805 &PatchParams::default(),
806 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
807 )
808 .await
809 .map(|_| ())
810 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
811}
812
813async fn patch_status(
814 restore: &KanidmRestore,
815 ctx: &RestoreContext,
816 mut status: KanidmRestoreStatus,
817) -> Result<()> {
818 let ns = restore.namespace().unwrap();
819 let previous_phase = restore.status.as_ref().map(|current| current.phase);
820 let next_phase = status.phase;
821 status.observed_generation = restore.metadata.generation;
822 update_restore_conditions(&mut status, restore.metadata.generation);
823
824 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
825 .patch_status(
826 &restore.name_any(),
827 &PatchParams::default(),
828 &Patch::Merge(json!({"status": &status})),
829 )
830 .await
831 .map_err(|e| {
832 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
833 })?;
834
835 if previous_phase != Some(next_phase) {
836 record_restore_transition(
837 restore,
838 ctx,
839 previous_phase,
840 next_phase,
841 status.message.as_deref(),
842 )
843 .await;
844 }
845 Ok(())
846}
847
848fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
849 let previous = status.conditions.clone();
850 let phase = status.phase;
851 let phase_reason = format!("{phase:?}");
852 let message = status
853 .message
854 .clone()
855 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
856 let terminal_success = phase == KanidmRestorePhase::Completed;
857 let terminal_failure = phase == KanidmRestorePhase::Failed;
858 let progressing = !terminal_success && !terminal_failure;
859
860 let break_glass = previous
861 .iter()
862 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
863 .cloned();
864 status.conditions = vec![
865 restore_condition(
866 &previous,
867 CONDITION_PROGRESSING,
868 if progressing {
869 CONDITION_TRUE
870 } else {
871 CONDITION_FALSE
872 },
873 &phase_reason,
874 &message,
875 generation,
876 ),
877 restore_condition(
878 &previous,
879 CONDITION_READY,
880 if terminal_success {
881 CONDITION_TRUE
882 } else {
883 CONDITION_FALSE
884 },
885 if terminal_success {
886 "RestoreCompleted"
887 } else {
888 "RestoreNotCompleted"
889 },
890 if terminal_success {
891 "Kanidm restore completed successfully."
892 } else {
893 "Kanidm restore has not completed successfully."
894 },
895 generation,
896 ),
897 restore_condition(
898 &previous,
899 CONDITION_FAILED,
900 if terminal_failure {
901 CONDITION_TRUE
902 } else {
903 CONDITION_FALSE
904 },
905 if terminal_failure {
906 "RestoreFailed"
907 } else {
908 "NoRestoreFailure"
909 },
910 if terminal_failure {
911 status
912 .message
913 .as_deref()
914 .unwrap_or("Kanidm restore failed.")
915 } else {
916 "No terminal restore failure has been recorded."
917 },
918 generation,
919 ),
920 ];
921 if let Some(condition) = break_glass {
922 status.conditions.push(condition);
923 }
924}
925
926fn restore_condition(
927 previous: &[Condition],
928 condition_type: &str,
929 condition_status: &str,
930 reason: &str,
931 message: &str,
932 generation: Option<i64>,
933) -> Condition {
934 let last_transition_time = previous
935 .iter()
936 .find(|condition| {
937 condition.type_ == condition_type
938 && condition.status == condition_status
939 && condition.reason == reason
940 })
941 .map(|condition| condition.last_transition_time.clone())
942 .unwrap_or_else(|| Time(Timestamp::now()));
943 Condition {
944 type_: condition_type.to_string(),
945 status: condition_status.to_string(),
946 reason: reason.to_string(),
947 message: message.to_string(),
948 last_transition_time,
949 observed_generation: generation,
950 }
951}
952
953async fn record_restore_transition(
954 restore: &KanidmRestore,
955 ctx: &RestoreContext,
956 previous_phase: Option<KanidmRestorePhase>,
957 phase: KanidmRestorePhase,
958 message: Option<&str>,
959) {
960 if phase == KanidmRestorePhase::Validating {
961 ctx.metrics.attempts.add(1, &[]);
962 }
963
964 let result = match phase {
965 KanidmRestorePhase::Completed => Some("success"),
966 KanidmRestorePhase::Failed => Some("failure"),
967 _ => None,
968 };
969 if let Some(result) = result {
970 let attributes = [KeyValue::new("result", result)];
971 ctx.metrics.outcomes.add(1, &attributes);
972 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
973 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
974 ctx.metrics.duration_seconds.record(elapsed, &attributes);
975 }
976 }
977
978 let reason = if phase == KanidmRestorePhase::Failed {
979 "RestoreFailed"
980 } else {
981 "RestorePhaseChanged"
982 };
983 let note = message.map(str::to_string).or_else(|| {
984 Some(format!(
985 "Kanidm restore phase changed from {} to {phase:?}.",
986 previous_phase
987 .map(|previous| format!("{previous:?}"))
988 .unwrap_or_else(|| "None".to_string())
989 ))
990 });
991 if let Err(error) = ctx
992 .recorder
993 .publish(
994 &Event {
995 type_: if phase == KanidmRestorePhase::Failed {
996 EventType::Warning
997 } else {
998 EventType::Normal
999 },
1000 reason: reason.to_string(),
1001 note,
1002 action: "Restore".to_string(),
1003 secondary: None,
1004 },
1005 &restore.object_ref(&()),
1006 )
1007 .await
1008 {
1009 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
1010 }
1011}
1012
1013async fn set_phase(
1014 restore: &KanidmRestore,
1015 ctx: &RestoreContext,
1016 phase: KanidmRestorePhase,
1017 message: Option<String>,
1018) -> Result<()> {
1019 let mut status = restore.status.clone().unwrap_or_default();
1020 status.phase = phase;
1021 status.message = message;
1022 patch_status(restore, ctx, status).await
1023}
1024
1025async fn fail_after_mutation(
1026 restore: &KanidmRestore,
1027 ctx: &RestoreContext,
1028 message: &str,
1029) -> Result<()> {
1030 set_phase(
1031 restore,
1032 ctx,
1033 KanidmRestorePhase::Failed,
1034 Some(message.to_string()),
1035 )
1036 .await
1037}
1038
1039fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1040 target
1041 .spec
1042 .replica_groups
1043 .iter()
1044 .find(|rg| rg.primary_node)
1045 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1046}
1047
1048async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1049 let ns = target.namespace().unwrap();
1050 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1051 for rg in &target.spec.replica_groups {
1052 let name = target.statefulset_name(&rg.name);
1053 api.patch(
1054 &name,
1055 &PatchParams::default(),
1056 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1057 )
1058 .await
1059 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1060 }
1061 Ok(())
1062}
1063
1064async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1065 let ns = target.namespace().unwrap();
1066 let rg = primary_group(target)?;
1067 let name = target.statefulset_name(&rg.name);
1068 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1069 .patch(
1070 &name,
1071 &PatchParams::default(),
1072 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1073 )
1074 .await
1075 .map(|_| ())
1076 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1077}
1078
1079async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1080 let ns = target.namespace().unwrap();
1081 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1082 for rg in &target.spec.replica_groups {
1083 let name = target.statefulset_name(&rg.name);
1084 api.patch(
1085 &name,
1086 &PatchParams::default(),
1087 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1088 )
1089 .await
1090 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1091 }
1092 Ok(())
1093}
1094
1095async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1096 let ns = target.namespace().unwrap();
1097 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1098 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1099 .await
1100 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1101 Ok(pods.items.is_empty())
1102}
1103
1104async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1105 let ns = target.namespace().unwrap();
1106 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1107 let mut pv_names = BTreeSet::new();
1108 let primary_name = primary_pvc_name(target)?;
1109 let primary_pvc = pvc_api
1110 .get(&primary_name)
1111 .await
1112 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1113 let primary_pv = primary_pvc
1114 .spec
1115 .as_ref()
1116 .and_then(|spec| spec.volume_name.clone())
1117 .ok_or_else(|| {
1118 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1119 })?;
1120 pv_names.insert(primary_pv);
1121
1122 for rg in &target.spec.replica_groups {
1123 let sts = target.statefulset_name(&rg.name);
1124 for ordinal in 0..rg.replicas {
1125 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1126 if name == primary_name {
1127 continue;
1128 }
1129 if let Some(pvc) = pvc_api
1130 .get_opt(&name)
1131 .await
1132 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1133 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1134 {
1135 pv_names.insert(pv_name);
1136 }
1137 }
1138 }
1139
1140 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1141 .list(&ListParams::default())
1142 .await
1143 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1144 Ok(!attachments.items.iter().any(|attachment| {
1145 attachment
1146 .spec
1147 .source
1148 .persistent_volume_name
1149 .as_ref()
1150 .is_some_and(|pv_name| pv_names.contains(pv_name))
1151 }))
1152}
1153
1154async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1155 let ns = target.namespace().unwrap();
1156 let name = target.statefulset_name(&primary_group(target)?.name);
1157 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1158 .get(&name)
1159 .await
1160 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1161 Ok(sts
1162 .status
1163 .as_ref()
1164 .and_then(|s| s.ready_replicas)
1165 .unwrap_or(0)
1166 >= 1)
1167}
1168
1169fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1170 let rg = primary_group(target)?;
1171 Ok(format!(
1172 "{DATA_VOLUME}-{}-0",
1173 target.statefulset_name(&rg.name)
1174 ))
1175}
1176
1177async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1178 let ns = target.namespace().unwrap();
1179 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1180 let mut all_absent = true;
1181 for rg in &target.spec.replica_groups {
1182 let sts = target.statefulset_name(&rg.name);
1183 for ordinal in 0..rg.replicas {
1184 if rg.primary_node && ordinal == 0 {
1185 continue;
1186 }
1187 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1188 match api
1189 .get_opt(&name)
1190 .await
1191 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1192 {
1193 None => {}
1194 Some(pvc) => {
1195 all_absent = false;
1196 if pvc.metadata.deletion_timestamp.is_none() {
1197 match api.delete(&name, &DeleteParams::default()).await {
1198 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1199 Err(kube::Error::Api(status)) if status.code == 404 => {}
1200 Err(error) => {
1201 return Err(Error::kube_error(
1202 "delete",
1203 "PersistentVolumeClaim",
1204 &ns,
1205 &name,
1206 error,
1207 ));
1208 }
1209 }
1210 }
1211 }
1212 }
1213 }
1214 }
1215 Ok(all_absent)
1216}
1217
1218async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1219 let ns = target.namespace().unwrap();
1220 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1221 let label_selector = format!(
1222 "{}={},{}={}",
1223 CLUSTER_LABEL,
1224 target.name_any(),
1225 SECRET_TYPE_LABEL,
1226 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1227 );
1228 let secrets = api
1229 .list(&ListParams::default().labels(&label_selector))
1230 .await
1231 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1232 let mut all_absent = secrets.items.is_empty();
1233 for secret in secrets.items {
1234 let name = secret.name_any();
1235 if secret.metadata.deletion_timestamp.is_none() {
1236 match api.delete(&name, &DeleteParams::default()).await {
1237 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1238 Err(kube::Error::Api(status)) if status.code == 404 => {}
1239 Err(error) => {
1240 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1241 }
1242 }
1243 all_absent = false;
1244 }
1245 }
1246 Ok(all_absent)
1247}
1248
1249async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1250 let ns = target.namespace().unwrap();
1251 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1252 let name = format!("{}-admin-passwords", target.name_any());
1253 match api
1254 .get_opt(&name)
1255 .await
1256 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1257 {
1258 None => Ok(true),
1259 Some(secret) => {
1260 if secret.metadata.deletion_timestamp.is_some() {
1261 return Ok(false);
1262 }
1263 match api.delete(&name, &DeleteParams::default()).await {
1264 Ok(_) => {
1265 debug!(secret = %name, "deleting stale admin secret");
1266 Ok(false)
1267 }
1268 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1269 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1270 }
1271 }
1272 }
1273}
1274
1275fn restore_job_name(restore: &KanidmRestore) -> String {
1276 format!("{}-restore", restore.name_any())
1277}
1278fn verify_job_name(restore: &KanidmRestore) -> String {
1279 format!("{}-verify", restore.name_any())
1280}
1281
1282fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1283 let origin = target
1284 .spec
1285 .origin
1286 .clone()
1287 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1288 vec![
1289 EnvVar {
1290 name: "KANIDM_DB_PATH".to_string(),
1291 value: Some("/data/kanidm.db".to_string()),
1292 ..Default::default()
1293 },
1294 EnvVar {
1295 name: "KANIDM_DOMAIN".to_string(),
1296 value: Some(target.spec.domain.clone()),
1297 ..Default::default()
1298 },
1299 EnvVar {
1300 name: "KANIDM_ORIGIN".to_string(),
1301 value: Some(origin),
1302 ..Default::default()
1303 },
1304 EnvVar {
1305 name: "KANIDM_TLS_CHAIN".to_string(),
1306 value: Some(format!("{TLS_PATH}/tls.crt")),
1307 ..Default::default()
1308 },
1309 EnvVar {
1310 name: "KANIDM_TLS_KEY".to_string(),
1311 value: Some(format!("{TLS_PATH}/tls.key")),
1312 ..Default::default()
1313 },
1314 ]
1315}
1316
1317async fn ensure_database_job(
1318 restore: &KanidmRestore,
1319 target: &Kanidm,
1320 ctx: &RestoreContext,
1321 name: &str,
1322 verify: bool,
1323) -> Result<()> {
1324 let ns = restore.namespace().unwrap();
1325 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1326 if api
1327 .get_opt(name)
1328 .await
1329 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1330 .is_some()
1331 {
1332 return Ok(());
1333 }
1334 let mut command = vec![
1335 "kanidmd".to_string(),
1336 "database".to_string(),
1337 if verify { "backup" } else { "restore" }.to_string(),
1338 ];
1339 if verify {
1340 command.push("/verify/verification.json.gz".to_string());
1341 } else {
1342 if is_remote_source(restore) {
1343 command.push(format!("{STAGING_PATH}/source-payload.json.gz"));
1344 } else {
1345 let file_name = restore
1346 .spec
1347 .source
1348 .local
1349 .as_ref()
1350 .map(|l| l.file_name.clone())
1351 .unwrap_or_default();
1352 command.push(format!("{BACKUP_PATH}/{file_name}"));
1353 }
1354 }
1355 let job = Job {
1356 metadata: ObjectMeta {
1357 name: Some(name.to_string()),
1358 namespace: Some(ns.clone()),
1359 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1360 ..Default::default()
1361 },
1362 spec: Some(JobSpec {
1363 backoff_limit: Some(0),
1364 template: PodTemplateSpec {
1365 metadata: Some(ObjectMeta::default()),
1366 spec: Some(PodSpec {
1367 automount_service_account_token: Some(false),
1368 restart_policy: Some("Never".to_string()),
1369 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1370 seccomp_profile: Some(SeccompProfile {
1371 type_: "RuntimeDefault".to_string(),
1372 ..Default::default()
1373 }),
1374 ..Default::default()
1375 }),
1376 containers: vec![Container {
1377 name: if verify { "verify" } else { "restore" }.to_string(),
1378 image: Some(restore.spec.restore_image.clone()),
1379 command: Some(command),
1380 env: Some(build_kanidm_db_env_vars(target)),
1381 security_context: Some(kanidm_job_security_context()),
1382 resources: Some(job_resource_requirements()),
1383 volume_mounts: Some(vec![
1384 VolumeMount {
1385 name: DATA_VOLUME.to_string(),
1386 mount_path: DATA_PATH.to_string(),
1387 ..Default::default()
1388 },
1389 VolumeMount {
1390 name: TLS_VOLUME.to_string(),
1391 mount_path: TLS_PATH.to_string(),
1392 read_only: Some(true),
1393 ..Default::default()
1394 },
1395 VolumeMount {
1396 name: "verification-output".to_string(),
1397 mount_path: "/verify".to_string(),
1398 ..Default::default()
1399 },
1400 ]),
1401 ..Default::default()
1402 }],
1403 volumes: Some(vec![
1404 Volume {
1405 name: DATA_VOLUME.to_string(),
1406 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1407 claim_name: primary_pvc_name(target)?,
1408 read_only: Some(false),
1409 }),
1410 ..Default::default()
1411 },
1412 Volume {
1413 name: TLS_VOLUME.to_string(),
1414 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1415 secret_name: Some(target.effective_tls_secret_name()),
1416 ..Default::default()
1417 }),
1418 ..Default::default()
1419 },
1420 Volume {
1421 name: "verification-output".to_string(),
1422 empty_dir: Some(Default::default()),
1423 ..Default::default()
1424 },
1425 ]),
1426 ..Default::default()
1427 }),
1428 },
1429 ..Default::default()
1430 }),
1431 ..Default::default()
1432 };
1433 api.create(&PostParams::default(), &job)
1434 .await
1435 .map(|_| ())
1436 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1437}
1438
1439enum JobState {
1440 Running,
1441 Complete,
1442 Failed,
1443}
1444
1445async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1446 let ns = restore.namespace().unwrap();
1447 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1448 .get(name)
1449 .await
1450 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1451 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1452 return Ok(JobState::Failed);
1453 }
1454 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1455 return Ok(JobState::Complete);
1456 }
1457 Ok(JobState::Running)
1458}
1459
1460fn is_remote_source(restore: &KanidmRestore) -> bool {
1461 restore.spec.source.backup_ref.is_some()
1462}
1463
1464fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1465 let safety = restore.spec.safety_backup.as_ref();
1466 let skip = safety.is_some_and(|s| s.skip);
1467 !skip
1468}
1469
1470fn validate_source(restore: &KanidmRestore) -> Result<()> {
1471 let has_local = restore.spec.source.local.is_some();
1472 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1473 if has_local && has_backup_ref {
1474 return Err(Error::MissingData(
1475 "source.local and source.backupRef are mutually exclusive".to_string(),
1476 ));
1477 }
1478 if !has_local && !has_backup_ref {
1479 return Err(Error::MissingData(
1480 "source must specify either local or backupRef".to_string(),
1481 ));
1482 }
1483 if let Some(local) = &restore.spec.source.local {
1484 if !safe_basename(&local.file_name) {
1485 return Err(Error::MissingData(
1486 "restore source fileName must be a safe basename".to_string(),
1487 ));
1488 }
1489 }
1490 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1491 if backup_ref.name.is_empty() {
1492 return Err(Error::MissingData(
1493 "source.backupRef.name must not be empty".to_string(),
1494 ));
1495 }
1496 }
1497 Ok(())
1498}
1499
1500fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1501 let safety = restore.spec.safety_backup.as_ref();
1502 let skip = safety.is_some_and(|s| s.skip);
1503 if skip {
1504 let annotations = restore.metadata.annotations.as_ref();
1505 let reason = annotations
1506 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1507 .map(|s| s.trim())
1508 .unwrap_or("");
1509 let approver = annotations
1510 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1511 .map(|s| s.trim())
1512 .unwrap_or("");
1513 if reason.is_empty() {
1514 return Err(Error::MissingData(format!(
1515 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1516 )));
1517 }
1518 if approver.is_empty() {
1519 return Err(Error::MissingData(format!(
1520 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1521 )));
1522 }
1523 }
1524 if is_remote_source(restore) && !skip {
1525 let has_repo = safety
1526 .and_then(|s| s.repository_ref.as_ref())
1527 .is_some_and(|r| !r.name.is_empty());
1528 if !has_repo {
1529 return Err(Error::MissingData(
1530 "remote restore requires safetyBackup.repositoryRef".to_string(),
1531 ));
1532 }
1533 }
1534 Ok(())
1535}
1536
1537fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1538 target
1539 .spec
1540 .replica_groups
1541 .iter()
1542 .map(|rg| ReplicaCountEntry {
1543 group: rg.name.clone(),
1544 replicas: rg.replicas,
1545 })
1546 .collect()
1547}
1548
1549async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1550 if let Ok(target) = get_target(restore, ctx).await {
1551 let owns_maintenance =
1552 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1553 if owns_maintenance {
1554 scale_desired(&target, ctx).await?;
1555 clear_restoring(restore, &target, ctx).await?;
1556 }
1557 }
1558 Ok(())
1559}
1560
1561async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1562 ctx.metrics.break_glass_total.add(1, &[]);
1563 let reason = restore
1564 .metadata
1565 .annotations
1566 .as_ref()
1567 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1568 .cloned()
1569 .unwrap_or_default();
1570 let approver = restore
1571 .metadata
1572 .annotations
1573 .as_ref()
1574 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1575 .cloned()
1576 .unwrap_or_default();
1577 let mut status = restore.status.clone().unwrap_or_default();
1578 let break_glass_condition = restore_condition(
1579 &status.conditions,
1580 CONDITION_BREAK_GLASS,
1581 CONDITION_TRUE,
1582 "BreakGlassOverride",
1583 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1584 restore.metadata.generation,
1585 );
1586 status
1587 .conditions
1588 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1589 status.conditions.push(break_glass_condition);
1590 patch_status(restore, ctx, status).await?;
1591 if let Err(error) = ctx
1592 .recorder
1593 .publish(
1594 &Event {
1595 type_: EventType::Warning,
1596 reason: "SafetyBackupSkipped".to_string(),
1597 note: Some(format!(
1598 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1599 )),
1600 action: "Restore".to_string(),
1601 secondary: None,
1602 },
1603 &restore.object_ref(&()),
1604 )
1605 .await
1606 {
1607 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1608 }
1609 warn!(
1610 restore = %restore.name_any(),
1611 reason = %reason,
1612 approver = %approver,
1613 "break-glass safety backup override activated"
1614 );
1615 Ok(())
1616}
1617
1618async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1619 let status = restore.status.as_ref().cloned().unwrap_or_default();
1620 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1621 if let Ok(start_ts) = start.parse::<Timestamp>() {
1622 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1623 ctx.metrics
1624 .safety_backup_duration_seconds
1625 .record(elapsed, &[]);
1626 }
1627 }
1628}
1629
1630fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1631 format!("{}-safety-backup", restore.name_any())
1632}
1633
1634fn source_prep_job_name(restore: &KanidmRestore) -> String {
1635 format!("{}-source-prep", restore.name_any())
1636}
1637
1638fn source_check_job_name(restore: &KanidmRestore) -> String {
1639 format!("{}-source-check", restore.name_any())
1640}
1641
1642async fn ensure_source_check_job(
1643 restore: &KanidmRestore,
1644 target: &Kanidm,
1645 ctx: &RestoreContext,
1646 name: &str,
1647) -> Result<()> {
1648 let ns = restore.namespace().unwrap();
1649 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1650 if api
1651 .get_opt(name)
1652 .await
1653 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1654 .is_some()
1655 {
1656 return Ok(());
1657 }
1658 let file_name = restore
1659 .spec
1660 .source
1661 .local
1662 .as_ref()
1663 .map(|l| l.file_name.clone())
1664 .unwrap_or_default();
1665 let operation_doc = serde_json::json!({
1666 "apiVersion": "backup.kaniop.rs/v1alpha1",
1667 "kind": "OperationDocument",
1668 "operation": "check",
1669 "path": format!("{BACKUP_PATH}/{file_name}"),
1670 "resultPath": "/run/kaniop-result/result.json",
1671 })
1672 .to_string();
1673 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1674 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1675 let data_mover_image = data_mover_image();
1676 let job = Job {
1677 metadata: ObjectMeta {
1678 name: Some(name.to_string()),
1679 namespace: Some(ns.clone()),
1680 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1681 ..Default::default()
1682 },
1683 spec: Some(JobSpec {
1684 backoff_limit: Some(0),
1685 template: PodTemplateSpec {
1686 metadata: Some(ObjectMeta::default()),
1687 spec: Some(PodSpec {
1688 automount_service_account_token: Some(false),
1689 restart_policy: Some("Never".to_string()),
1690 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
1691 run_as_non_root: Some(true),
1692 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
1693 type_: "RuntimeDefault".to_string(),
1694 ..Default::default()
1695 }),
1696 ..Default::default()
1697 }),
1698 containers: vec![Container {
1699 name: "source-check".to_string(),
1700 image: Some(data_mover_image),
1701 command: Some(vec![
1702 "/bin/kaniop-data-mover".to_string(),
1703 "check".to_string(),
1704 "--operation-doc".to_string(),
1705 "/run/kaniop/operation.json".to_string(),
1706 ]),
1707 security_context: Some(hardened_security_context()),
1708 resources: Some(job_resource_requirements()),
1709 env: Some(vec![EnvVar {
1710 name: "RUST_LOG".to_string(),
1711 value: Some("info".to_string()),
1712 ..Default::default()
1713 }]),
1714 volume_mounts: Some(vec![
1715 VolumeMount {
1716 name: DATA_VOLUME.to_string(),
1717 mount_path: DATA_PATH.to_string(),
1718 read_only: Some(true),
1719 ..Default::default()
1720 },
1721 VolumeMount {
1722 name: "operation".to_string(),
1723 mount_path: "/run/kaniop".to_string(),
1724 read_only: Some(true),
1725 ..Default::default()
1726 },
1727 VolumeMount {
1728 name: "result".to_string(),
1729 mount_path: "/run/kaniop-result".to_string(),
1730 ..Default::default()
1731 },
1732 ]),
1733 ..Default::default()
1734 }],
1735 volumes: Some(vec![
1736 Volume {
1737 name: DATA_VOLUME.to_string(),
1738 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1739 claim_name: primary_pvc_name(target)?,
1740 read_only: Some(true),
1741 }),
1742 ..Default::default()
1743 },
1744 Volume {
1745 name: "operation".to_string(),
1746 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1747 name: operation_cm_name,
1748 ..Default::default()
1749 }),
1750 ..Default::default()
1751 },
1752 Volume {
1753 name: "result".to_string(),
1754 empty_dir: Some(Default::default()),
1755 ..Default::default()
1756 },
1757 ]),
1758 ..Default::default()
1759 }),
1760 },
1761 ..Default::default()
1762 }),
1763 ..Default::default()
1764 };
1765 api.create(&PostParams::default(), &job)
1766 .await
1767 .map(|_| ())
1768 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1769}
1770
1771const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1772
1773fn compute_safety_backup_id(restore: &KanidmRestore) -> String {
1774 let ns = restore.namespace().unwrap_or_default();
1775 let name = restore.name_any();
1776 let namespace_uuid =
1777 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1778 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1779 uuid::Uuid::new_v5(&name_uuid, name.as_bytes()).to_string()
1780}
1781
1782fn compute_safety_manifest_key(restore: &KanidmRestore) -> String {
1783 let ns = restore.namespace().unwrap_or_default();
1784 let kanidm_uid = &restore.spec.target_ref.uid;
1785 let backup_id = compute_safety_backup_id(restore);
1786 format!("v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json")
1787}
1788
1789struct VerifiedSafetyBackup {
1790 manifest_key: String,
1791 payload_sha256: String,
1792}
1793
1794async fn read_safety_backup_result(
1795 restore: &KanidmRestore,
1796 ctx: &RestoreContext,
1797 job_name: &str,
1798 expected_backup_id: &str,
1799) -> Result<VerifiedSafetyBackup> {
1800 let ns = restore.namespace().unwrap();
1801 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1802 let pod_list = pods
1803 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1804 .await
1805 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1806 let pod = pod_list.items.first().ok_or_else(|| {
1807 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1808 })?;
1809 let pod_name = pod.name_any();
1810 let container_status = pod
1811 .status
1812 .as_ref()
1813 .and_then(|s| s.container_statuses.as_ref())
1814 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1815 .ok_or_else(|| {
1816 Error::MissingData(format!(
1817 "safety-upload container status not found in pod {pod_name}"
1818 ))
1819 })?;
1820 let termination_message = container_status
1821 .state
1822 .as_ref()
1823 .and_then(|state| state.terminated.as_ref())
1824 .and_then(|t| t.message.as_ref())
1825 .ok_or_else(|| {
1826 Error::MissingData(format!(
1827 "termination message not found in pod {pod_name}; result transport failed"
1828 ))
1829 })?;
1830 if termination_message.is_empty() {
1831 return Err(Error::MissingData(format!(
1832 "termination message is empty in pod {pod_name}; result document was not written"
1833 )));
1834 }
1835 let result_doc = parse_result_document(termination_message).map_err(|e| {
1836 Error::ParseError(format!(
1837 "failed to parse result document from pod {pod_name}: {e}"
1838 ))
1839 })?;
1840 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1841 return Err(Error::ParseError(format!(
1842 "unsupported result document version: {}",
1843 result_doc.api_version
1844 )));
1845 }
1846 if result_doc.kind != "ResultDocument" {
1847 return Err(Error::ParseError(format!(
1848 "invalid result document kind: {}",
1849 result_doc.kind
1850 )));
1851 }
1852 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1853 return Err(Error::ParseError(format!(
1854 "result document operation mismatch: expected '{}', got '{}'",
1855 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1856 )));
1857 }
1858 if !result_doc.success {
1859 return Err(Error::ParseError(format!(
1860 "safety backup reported failure with exit code {:?}",
1861 result_doc.exit_code
1862 )));
1863 }
1864 if result_doc.exit_code != ExitCode::Success {
1865 return Err(Error::ParseError(format!(
1866 "safety backup exit code is not Success: {:?}",
1867 result_doc.exit_code
1868 )));
1869 }
1870 let doc_backup_id = result_doc
1871 .backup_id
1872 .as_deref()
1873 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1874 if doc_backup_id != expected_backup_id {
1875 return Err(Error::ParseError(format!(
1876 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1877 )));
1878 }
1879 let manifest_key = result_doc
1880 .manifest_key
1881 .clone()
1882 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1883 if manifest_key.is_empty() {
1884 return Err(Error::ParseError(
1885 "result document manifestKey is empty".to_string(),
1886 ));
1887 }
1888 let payload_sha256 = result_doc
1889 .payload_sha256
1890 .clone()
1891 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1892 if payload_sha256.is_empty() {
1893 return Err(Error::ParseError(
1894 "result document payloadSha256 is empty".to_string(),
1895 ));
1896 }
1897 info!(
1898 restore = %restore.name_any(),
1899 backup_id = %expected_backup_id,
1900 manifest_key = %manifest_key,
1901 "safety backup result verified"
1902 );
1903 Ok(VerifiedSafetyBackup {
1904 manifest_key,
1905 payload_sha256,
1906 })
1907}
1908
1909struct VerifiedSourcePrepResult {
1910 #[allow(dead_code)]
1911 manifest_key: String,
1912 #[allow(dead_code)]
1913 payload_sha256: String,
1914}
1915
1916async fn read_source_prep_result(
1917 restore: &KanidmRestore,
1918 ctx: &RestoreContext,
1919 job_name: &str,
1920 expected_backup_id: &str,
1921) -> Result<VerifiedSourcePrepResult> {
1922 let ns = restore.namespace().unwrap();
1923 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1924 let pod_list = pods
1925 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1926 .await
1927 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1928 let pod = pod_list.items.first().ok_or_else(|| {
1929 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1930 })?;
1931 let pod_name = pod.name_any();
1932 let container_status = pod
1933 .status
1934 .as_ref()
1935 .and_then(|s| s.container_statuses.as_ref())
1936 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1937 .ok_or_else(|| {
1938 Error::MissingData(format!(
1939 "source-prep container status not found in pod {pod_name}"
1940 ))
1941 })?;
1942 let termination_message = container_status
1943 .state
1944 .as_ref()
1945 .and_then(|state| state.terminated.as_ref())
1946 .and_then(|t| t.message.as_ref())
1947 .ok_or_else(|| {
1948 Error::MissingData(format!(
1949 "termination message not found in pod {pod_name}; result transport failed"
1950 ))
1951 })?;
1952 if termination_message.is_empty() {
1953 return Err(Error::MissingData(format!(
1954 "termination message is empty in pod {pod_name}; result document was not written"
1955 )));
1956 }
1957 let result_doc = parse_result_document(termination_message).map_err(|e| {
1958 Error::ParseError(format!(
1959 "failed to parse result document from pod {pod_name}: {e}"
1960 ))
1961 })?;
1962 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1963 return Err(Error::ParseError(format!(
1964 "unsupported result document version: {}",
1965 result_doc.api_version
1966 )));
1967 }
1968 if result_doc.kind != "ResultDocument" {
1969 return Err(Error::ParseError(format!(
1970 "invalid result document kind: {}",
1971 result_doc.kind
1972 )));
1973 }
1974 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1975 return Err(Error::ParseError(format!(
1976 "result document operation mismatch: expected '{}', got '{}'",
1977 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1978 )));
1979 }
1980 if !result_doc.success {
1981 return Err(Error::ParseError(format!(
1982 "source prep reported failure with exit code {:?}",
1983 result_doc.exit_code
1984 )));
1985 }
1986 if result_doc.exit_code != ExitCode::Success {
1987 return Err(Error::ParseError(format!(
1988 "source prep exit code is not Success: {:?}",
1989 result_doc.exit_code
1990 )));
1991 }
1992 let doc_backup_id = result_doc
1993 .backup_id
1994 .as_deref()
1995 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1996 if doc_backup_id != expected_backup_id {
1997 return Err(Error::ParseError(format!(
1998 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1999 )));
2000 }
2001 let manifest_key = result_doc
2002 .manifest_key
2003 .clone()
2004 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
2005 if manifest_key.is_empty() {
2006 return Err(Error::ParseError(
2007 "result document manifestKey is empty".to_string(),
2008 ));
2009 }
2010 let payload_sha256 = result_doc
2011 .payload_sha256
2012 .clone()
2013 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
2014 if payload_sha256.is_empty() {
2015 return Err(Error::ParseError(
2016 "result document payloadSha256 is empty".to_string(),
2017 ));
2018 }
2019 info!(
2020 restore = %restore.name_any(),
2021 backup_id = %expected_backup_id,
2022 manifest_key = %manifest_key,
2023 payload_sha256 = %payload_sha256,
2024 "source prep result verified"
2025 );
2026 Ok(VerifiedSourcePrepResult {
2027 manifest_key,
2028 payload_sha256,
2029 })
2030}
2031
2032fn hardened_security_context() -> SecurityContext {
2033 SecurityContext {
2034 allow_privilege_escalation: Some(false),
2035 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2036 drop: Some(vec!["ALL".to_string()]),
2037 ..Default::default()
2038 }),
2039 read_only_root_filesystem: Some(true),
2040 run_as_non_root: Some(true),
2041 seccomp_profile: Some(SeccompProfile {
2042 type_: "RuntimeDefault".to_string(),
2043 ..Default::default()
2044 }),
2045 ..Default::default()
2046 }
2047}
2048
2049fn kanidm_job_security_context() -> SecurityContext {
2050 SecurityContext {
2051 allow_privilege_escalation: Some(false),
2052 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2053 drop: Some(vec!["ALL".to_string()]),
2054 ..Default::default()
2055 }),
2056 seccomp_profile: Some(SeccompProfile {
2057 type_: "RuntimeDefault".to_string(),
2058 ..Default::default()
2059 }),
2060 ..Default::default()
2061 }
2062}
2063
2064fn job_resource_requirements() -> ResourceRequirements {
2065 ResourceRequirements {
2066 requests: Some(std::collections::BTreeMap::from([
2067 ("cpu".to_string(), Quantity("100m".to_string())),
2068 ("memory".to_string(), Quantity("128Mi".to_string())),
2069 ])),
2070 limits: Some(std::collections::BTreeMap::from([
2071 ("cpu".to_string(), Quantity("2".to_string())),
2072 ("memory".to_string(), Quantity("2Gi".to_string())),
2073 ])),
2074 ..Default::default()
2075 }
2076}
2077
2078async fn ensure_safety_backup_job(
2079 restore: &KanidmRestore,
2080 target: &Kanidm,
2081 ctx: &RestoreContext,
2082 name: &str,
2083 backup_id: &str,
2084 manifest_key: &str,
2085) -> Result<()> {
2086 let ns = restore.namespace().unwrap();
2087 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2088 if api
2089 .get_opt(name)
2090 .await
2091 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2092 .is_some()
2093 {
2094 return Ok(());
2095 }
2096 let safety = restore
2097 .spec
2098 .safety_backup
2099 .as_ref()
2100 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2101 let repo_ref = safety
2102 .repository_ref
2103 .as_ref()
2104 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2105 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2106 .get(&repo_ref.name)
2107 .await
2108 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2109 let _namespace_uid = &ns;
2110 let _kanidm_uid = &restore.spec.target_ref.uid;
2111 let endpoint = repo
2112 .spec
2113 .s3
2114 .endpoint
2115 .as_deref()
2116 .unwrap_or("https://s3.amazonaws.com");
2117 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2118 let operation_doc = build_safety_upload_operation_doc(
2119 restore,
2120 target,
2121 backup_id,
2122 manifest_key,
2123 &repo.spec.s3.bucket,
2124 &repo.spec.s3.prefix,
2125 endpoint,
2126 region,
2127 repo.spec.s3.force_path_style,
2128 repo.spec.s3.insecure,
2129 repo.spec
2130 .s3
2131 .ca_bundle_ref
2132 .as_deref()
2133 .map(|_| ca_bundle_path())
2134 .as_deref(),
2135 )?;
2136 let operation_cm_name = format!("{name}-op");
2137 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2138 let auth_method = &repo.spec.authentication.writer;
2139 let data_mover_image = data_mover_image();
2140 let init_command = vec![
2141 "kanidmd".to_string(),
2142 "database".to_string(),
2143 "backup".to_string(),
2144 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2145 ];
2146 let upload_script = format!(
2147 r#"set -eu
2148/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2149RESULT_FILE="/run/kaniop-result/result.json"
2150TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2151if [ ! -f "$RESULT_FILE" ]; then
2152 echo "ERROR: result document not found at $RESULT_FILE"
2153 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2154 exit 1
2155fi
2156if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2157 echo "ERROR: safety backup upload reported failure"
2158 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2159 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2160 exit 1
2161fi
2162dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2163echo "safety backup upload completed successfully"
2164"#
2165 );
2166 let job = Job {
2167 metadata: ObjectMeta {
2168 name: Some(name.to_string()),
2169 namespace: Some(ns.clone()),
2170 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2171 ..Default::default()
2172 },
2173 spec: Some(JobSpec {
2174 backoff_limit: Some(0),
2175 template: PodTemplateSpec {
2176 metadata: Some(ObjectMeta::default()),
2177 spec: Some(PodSpec {
2178 automount_service_account_token: Some(false),
2179 restart_policy: Some("Never".to_string()),
2180 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2181 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2182 type_: "RuntimeDefault".to_string(),
2183 ..Default::default()
2184 }),
2185 ..Default::default()
2186 }),
2187 init_containers: Some(vec![Container {
2188 name: "safety-backup".to_string(),
2189 image: Some(restore.spec.restore_image.clone()),
2190 command: Some(init_command),
2191 security_context: Some(SecurityContext {
2192 allow_privilege_escalation: Some(false),
2193 capabilities: Some(Capabilities {
2194 drop: Some(vec!["ALL".to_string()]),
2195 ..Default::default()
2196 }),
2197 read_only_root_filesystem: Some(true),
2198 seccomp_profile: Some(SeccompProfile {
2199 type_: "RuntimeDefault".to_string(),
2200 ..Default::default()
2201 }),
2202 ..Default::default()
2203 }),
2204 resources: Some(job_resource_requirements()),
2205 env: Some(build_kanidm_db_env_vars(target)),
2206 volume_mounts: Some(vec![
2207 VolumeMount {
2208 name: DATA_VOLUME.to_string(),
2209 mount_path: DATA_PATH.to_string(),
2210 read_only: Some(false),
2211 ..Default::default()
2212 },
2213 VolumeMount {
2214 name: TLS_VOLUME.to_string(),
2215 mount_path: TLS_PATH.to_string(),
2216 read_only: Some(true),
2217 ..Default::default()
2218 },
2219 VolumeMount {
2220 name: SHARED_VOLUME.to_string(),
2221 mount_path: SHARED_VOL_PATH.to_string(),
2222 ..Default::default()
2223 },
2224 ]),
2225 ..Default::default()
2226 }]),
2227 containers: vec![Container {
2228 name: "safety-upload".to_string(),
2229 image: Some(data_mover_image),
2230 command: Some(vec!["/bin/sh".to_string()]),
2231 args: Some(vec!["-c".to_string(), upload_script]),
2232 security_context: Some(hardened_security_context()),
2233 resources: Some(job_resource_requirements()),
2234 env: {
2235 let mut env_vars =
2236 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2237 env_vars.push(EnvVar {
2238 name: "RUST_LOG".to_string(),
2239 value: Some("info".to_string()),
2240 ..Default::default()
2241 });
2242 if repo.spec.s3.ca_bundle_ref.is_some() {
2243 env_vars.push(ca_bundle_env_var());
2244 }
2245 Some(env_vars)
2246 },
2247 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2248 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2249 volume_mounts: {
2250 let mut mounts = vec![
2251 VolumeMount {
2252 name: SHARED_VOLUME.to_string(),
2253 mount_path: SHARED_VOL_PATH.to_string(),
2254 read_only: Some(true),
2255 ..Default::default()
2256 },
2257 VolumeMount {
2258 name: "operation".to_string(),
2259 mount_path: "/run/kaniop".to_string(),
2260 read_only: Some(true),
2261 ..Default::default()
2262 },
2263 VolumeMount {
2264 name: "result".to_string(),
2265 mount_path: "/run/kaniop-result".to_string(),
2266 ..Default::default()
2267 },
2268 ];
2269 mounts.extend(build_auth_volume_mounts(auth_method));
2270 if repo.spec.s3.ca_bundle_ref.is_some() {
2271 mounts.push(build_ca_bundle_volume_mount());
2272 }
2273 Some(mounts)
2274 },
2275 ..Default::default()
2276 }],
2277 volumes: {
2278 let mut vols = vec![
2279 Volume {
2280 name: DATA_VOLUME.to_string(),
2281 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2282 claim_name: primary_pvc_name(target)?,
2283 read_only: Some(false),
2284 }),
2285 ..Default::default()
2286 },
2287 Volume {
2288 name: TLS_VOLUME.to_string(),
2289 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2290 secret_name: Some(target.effective_tls_secret_name()),
2291 ..Default::default()
2292 }),
2293 ..Default::default()
2294 },
2295 Volume {
2296 name: SHARED_VOLUME.to_string(),
2297 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2298 size_limit: Some(Quantity("10Gi".to_string())),
2299 ..Default::default()
2300 }),
2301 ..Default::default()
2302 },
2303 Volume {
2304 name: "operation".to_string(),
2305 config_map: Some(ConfigMapVolumeSource {
2306 name: operation_cm_name,
2307 ..Default::default()
2308 }),
2309 ..Default::default()
2310 },
2311 Volume {
2312 name: "result".to_string(),
2313 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2314 size_limit: Some(Quantity("16Mi".to_string())),
2315 ..Default::default()
2316 }),
2317 ..Default::default()
2318 },
2319 ];
2320 vols.extend(build_auth_volumes(auth_method));
2321 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2322 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2323 }
2324 Some(vols)
2325 },
2326 ..Default::default()
2327 }),
2328 },
2329 ..Default::default()
2330 }),
2331 ..Default::default()
2332 };
2333 api.create(&PostParams::default(), &job)
2334 .await
2335 .map(|_| ())
2336 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2337}
2338
2339async fn ensure_source_prep_job(
2340 restore: &KanidmRestore,
2341 target: &Kanidm,
2342 ctx: &RestoreContext,
2343 name: &str,
2344) -> Result<()> {
2345 let ns = restore.namespace().unwrap();
2346 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2347 if api
2348 .get_opt(name)
2349 .await
2350 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2351 .is_some()
2352 {
2353 return Ok(());
2354 }
2355 let backup_name = restore
2356 .spec
2357 .source
2358 .backup_ref
2359 .as_ref()
2360 .map(|b| b.name.clone())
2361 .unwrap_or_default();
2362 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2363 .get(&backup_name)
2364 .await
2365 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2366 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2367 .get(&backup.spec.repository_ref.name)
2368 .await
2369 .map_err(|e| {
2370 Error::kube_error(
2371 "get",
2372 "KanidmBackupRepository",
2373 &ns,
2374 &backup.spec.repository_ref.name,
2375 e,
2376 )
2377 })?;
2378 let endpoint = repo
2379 .spec
2380 .s3
2381 .endpoint
2382 .as_deref()
2383 .unwrap_or("https://s3.amazonaws.com");
2384 let region = repo.spec.s3.region.as_deref().unwrap_or("us-east-1");
2385 let operation_doc = build_download_operation_doc(
2386 restore,
2387 target,
2388 &backup.spec.manifest_key,
2389 &backup.spec.backup_id,
2390 &repo.spec.s3.bucket,
2391 &repo.spec.s3.prefix,
2392 endpoint,
2393 region,
2394 repo.spec.s3.force_path_style,
2395 repo.spec.s3.insecure,
2396 repo.spec
2397 .s3
2398 .ca_bundle_ref
2399 .as_deref()
2400 .map(|_| ca_bundle_path())
2401 .as_deref(),
2402 );
2403 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2404 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2405 let auth_method = &repo.spec.authentication.reader;
2406 let data_mover_image = data_mover_image();
2407 let download_script = format!(
2408 r#"set -eu
2409/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2410RESULT_FILE="/run/kaniop-result/result.json"
2411TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2412if [ ! -f "$RESULT_FILE" ]; then
2413 echo "ERROR: result document not found at $RESULT_FILE"
2414 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2415 exit 1
2416fi
2417if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2418 echo "ERROR: source preparation download reported failure"
2419 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2420 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2421 exit 1
2422fi
2423dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2424echo "source preparation download completed successfully"
2425"#
2426 );
2427 let job = Job {
2428 metadata: ObjectMeta {
2429 name: Some(name.to_string()),
2430 namespace: Some(ns.clone()),
2431 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2432 ..Default::default()
2433 },
2434 spec: Some(JobSpec {
2435 backoff_limit: Some(0),
2436 template: PodTemplateSpec {
2437 metadata: Some(ObjectMeta::default()),
2438 spec: Some(PodSpec {
2439 automount_service_account_token: Some(false),
2440 restart_policy: Some("Never".to_string()),
2441 security_context: Some(k8s_openapi::api::core::v1::PodSecurityContext {
2442 run_as_non_root: Some(true),
2443 seccomp_profile: Some(k8s_openapi::api::core::v1::SeccompProfile {
2444 type_: "RuntimeDefault".to_string(),
2445 ..Default::default()
2446 }),
2447 ..Default::default()
2448 }),
2449 containers: vec![Container {
2450 name: "source-prep".to_string(),
2451 image: Some(data_mover_image),
2452 command: Some(vec!["/bin/sh".to_string()]),
2453 args: Some(vec!["-c".to_string(), download_script]),
2454 security_context: Some(hardened_security_context()),
2455 resources: Some(job_resource_requirements()),
2456 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2457 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2458 env: {
2459 let mut env_vars = build_auth_env_vars(
2460 auth_method,
2461 &backup.spec.repository_ref.name,
2462 AuthRole::Reader,
2463 );
2464 env_vars.push(EnvVar {
2465 name: "RUST_LOG".to_string(),
2466 value: Some("info".to_string()),
2467 ..Default::default()
2468 });
2469 if repo.spec.s3.ca_bundle_ref.is_some() {
2470 env_vars.push(ca_bundle_env_var());
2471 }
2472 Some(env_vars)
2473 },
2474 volume_mounts: {
2475 let mut mounts = vec![
2476 VolumeMount {
2477 name: STAGING_VOLUME.to_string(),
2478 mount_path: STAGING_PATH.to_string(),
2479 ..Default::default()
2480 },
2481 VolumeMount {
2482 name: "operation".to_string(),
2483 mount_path: "/run/kaniop".to_string(),
2484 read_only: Some(true),
2485 ..Default::default()
2486 },
2487 VolumeMount {
2488 name: "result".to_string(),
2489 mount_path: "/run/kaniop-result".to_string(),
2490 ..Default::default()
2491 },
2492 ];
2493 mounts.extend(build_auth_volume_mounts(auth_method));
2494 if repo.spec.s3.ca_bundle_ref.is_some() {
2495 mounts.push(build_ca_bundle_volume_mount());
2496 }
2497 Some(mounts)
2498 },
2499 ..Default::default()
2500 }],
2501 volumes: {
2502 let mut vols = vec![
2503 Volume {
2504 name: STAGING_VOLUME.to_string(),
2505 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2506 size_limit: Some(Quantity("10Gi".to_string())),
2507 ..Default::default()
2508 }),
2509 ..Default::default()
2510 },
2511 Volume {
2512 name: "operation".to_string(),
2513 config_map: Some(ConfigMapVolumeSource {
2514 name: operation_cm_name,
2515 ..Default::default()
2516 }),
2517 ..Default::default()
2518 },
2519 Volume {
2520 name: "result".to_string(),
2521 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2522 size_limit: Some(Quantity("16Mi".to_string())),
2523 ..Default::default()
2524 }),
2525 ..Default::default()
2526 },
2527 ];
2528 vols.extend(build_auth_volumes(auth_method));
2529 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2530 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2531 }
2532 Some(vols)
2533 },
2534 ..Default::default()
2535 }),
2536 },
2537 ..Default::default()
2538 }),
2539 ..Default::default()
2540 };
2541 api.create(&PostParams::default(), &job)
2542 .await
2543 .map(|_| ())
2544 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2545}
2546fn data_mover_image() -> String {
2547 std::env::var("DATA_MOVER_IMAGE")
2548 .unwrap_or_else(|_| "ghcr.io/pando85/kaniop-data-mover:latest".to_string())
2549}
2550
2551#[allow(clippy::too_many_arguments)]
2552fn build_safety_upload_operation_doc(
2553 restore: &KanidmRestore,
2554 target: &Kanidm,
2555 backup_id: &str,
2556 manifest_key: &str,
2557 bucket: &str,
2558 prefix: &str,
2559 endpoint: &str,
2560 region: &str,
2561 force_path_style: bool,
2562 insecure: bool,
2563 ca_bundle_path: Option<&str>,
2564) -> Result<String> {
2565 let op = UploadOperation {
2566 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2567 bucket: bucket.to_string(),
2568 prefix: prefix.to_string(),
2569 endpoint: endpoint.to_string(),
2570 region: region.to_string(),
2571 force_path_style,
2572 insecure,
2573 ca_bundle_path: ca_bundle_path.map(str::to_string),
2574 backup_id: backup_id.to_string(),
2575 namespace_uid: restore.namespace().unwrap_or_default(),
2576 kanidm_uid: restore.spec.target_ref.uid.clone(),
2577 kanidm_name: restore.spec.target_ref.name.clone(),
2578 domain: target.spec.domain.clone(),
2579 kanidm_version: target
2580 .status
2581 .as_ref()
2582 .and_then(|s| s.version.as_ref())
2583 .map(|v| v.image_tag.clone())
2584 .unwrap_or_default(),
2585 image_digest: None,
2586 consistency: "kanidm-offline".to_string(),
2587 reason: "restore-safety".to_string(),
2588 encryption_mode: None,
2589 encryption_key_id: None,
2590 result_path: "/run/kaniop-result/result.json".to_string(),
2591 max_concurrent_parts: 4,
2592 max_retries: 3,
2593 };
2594 let _ = manifest_key;
2595 let doc = OperationDocument {
2596 api_version: OPERATION_DOC_VERSION.to_string(),
2597 kind: "OperationDocument".to_string(),
2598 spec: OperationSpec::Upload(op),
2599 };
2600 serde_json::to_string(&doc)
2601 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2602}
2603
2604async fn validate_backup_ref(
2605 restore: &KanidmRestore,
2606 target: &Kanidm,
2607 ctx: &RestoreContext,
2608) -> Result<()> {
2609 let backup_ref = restore
2610 .spec
2611 .source
2612 .backup_ref
2613 .as_ref()
2614 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2615 let ns = restore.namespace().unwrap();
2616 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2617 .get(&backup_ref.name)
2618 .await
2619 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2620 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2621 return Err(Error::MissingData(format!(
2622 "KanidmBackup '{}' is not Ready (phase: {:?})",
2623 backup_ref.name,
2624 backup.status.as_ref().map(|s| &s.phase)
2625 )));
2626 }
2627 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2628 return Err(Error::MissingData(format!(
2629 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2630 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2631 )));
2632 }
2633 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2634 return Err(Error::MissingData(format!(
2635 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2636 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2637 )));
2638 }
2639 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2640 .get(&backup.spec.repository_ref.name)
2641 .await
2642 .map_err(|e| {
2643 Error::kube_error(
2644 "get",
2645 "KanidmBackupRepository",
2646 &ns,
2647 &backup.spec.repository_ref.name,
2648 e,
2649 )
2650 })?;
2651 if !has_accepted_condition(
2652 &repo
2653 .status
2654 .as_ref()
2655 .map(|s| &s.conditions)
2656 .cloned()
2657 .unwrap_or_default(),
2658 ) {
2659 return Err(Error::MissingData(format!(
2660 "KanidmBackupRepository '{}' configuration has not been accepted",
2661 backup.spec.repository_ref.name
2662 )));
2663 }
2664 validate_backup_compatibility(restore, target, &backup)?;
2665 Ok(())
2666}
2667
2668fn has_accepted_condition(conditions: &[Condition]) -> bool {
2669 conditions
2670 .iter()
2671 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2672}
2673
2674fn validate_backup_compatibility(
2675 restore: &KanidmRestore,
2676 target: &Kanidm,
2677 backup: &KanidmBackup,
2678) -> Result<()> {
2679 let status = backup.status.as_ref();
2680 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2681 let target_version = target
2682 .status
2683 .as_ref()
2684 .and_then(|s| s.version.as_ref())
2685 .map(|v| v.image_tag.as_str());
2686 if let Some(tv) = target_version {
2687 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2688 return Err(Error::MissingData(format!(
2689 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2690 )));
2691 }
2692 }
2693 }
2694 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2695 if !backup_digest.is_empty()
2696 && restore.spec.restore_image.contains('@')
2697 && !restore.spec.restore_image.contains(backup_digest)
2698 {
2699 return Err(Error::MissingData(format!(
2700 "restore image digest does not match backup image digest '{backup_digest}'"
2701 )));
2702 }
2703 }
2704 Ok(())
2705}
2706
2707#[allow(clippy::too_many_arguments)]
2708fn build_download_operation_doc(
2709 restore: &KanidmRestore,
2710 target: &Kanidm,
2711 manifest_key: &str,
2712 expected_backup_id: &str,
2713 bucket: &str,
2714 prefix: &str,
2715 endpoint: &str,
2716 region: &str,
2717 force_path_style: bool,
2718 insecure: bool,
2719 ca_bundle_path: Option<&str>,
2720) -> String {
2721 serde_json::json!({
2722 "apiVersion": "backup.kaniop.rs/v1alpha1",
2723 "kind": "OperationDocument",
2724 "operation": "download",
2725 "manifestKey": manifest_key,
2726 "bucket": bucket,
2727 "prefix": prefix,
2728 "endpoint": endpoint,
2729 "region": region,
2730 "forcePathStyle": force_path_style,
2731 "insecure": insecure,
2732 "caBundlePath": ca_bundle_path,
2733 "expectedBackupId": expected_backup_id,
2734 "expectedKanidmUid": restore.spec.target_ref.uid,
2735 "expectedDomain": target.spec.domain,
2736 "outputPath": format!("{STAGING_PATH}/source-payload.json.gz"),
2737 "resultPath": "/run/kaniop-result/result.json",
2738 "maxRetries": 3,
2739 })
2740 .to_string()
2741}
2742
2743async fn ensure_operation_configmap(
2744 restore: &KanidmRestore,
2745 name: &str,
2746 data: &str,
2747 ns: &str,
2748 ctx: &RestoreContext,
2749) -> Result<()> {
2750 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2751 if api
2752 .get_opt(name)
2753 .await
2754 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2755 .is_some()
2756 {
2757 return Ok(());
2758 }
2759 let cm = ConfigMap {
2760 metadata: ObjectMeta {
2761 name: Some(name.to_string()),
2762 namespace: Some(ns.to_string()),
2763 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2764 ..Default::default()
2765 },
2766 data: Some(std::collections::BTreeMap::from([(
2767 "operation.json".to_string(),
2768 data.to_string(),
2769 )])),
2770 ..Default::default()
2771 };
2772 api.create(&PostParams::default(), &cm)
2773 .await
2774 .map(|_| ())
2775 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2776}
2777
2778#[cfg(test)]
2779mod tests {
2780 use super::{
2781 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2782 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2783 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2784 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2785 hardened_security_context, has_accepted_condition, is_remote_source,
2786 kanidm_job_security_context, mutable_image, requires_safety_backup, safe_basename,
2787 validate_safety_backup_config, validate_source,
2788 };
2789 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2790 use kube::api::ObjectMeta as ApiObjectMeta;
2791 use std::collections::BTreeMap;
2792
2793 fn make_restore(
2794 source: KanidmRestoreSource,
2795 safety: Option<SafetyBackupConfig>,
2796 ) -> KanidmRestore {
2797 KanidmRestore {
2798 metadata: ApiObjectMeta {
2799 name: Some("test-restore".to_string()),
2800 namespace: Some("default".to_string()),
2801 ..Default::default()
2802 },
2803 spec: KanidmRestoreSpec {
2804 target_ref: KanidmRestoreTargetRef {
2805 name: "corp-idm".to_string(),
2806 uid: "test-uid-123".to_string(),
2807 },
2808 source,
2809 restore_image: "kanidm/server@sha256:abc".to_string(),
2810 safety_backup: safety,
2811 },
2812 status: None,
2813 }
2814 }
2815
2816 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2817 super::super::crd::Kanidm {
2818 status: Some(super::super::crd::KanidmStatus {
2819 available_replicas: 1,
2820 replicas: 1,
2821 unavailable_replicas: 0,
2822 updated_replicas: 1,
2823 replica_statuses: vec![],
2824 replica_column: "1/1".to_string(),
2825 secret_name: None,
2826 version: Some(super::super::crd::KanidmVersionStatus {
2827 image_tag: image_tag.to_string(),
2828 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2829 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2830 }),
2831 domain_appearance_image: None,
2832 mail_sender: None,
2833 conditions: None,
2834 }),
2835 ..Default::default()
2836 }
2837 }
2838
2839 #[test]
2840 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2841 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2842 }
2843
2844 #[test]
2845 fn completed_restore_sets_ready_condition() {
2846 let mut status = KanidmRestoreStatus {
2847 phase: KanidmRestorePhase::Completed,
2848 ..Default::default()
2849 };
2850 super::update_restore_conditions(&mut status, Some(7));
2851 assert!(status.conditions.iter().any(|condition| {
2852 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2853 }));
2854 assert!(!status.conditions.iter().any(|condition| {
2855 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2856 }));
2857 }
2858
2859 #[test]
2860 fn failed_restore_sets_failed_condition() {
2861 let mut status = KanidmRestoreStatus {
2862 phase: KanidmRestorePhase::Failed,
2863 message: Some("verification failed".to_string()),
2864 ..Default::default()
2865 };
2866 super::update_restore_conditions(&mut status, Some(8));
2867 assert!(status.conditions.iter().any(|condition| {
2868 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2869 }));
2870 }
2871
2872 #[test]
2873 fn restore_filename_is_confined_to_backup_directory() {
2874 assert!(safe_basename("backup.json.gz"));
2875 assert!(!safe_basename("../backup.json.gz"));
2876 assert!(!safe_basename("a/b"));
2877 assert!(!safe_basename(""));
2878 }
2879
2880 #[test]
2881 fn restore_image_rejects_latest_and_unpinned_names() {
2882 assert!(mutable_image("kanidm/server:latest"));
2883 assert!(mutable_image("kanidm/server"));
2884 assert!(!mutable_image("kanidm/server:1.10.0"));
2885 assert!(!mutable_image("kanidm/server@sha256:abc"));
2886 }
2887
2888 #[test]
2889 fn source_validation_rejects_both_local_and_backup_ref() {
2890 let source = KanidmRestoreSource {
2891 local: Some(super::KanidmRestoreLocalSource {
2892 file_name: "backup.json".to_string(),
2893 }),
2894 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2895 name: "some-backup".to_string(),
2896 }),
2897 };
2898 let restore = make_restore(source, None);
2899 assert!(validate_source(&restore).is_err());
2900 }
2901
2902 #[test]
2903 fn source_validation_rejects_empty_source() {
2904 let source = KanidmRestoreSource {
2905 local: None,
2906 backup_ref: None,
2907 };
2908 let restore = make_restore(source, None);
2909 assert!(validate_source(&restore).is_err());
2910 }
2911
2912 #[test]
2913 fn source_validation_accepts_local_source() {
2914 let source = KanidmRestoreSource {
2915 local: Some(super::KanidmRestoreLocalSource {
2916 file_name: "backup.json".to_string(),
2917 }),
2918 backup_ref: None,
2919 };
2920 let restore = make_restore(source, None);
2921 assert!(validate_source(&restore).is_ok());
2922 }
2923
2924 #[test]
2925 fn source_validation_accepts_backup_ref_source() {
2926 let source = KanidmRestoreSource {
2927 local: None,
2928 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2929 name: "corp-idm-019c7c76".to_string(),
2930 }),
2931 };
2932 let restore = make_restore(source, None);
2933 assert!(validate_source(&restore).is_ok());
2934 }
2935
2936 #[test]
2937 fn source_validation_rejects_empty_backup_ref_name() {
2938 let source = KanidmRestoreSource {
2939 local: None,
2940 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2941 name: "".to_string(),
2942 }),
2943 };
2944 let restore = make_restore(source, None);
2945 assert!(validate_source(&restore).is_err());
2946 }
2947
2948 #[test]
2949 fn is_remote_source_true_for_backup_ref() {
2950 let source = KanidmRestoreSource {
2951 local: None,
2952 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2953 name: "backup-1".to_string(),
2954 }),
2955 };
2956 let restore = make_restore(source, None);
2957 assert!(is_remote_source(&restore));
2958 }
2959
2960 #[test]
2961 fn is_remote_source_false_for_local() {
2962 let source = KanidmRestoreSource {
2963 local: Some(super::KanidmRestoreLocalSource {
2964 file_name: "backup.json".to_string(),
2965 }),
2966 backup_ref: None,
2967 };
2968 let restore = make_restore(source, None);
2969 assert!(!is_remote_source(&restore));
2970 }
2971
2972 #[test]
2973 fn requires_safety_backup_default_true() {
2974 let source = KanidmRestoreSource {
2975 local: None,
2976 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2977 name: "backup-1".to_string(),
2978 }),
2979 };
2980 let restore = make_restore(source, None);
2981 assert!(requires_safety_backup(&restore));
2982 }
2983
2984 #[test]
2985 fn requires_safety_backup_false_when_skip() {
2986 let source = KanidmRestoreSource {
2987 local: None,
2988 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2989 name: "backup-1".to_string(),
2990 }),
2991 };
2992 let safety = SafetyBackupConfig {
2993 repository_ref: Some(SafetyBackupRepositoryRef {
2994 name: "offsite".to_string(),
2995 }),
2996 skip: true,
2997 };
2998 let restore = make_restore(source, Some(safety));
2999 assert!(!requires_safety_backup(&restore));
3000 }
3001
3002 #[test]
3003 fn break_glass_validation_rejects_skip_without_reason() {
3004 let source = KanidmRestoreSource {
3005 local: None,
3006 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3007 name: "backup-1".to_string(),
3008 }),
3009 };
3010 let safety = SafetyBackupConfig {
3011 repository_ref: Some(SafetyBackupRepositoryRef {
3012 name: "offsite".to_string(),
3013 }),
3014 skip: true,
3015 };
3016 let mut restore = make_restore(source, Some(safety));
3017 restore.metadata.annotations = Some(BTreeMap::from([(
3018 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3019 "commander@example.com".to_string(),
3020 )]));
3021 assert!(validate_safety_backup_config(&restore).is_err());
3022 }
3023
3024 #[test]
3025 fn break_glass_validation_rejects_skip_without_approver() {
3026 let source = KanidmRestoreSource {
3027 local: None,
3028 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3029 name: "backup-1".to_string(),
3030 }),
3031 };
3032 let safety = SafetyBackupConfig {
3033 repository_ref: Some(SafetyBackupRepositoryRef {
3034 name: "offsite".to_string(),
3035 }),
3036 skip: true,
3037 };
3038 let mut restore = make_restore(source, Some(safety));
3039 restore.metadata.annotations = Some(BTreeMap::from([(
3040 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3041 "PVC is unreadable".to_string(),
3042 )]));
3043 assert!(validate_safety_backup_config(&restore).is_err());
3044 }
3045
3046 #[test]
3047 fn break_glass_validation_accepts_valid_annotations() {
3048 let source = KanidmRestoreSource {
3049 local: None,
3050 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3051 name: "backup-1".to_string(),
3052 }),
3053 };
3054 let safety = SafetyBackupConfig {
3055 repository_ref: Some(SafetyBackupRepositoryRef {
3056 name: "offsite".to_string(),
3057 }),
3058 skip: true,
3059 };
3060 let mut restore = make_restore(source, Some(safety));
3061 restore.metadata.annotations = Some(BTreeMap::from([
3062 (
3063 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3064 "PVC is unreadable".to_string(),
3065 ),
3066 (
3067 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3068 "commander@example.com".to_string(),
3069 ),
3070 ]));
3071 assert!(validate_safety_backup_config(&restore).is_ok());
3072 }
3073
3074 #[test]
3075 fn remote_source_requires_repository_ref_when_not_skipping() {
3076 let source = KanidmRestoreSource {
3077 local: None,
3078 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3079 name: "backup-1".to_string(),
3080 }),
3081 };
3082 let safety = SafetyBackupConfig {
3083 repository_ref: None,
3084 skip: false,
3085 };
3086 let restore = make_restore(source, Some(safety));
3087 assert!(validate_safety_backup_config(&restore).is_err());
3088 }
3089
3090 #[test]
3091 fn remote_source_accepts_repository_ref() {
3092 let source = KanidmRestoreSource {
3093 local: None,
3094 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3095 name: "backup-1".to_string(),
3096 }),
3097 };
3098 let safety = SafetyBackupConfig {
3099 repository_ref: Some(SafetyBackupRepositoryRef {
3100 name: "offsite".to_string(),
3101 }),
3102 skip: false,
3103 };
3104 let restore = make_restore(source, Some(safety));
3105 assert!(validate_safety_backup_config(&restore).is_ok());
3106 }
3107
3108 #[test]
3109 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3110 let phases = vec![
3111 KanidmRestorePhase::Pending,
3112 KanidmRestorePhase::Validating,
3113 KanidmRestorePhase::Quiescing,
3114 KanidmRestorePhase::SafetyBackup,
3115 KanidmRestorePhase::PreparingSource,
3116 KanidmRestorePhase::RestoringPrimary,
3117 KanidmRestorePhase::Verifying,
3118 KanidmRestorePhase::RebuildingReplicas,
3119 KanidmRestorePhase::Resuming,
3120 KanidmRestorePhase::Completed,
3121 ];
3122 assert_eq!(phases.len(), 10);
3123 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3124 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3125 }
3126
3127 #[test]
3128 fn status_default_has_empty_phase_timestamps() {
3129 let status = KanidmRestoreStatus::default();
3130 assert!(status.phase_timestamps.is_empty());
3131 }
3132
3133 #[test]
3134 fn status_default_has_empty_original_replicas() {
3135 let status = KanidmRestoreStatus::default();
3136 assert!(status.original_replicas.is_empty());
3137 }
3138
3139 #[test]
3140 fn hardened_security_context_drops_all_capabilities() {
3141 let ctx = hardened_security_context();
3142 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3143 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3144 assert_eq!(ctx.run_as_non_root, Some(true));
3145 let caps = ctx.capabilities.unwrap();
3146 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3147 assert_eq!(
3148 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3149 Some("RuntimeDefault")
3150 );
3151 }
3152
3153 #[test]
3154 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3155 let ctx = kanidm_job_security_context();
3156 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3157 assert!(ctx.run_as_non_root.is_none());
3158 assert!(ctx.read_only_root_filesystem.is_none());
3159 let caps = ctx.capabilities.unwrap();
3160 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3161 assert_eq!(
3162 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3163 Some("RuntimeDefault")
3164 );
3165 }
3166
3167 #[test]
3168 fn safety_backup_job_name_is_deterministic() {
3169 let source = KanidmRestoreSource {
3170 local: Some(super::KanidmRestoreLocalSource {
3171 file_name: "backup.json".to_string(),
3172 }),
3173 backup_ref: None,
3174 };
3175 let restore = make_restore(source, None);
3176 let name1 = super::safety_backup_job_name(&restore);
3177 let name2 = super::safety_backup_job_name(&restore);
3178 assert_eq!(name1, name2);
3179 assert_eq!(name1, "test-restore-safety-backup");
3180 }
3181
3182 #[test]
3183 fn source_prep_job_name_is_deterministic() {
3184 let source = KanidmRestoreSource {
3185 local: None,
3186 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3187 name: "backup-1".to_string(),
3188 }),
3189 };
3190 let restore = make_restore(source, None);
3191 let name = super::source_prep_job_name(&restore);
3192 assert_eq!(name, "test-restore-source-prep");
3193 }
3194
3195 #[test]
3196 fn break_glass_condition_is_recorded_in_status() {
3197 let mut status = KanidmRestoreStatus::default();
3198 let condition = super::restore_condition(
3199 &status.conditions,
3200 CONDITION_BREAK_GLASS,
3201 CONDITION_TRUE,
3202 "BreakGlassOverride",
3203 "Safety backup skipped",
3204 Some(1),
3205 );
3206 status.conditions.push(condition);
3207 assert!(status.conditions.iter().any(|c| {
3208 c.type_ == CONDITION_BREAK_GLASS
3209 && c.status == CONDITION_TRUE
3210 && c.reason == "BreakGlassOverride"
3211 }));
3212 }
3213
3214 #[test]
3215 fn replica_count_entry_serialization() {
3216 let entry = ReplicaCountEntry {
3217 group: "primary".to_string(),
3218 replicas: 3,
3219 };
3220 let json = serde_json::to_string(&entry).unwrap();
3221 assert!(json.contains("\"group\":\"primary\""));
3222 assert!(json.contains("\"replicas\":3"));
3223 }
3224
3225 #[test]
3226 fn local_source_does_not_require_repository_ref() {
3227 let source = KanidmRestoreSource {
3228 local: Some(super::KanidmRestoreLocalSource {
3229 file_name: "backup.json".to_string(),
3230 }),
3231 backup_ref: None,
3232 };
3233 let safety = SafetyBackupConfig {
3234 repository_ref: None,
3235 skip: false,
3236 };
3237 let restore = make_restore(source, Some(safety));
3238 assert!(validate_safety_backup_config(&restore).is_ok());
3239 }
3240
3241 #[test]
3242 fn skip_without_break_glass_annotations_is_rejected() {
3243 let source = KanidmRestoreSource {
3244 local: Some(super::KanidmRestoreLocalSource {
3245 file_name: "backup.json".to_string(),
3246 }),
3247 backup_ref: None,
3248 };
3249 let safety = SafetyBackupConfig {
3250 repository_ref: None,
3251 skip: true,
3252 };
3253 let restore = make_restore(source, Some(safety));
3254 assert!(validate_safety_backup_config(&restore).is_err());
3255 }
3256
3257 #[test]
3258 fn has_accepted_condition_true() {
3259 let conditions = vec![Condition {
3260 type_: "Ready".to_string(),
3261 status: "True".to_string(),
3262 reason: "Accepted".to_string(),
3263 message: String::new(),
3264 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3265 k8s_openapi::jiff::Timestamp::now(),
3266 ),
3267 observed_generation: None,
3268 }];
3269 assert!(has_accepted_condition(&conditions));
3270 }
3271
3272 #[test]
3273 fn has_accepted_condition_false_when_not_accepted_reason() {
3274 let conditions = vec![Condition {
3275 type_: "Ready".to_string(),
3276 status: "True".to_string(),
3277 reason: "SomeOtherReason".to_string(),
3278 message: String::new(),
3279 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3280 k8s_openapi::jiff::Timestamp::now(),
3281 ),
3282 observed_generation: None,
3283 }];
3284 assert!(!has_accepted_condition(&conditions));
3285 }
3286
3287 #[test]
3288 fn has_accepted_condition_false_when_status_false() {
3289 let conditions = vec![Condition {
3290 type_: "Ready".to_string(),
3291 status: "False".to_string(),
3292 reason: "Accepted".to_string(),
3293 message: String::new(),
3294 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3295 k8s_openapi::jiff::Timestamp::now(),
3296 ),
3297 observed_generation: None,
3298 }];
3299 assert!(!has_accepted_condition(&conditions));
3300 }
3301
3302 #[test]
3303 fn has_accepted_condition_empty() {
3304 assert!(!has_accepted_condition(&[]));
3305 }
3306
3307 #[test]
3308 fn safety_backup_upload_operation_doc_is_valid() {
3309 let source = KanidmRestoreSource {
3310 local: None,
3311 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3312 name: "backup-1".to_string(),
3313 }),
3314 };
3315 let mut restore = make_restore(source, None);
3316 restore.metadata.namespace = Some("test-ns".to_string());
3317 let target = super::super::crd::Kanidm::default();
3318 let doc_str = super::build_safety_upload_operation_doc(
3319 &restore,
3320 &target,
3321 "test-backup-id",
3322 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3323 "my-bucket",
3324 "prod",
3325 "https://s3.example.com",
3326 "us-east-1",
3327 false,
3328 false,
3329 None,
3330 )
3331 .unwrap();
3332 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3333 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3334 assert_eq!(parsed["kind"], "OperationDocument");
3335 assert_eq!(parsed["operation"], "upload");
3336 assert_eq!(parsed["bucket"], "my-bucket");
3337 assert_eq!(parsed["prefix"], "prod");
3338 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3339 assert_eq!(parsed["region"], "us-east-1");
3340 assert_eq!(parsed["backupId"], "test-backup-id");
3341 assert_eq!(parsed["consistency"], "kanidm-offline");
3342 assert_eq!(parsed["reason"], "restore-safety");
3343 assert_eq!(
3344 parsed["payloadPath"],
3345 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3346 );
3347 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3348 }
3349
3350 #[test]
3351 fn download_operation_doc_uses_real_repository_data() {
3352 let source = KanidmRestoreSource {
3353 local: None,
3354 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3355 name: "backup-1".to_string(),
3356 }),
3357 };
3358 let restore = make_restore(source, None);
3359 let target = super::super::crd::Kanidm::default();
3360 let doc_str = super::build_download_operation_doc(
3361 &restore,
3362 &target,
3363 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3364 "019c7c76-f423-7a12-8f41-2bea7588a303",
3365 "real-bucket",
3366 "real-prefix",
3367 "https://real-endpoint.com",
3368 "eu-west-1",
3369 true,
3370 false,
3371 None,
3372 );
3373 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3374 assert_eq!(parsed["bucket"], "real-bucket");
3375 assert_eq!(parsed["prefix"], "real-prefix");
3376 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3377 assert_eq!(parsed["region"], "eu-west-1");
3378 assert_eq!(parsed["forcePathStyle"], true);
3379 assert_eq!(
3380 parsed["manifestKey"],
3381 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3382 );
3383 assert_eq!(
3384 parsed["expectedBackupId"],
3385 "019c7c76-f423-7a12-8f41-2bea7588a303"
3386 );
3387 }
3388
3389 #[test]
3390 fn safety_backup_ref_is_set_after_verification() {
3391 let status = KanidmRestoreStatus {
3392 phase: KanidmRestorePhase::SafetyBackup,
3393 safety_backup_job_name: Some("test-safety-backup".to_string()),
3394 safety_backup_ref: Some("safety-test-restore".to_string()),
3395 safety_backup_expected_backup_id: Some(
3396 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3397 ),
3398 safety_backup_manifest_key: Some(
3399 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3400 ),
3401 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3402 ..Default::default()
3403 };
3404 assert!(status.safety_backup_ref.is_some());
3405 assert!(
3406 status
3407 .safety_backup_ref
3408 .as_ref()
3409 .unwrap()
3410 .starts_with("safety-")
3411 );
3412 assert!(status.safety_backup_expected_backup_id.is_some());
3413 assert!(status.safety_backup_manifest_key.is_some());
3414 assert!(status.safety_backup_payload_sha256.is_some());
3415 }
3416
3417 #[test]
3418 fn database_mutation_not_started_without_safety_backup_ref() {
3419 let status = KanidmRestoreStatus {
3420 phase: KanidmRestorePhase::PreparingSource,
3421 safety_backup_ref: None,
3422 database_mutation_started: false,
3423 ..Default::default()
3424 };
3425 assert!(!status.database_mutation_started);
3426 assert!(status.safety_backup_ref.is_none());
3427 }
3428
3429 #[test]
3430 fn pre_boundary_failure_does_not_set_mutation_started() {
3431 let status = KanidmRestoreStatus {
3432 phase: KanidmRestorePhase::Failed,
3433 message: Some(
3434 "safety backup job failed; target restored to original state".to_string(),
3435 ),
3436 database_mutation_started: false,
3437 ..Default::default()
3438 };
3439 assert!(!status.database_mutation_started);
3440 }
3441
3442 #[test]
3443 fn shared_volume_constants_are_distinct() {
3444 assert_ne!(super::SHARED_VOLUME, super::STAGING_VOLUME);
3445 assert_ne!(super::SHARED_VOL_PATH, super::STAGING_PATH);
3446 }
3447
3448 #[test]
3449 fn validate_backup_compatibility_matching_versions_passes() {
3450 let source = KanidmRestoreSource {
3451 local: None,
3452 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3453 name: "backup-1".to_string(),
3454 }),
3455 };
3456 let restore = make_restore(source, None);
3457 let target = kanidm_with_version("1.10.4");
3458 let backup = kaniop_backup_core::crd::KanidmBackup {
3459 metadata: kube::api::ObjectMeta::default(),
3460 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3461 backup_id: "id".to_string(),
3462 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3463 name: "k".to_string(),
3464 uid: "uid".to_string(),
3465 },
3466 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3467 name: "repo".to_string(),
3468 },
3469 manifest_key: "key".to_string(),
3470 },
3471 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3472 kanidm_version: Some("1.10.4".to_string()),
3473 ..Default::default()
3474 }),
3475 };
3476 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3477 }
3478
3479 #[test]
3480 fn validate_backup_compatibility_version_mismatch_fails() {
3481 let source = KanidmRestoreSource {
3482 local: None,
3483 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3484 name: "backup-1".to_string(),
3485 }),
3486 };
3487 let restore = make_restore(source, None);
3488 let target = kanidm_with_version("1.11.0");
3489 let backup = kaniop_backup_core::crd::KanidmBackup {
3490 metadata: kube::api::ObjectMeta::default(),
3491 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3492 backup_id: "id".to_string(),
3493 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3494 name: "k".to_string(),
3495 uid: "uid".to_string(),
3496 },
3497 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3498 name: "repo".to_string(),
3499 },
3500 manifest_key: "key".to_string(),
3501 },
3502 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3503 kanidm_version: Some("1.10.4".to_string()),
3504 ..Default::default()
3505 }),
3506 };
3507 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3508 }
3509
3510 #[test]
3511 fn validate_backup_compatibility_no_backup_version_passes() {
3512 let source = KanidmRestoreSource {
3513 local: None,
3514 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3515 name: "backup-1".to_string(),
3516 }),
3517 };
3518 let restore = make_restore(source, None);
3519 let target = super::super::crd::Kanidm::default();
3520 let backup = kaniop_backup_core::crd::KanidmBackup {
3521 metadata: kube::api::ObjectMeta::default(),
3522 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3523 backup_id: "id".to_string(),
3524 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3525 name: "k".to_string(),
3526 uid: "uid".to_string(),
3527 },
3528 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3529 name: "repo".to_string(),
3530 },
3531 manifest_key: "key".to_string(),
3532 },
3533 status: None,
3534 };
3535 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3536 }
3537
3538 #[test]
3539 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3540 let source = KanidmRestoreSource {
3541 local: None,
3542 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3543 name: "backup-1".to_string(),
3544 }),
3545 };
3546 let mut restore = make_restore(source, None);
3547 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3548 let target = super::super::crd::Kanidm::default();
3549 let backup = kaniop_backup_core::crd::KanidmBackup {
3550 metadata: kube::api::ObjectMeta::default(),
3551 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3552 backup_id: "id".to_string(),
3553 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3554 name: "k".to_string(),
3555 uid: "uid".to_string(),
3556 },
3557 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3558 name: "repo".to_string(),
3559 },
3560 manifest_key: "key".to_string(),
3561 },
3562 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3563 image_digest: Some("sha256:originaldigest".to_string()),
3564 ..Default::default()
3565 }),
3566 };
3567 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3568 }
3569
3570 #[test]
3571 fn validate_backup_compatibility_image_digest_match_passes() {
3572 let source = KanidmRestoreSource {
3573 local: None,
3574 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3575 name: "backup-1".to_string(),
3576 }),
3577 };
3578 let mut restore = make_restore(source, None);
3579 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3580 let target = super::super::crd::Kanidm::default();
3581 let backup = kaniop_backup_core::crd::KanidmBackup {
3582 metadata: kube::api::ObjectMeta::default(),
3583 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3584 backup_id: "id".to_string(),
3585 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3586 name: "k".to_string(),
3587 uid: "uid".to_string(),
3588 },
3589 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3590 name: "repo".to_string(),
3591 },
3592 manifest_key: "key".to_string(),
3593 },
3594 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3595 image_digest: Some("sha256:abc123".to_string()),
3596 ..Default::default()
3597 }),
3598 };
3599 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3600 }
3601
3602 #[test]
3603 fn safety_backup_id_is_deterministic() {
3604 let source = KanidmRestoreSource {
3605 local: None,
3606 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3607 name: "backup-1".to_string(),
3608 }),
3609 };
3610 let restore = make_restore(source, None);
3611 let id1 = super::compute_safety_backup_id(&restore);
3612 let id2 = super::compute_safety_backup_id(&restore);
3613 assert_eq!(id1, id2);
3614 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3615 }
3616
3617 #[test]
3618 fn safety_backup_id_differs_for_different_restores() {
3619 let source1 = KanidmRestoreSource {
3620 local: None,
3621 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3622 name: "backup-1".to_string(),
3623 }),
3624 };
3625 let restore1 = make_restore(source1, None);
3626 let mut restore2 = make_restore(
3627 KanidmRestoreSource {
3628 local: None,
3629 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3630 name: "backup-2".to_string(),
3631 }),
3632 },
3633 None,
3634 );
3635 restore2.metadata.name = Some("different-restore".to_string());
3636 let id1 = super::compute_safety_backup_id(&restore1);
3637 let id2 = super::compute_safety_backup_id(&restore2);
3638 assert_ne!(id1, id2);
3639 }
3640
3641 #[test]
3642 fn safety_manifest_key_is_deterministic() {
3643 let source = KanidmRestoreSource {
3644 local: None,
3645 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3646 name: "backup-1".to_string(),
3647 }),
3648 };
3649 let restore = make_restore(source, None);
3650 let key1 = super::compute_safety_manifest_key(&restore);
3651 let key2 = super::compute_safety_manifest_key(&restore);
3652 assert_eq!(key1, key2);
3653 assert!(key1.contains("manifest.json"));
3654 assert!(key1.starts_with("v1/tenants/"));
3655 }
3656
3657 #[test]
3658 fn safety_backup_id_survives_restart() {
3659 let source = KanidmRestoreSource {
3660 local: None,
3661 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3662 name: "backup-1".to_string(),
3663 }),
3664 };
3665 let mut restore = make_restore(source, None);
3666 let original_id = super::compute_safety_backup_id(&restore);
3667 restore.status = Some(KanidmRestoreStatus {
3668 phase: KanidmRestorePhase::SafetyBackup,
3669 safety_backup_expected_backup_id: Some(original_id.clone()),
3670 ..Default::default()
3671 });
3672 let recomputed_id = super::compute_safety_backup_id(&restore);
3673 assert_eq!(original_id, recomputed_id);
3674 }
3675
3676 #[test]
3677 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3678 let source = KanidmRestoreSource {
3679 local: None,
3680 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3681 name: "backup-1".to_string(),
3682 }),
3683 };
3684 let restore = make_restore(source, None);
3685 assert!(super::requires_safety_backup(&restore));
3686 let status_without_ref = KanidmRestoreStatus {
3687 phase: KanidmRestorePhase::RestoringPrimary,
3688 safety_backup_ref: None,
3689 database_mutation_started: false,
3690 ..Default::default()
3691 };
3692 assert!(status_without_ref.safety_backup_ref.is_none());
3693 assert!(!status_without_ref.database_mutation_started);
3694 }
3695
3696 #[test]
3697 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3698 let status_with_ref = KanidmRestoreStatus {
3699 phase: KanidmRestorePhase::RestoringPrimary,
3700 safety_backup_ref: Some("safety-test-restore".to_string()),
3701 safety_backup_manifest_key: Some(
3702 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3703 ),
3704 safety_backup_payload_sha256: Some("abc123".to_string()),
3705 database_mutation_started: false,
3706 ..Default::default()
3707 };
3708 assert!(status_with_ref.safety_backup_ref.is_some());
3709 assert!(!status_with_ref.database_mutation_started);
3710 }
3711
3712 #[test]
3713 fn result_document_verification_rejects_missing_backup_id() {
3714 let result_json = r#"{
3715 "apiVersion": "backup.kaniop.rs/v1alpha1",
3716 "kind": "ResultDocument",
3717 "operation": "upload",
3718 "success": true,
3719 "exitCode": "success",
3720 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3721 "payloadSha256": "abc123"
3722 }"#;
3723 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3724 assert!(result.backup_id.is_none());
3725 }
3726
3727 #[test]
3728 fn result_document_verification_rejects_missing_manifest_key() {
3729 let result_json = r#"{
3730 "apiVersion": "backup.kaniop.rs/v1alpha1",
3731 "kind": "ResultDocument",
3732 "operation": "upload",
3733 "success": true,
3734 "exitCode": "success",
3735 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3736 "payloadSha256": "abc123"
3737 }"#;
3738 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3739 assert!(result.manifest_key.is_none());
3740 }
3741
3742 #[test]
3743 fn result_document_verification_rejects_missing_payload_sha256() {
3744 let result_json = r#"{
3745 "apiVersion": "backup.kaniop.rs/v1alpha1",
3746 "kind": "ResultDocument",
3747 "operation": "upload",
3748 "success": true,
3749 "exitCode": "success",
3750 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3751 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3752 }"#;
3753 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3754 assert!(result.payload_sha256.is_none());
3755 }
3756
3757 #[test]
3758 fn result_document_verification_rejects_failure_exit_code() {
3759 let result_json = r#"{
3760 "apiVersion": "backup.kaniop.rs/v1alpha1",
3761 "kind": "ResultDocument",
3762 "operation": "upload",
3763 "success": false,
3764 "exitCode": "integrity",
3765 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3766 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3767 "payloadSha256": "abc123"
3768 }"#;
3769 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3770 assert!(!result.success);
3771 assert_eq!(
3772 result.exit_code,
3773 kaniop_backup_core::result::ExitCode::Integrity
3774 );
3775 }
3776
3777 #[test]
3778 fn result_document_verification_rejects_wrong_operation() {
3779 let result_json = r#"{
3780 "apiVersion": "backup.kaniop.rs/v1alpha1",
3781 "kind": "ResultDocument",
3782 "operation": "download",
3783 "success": true,
3784 "exitCode": "success",
3785 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3786 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3787 "payloadSha256": "abc123"
3788 }"#;
3789 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3790 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3791 }
3792
3793 #[test]
3794 fn result_document_verification_accepts_valid_document() {
3795 let result_json = r#"{
3796 "apiVersion": "backup.kaniop.rs/v1alpha1",
3797 "kind": "ResultDocument",
3798 "operation": "upload",
3799 "success": true,
3800 "exitCode": "success",
3801 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3802 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3803 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3804 }"#;
3805 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3806 assert!(result.success);
3807 assert_eq!(
3808 result.exit_code,
3809 kaniop_backup_core::result::ExitCode::Success
3810 );
3811 assert_eq!(result.operation, "upload");
3812 assert!(result.backup_id.is_some());
3813 assert!(result.manifest_key.is_some());
3814 assert!(result.payload_sha256.is_some());
3815 }
3816
3817 #[test]
3818 fn termination_message_path_constant_is_valid() {
3819 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3820 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3821 }
3822
3823 #[test]
3824 fn status_tracks_safety_backup_evidence() {
3825 let status = KanidmRestoreStatus {
3826 phase: KanidmRestorePhase::PreparingSource,
3827 safety_backup_ref: Some("safety-test-restore".to_string()),
3828 safety_backup_expected_backup_id: Some(
3829 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3830 ),
3831 safety_backup_manifest_key: Some(
3832 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3833 ),
3834 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3835 ..Default::default()
3836 };
3837 assert!(status.safety_backup_ref.is_some());
3838 assert!(status.safety_backup_expected_backup_id.is_some());
3839 assert!(status.safety_backup_manifest_key.is_some());
3840 assert!(status.safety_backup_payload_sha256.is_some());
3841 }
3842
3843 #[test]
3844 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3845 let invalid_json = "not valid json";
3846 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3847 assert!(result.is_err());
3848 }
3849
3850 #[test]
3851 fn fail_closed_semantics_preserved_on_oversized_result() {
3852 let oversized_json = format!(
3853 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3854 kaniop_backup_core::result::RESULT_DOC_VERSION,
3855 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3856 );
3857 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3858 assert!(result.is_err());
3859 }
3860
3861 #[test]
3862 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3863 let source = KanidmRestoreSource {
3864 local: None,
3865 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3866 name: "backup-1".to_string(),
3867 }),
3868 };
3869 let mut restore = make_restore(source, None);
3870 restore.metadata.namespace = Some("test-ns".to_string());
3871 let target = super::super::crd::Kanidm::default();
3872 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3873 let doc_str = super::build_safety_upload_operation_doc(
3874 &restore,
3875 &target,
3876 "test-backup-id",
3877 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3878 "my-bucket",
3879 "prod",
3880 "https://s3.example.com",
3881 "us-east-1",
3882 false,
3883 false,
3884 Some(&ca_path),
3885 )
3886 .unwrap();
3887 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3888 assert_eq!(parsed["caBundlePath"], ca_path);
3889 }
3890
3891 #[test]
3892 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3893 let source = KanidmRestoreSource {
3894 local: None,
3895 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3896 name: "backup-1".to_string(),
3897 }),
3898 };
3899 let mut restore = make_restore(source, None);
3900 restore.metadata.namespace = Some("test-ns".to_string());
3901 let target = super::super::crd::Kanidm::default();
3902 let doc_str = super::build_safety_upload_operation_doc(
3903 &restore,
3904 &target,
3905 "test-backup-id",
3906 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3907 "my-bucket",
3908 "prod",
3909 "https://s3.example.com",
3910 "us-east-1",
3911 false,
3912 false,
3913 None,
3914 )
3915 .unwrap();
3916 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3917 assert!(parsed["caBundlePath"].is_null());
3918 }
3919
3920 #[test]
3921 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3922 let source = KanidmRestoreSource {
3923 local: None,
3924 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3925 name: "backup-1".to_string(),
3926 }),
3927 };
3928 let restore = make_restore(source, None);
3929 let target = super::super::crd::Kanidm::default();
3930 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3931 let doc_str = super::build_download_operation_doc(
3932 &restore,
3933 &target,
3934 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3935 "019c7c76-f423-7a12-8f41-2bea7588a303",
3936 "real-bucket",
3937 "real-prefix",
3938 "https://real-endpoint.com",
3939 "eu-west-1",
3940 true,
3941 false,
3942 Some(&ca_path),
3943 );
3944 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3945 assert_eq!(parsed["caBundlePath"], ca_path);
3946 }
3947
3948 #[test]
3949 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
3950 let source = KanidmRestoreSource {
3951 local: None,
3952 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3953 name: "backup-1".to_string(),
3954 }),
3955 };
3956 let restore = make_restore(source, None);
3957 let target = super::super::crd::Kanidm::default();
3958 let doc_str = super::build_download_operation_doc(
3959 &restore,
3960 &target,
3961 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3962 "019c7c76-f423-7a12-8f41-2bea7588a303",
3963 "real-bucket",
3964 "real-prefix",
3965 "https://real-endpoint.com",
3966 "eu-west-1",
3967 true,
3968 false,
3969 None,
3970 );
3971 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3972 assert!(parsed["caBundlePath"].is_null());
3973 }
3974
3975 #[test]
3976 fn source_prep_result_operation_is_download() {
3977 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
3978 assert_ne!(
3979 super::SOURCE_PREP_RESULT_OPERATION,
3980 super::SAFETY_BACKUP_RESULT_OPERATION
3981 );
3982 }
3983
3984 #[test]
3985 fn download_result_document_verification_accepts_valid_document() {
3986 let result_json = r#"{
3987 "apiVersion": "backup.kaniop.rs/v1alpha1",
3988 "kind": "ResultDocument",
3989 "operation": "download",
3990 "success": true,
3991 "exitCode": "success",
3992 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3993 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3994 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3995 }"#;
3996 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3997 assert!(result.success);
3998 assert_eq!(
3999 result.exit_code,
4000 kaniop_backup_core::result::ExitCode::Success
4001 );
4002 assert_eq!(result.operation, "download");
4003 assert_eq!(
4004 result.backup_id.as_deref(),
4005 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4006 );
4007 assert!(result.manifest_key.is_some());
4008 assert!(result.payload_sha256.is_some());
4009 }
4010
4011 #[test]
4012 fn download_result_document_rejects_wrong_operation() {
4013 let result_json = r#"{
4014 "apiVersion": "backup.kaniop.rs/v1alpha1",
4015 "kind": "ResultDocument",
4016 "operation": "upload",
4017 "success": true,
4018 "exitCode": "success",
4019 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4020 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4021 "payloadSha256": "abc123"
4022 }"#;
4023 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4024 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4025 }
4026
4027 #[test]
4028 fn download_result_document_rejects_failure_success_field() {
4029 let result_json = r#"{
4030 "apiVersion": "backup.kaniop.rs/v1alpha1",
4031 "kind": "ResultDocument",
4032 "operation": "download",
4033 "success": false,
4034 "exitCode": "retryable",
4035 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4036 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4037 "payloadSha256": "abc123"
4038 }"#;
4039 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4040 assert!(!result.success);
4041 assert_ne!(
4042 result.exit_code,
4043 kaniop_backup_core::result::ExitCode::Success
4044 );
4045 }
4046
4047 #[test]
4048 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4049 let status = KanidmRestoreStatus {
4050 phase: KanidmRestorePhase::PreparingSource,
4051 database_mutation_started: false,
4052 ..Default::default()
4053 };
4054 assert!(!status.database_mutation_started);
4055 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4056 }
4057}