1use super::crd::Kanidm;
2use super::reconcile::CLUSTER_LABEL;
3use super::reconcile::secret::{SECRET_TYPE_LABEL, SecretType};
4use super::reconcile::statefulset::StatefulSetExt;
5
6use kaniop_backup_core::auth::{
7 AuthRole, build_auth_env_vars, build_auth_volume_mounts, build_auth_volumes,
8 build_ca_bundle_volume, build_ca_bundle_volume_mount, build_encryption_env_vars,
9 ca_bundle_env_var, ca_bundle_path,
10};
11use kaniop_backup_core::crd::{KanidmBackup, KanidmBackupPhase, KanidmBackupRepository};
12use kaniop_backup_core::image::data_mover_image;
13use kaniop_backup_core::operation::{
14 OPERATION_DOC_VERSION, OperationDocument, OperationSpec, UploadOperation,
15};
16use kaniop_backup_core::result::{ExitCode, parse_result_document};
17use kaniop_k8s_util::error::{Error, Result};
18
19use std::collections::BTreeSet;
20use std::sync::Arc;
21use std::time::Duration;
22
23use futures::StreamExt;
24use k8s_openapi::api::apps::v1::StatefulSet;
25use k8s_openapi::api::batch::v1::{Job, JobSpec};
26use k8s_openapi::api::core::v1::{
27 Capabilities, ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
28 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, ResourceRequirements,
29 SeccompProfile, Secret, SecurityContext, Volume, VolumeMount,
30};
31use k8s_openapi::api::storage::v1::VolumeAttachment;
32use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
33use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
34use k8s_openapi::jiff::Timestamp;
35use kube::api::{DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams};
36use kube::runtime::controller::{Action, Controller};
37use kube::runtime::events::{Event, EventType, Recorder, Reporter};
38use kube::runtime::finalizer::{Event as Finalizer, finalizer};
39use kube::runtime::watcher;
40use kube::{Api, Client, CustomResource, Resource, ResourceExt};
41use opentelemetry::metrics::{Counter, Histogram};
42use opentelemetry::{KeyValue, global};
43#[cfg(feature = "schemars")]
44use schemars::JsonSchema;
45use serde::{Deserialize, Serialize};
46use serde_json::json;
47use tracing::{debug, error, info, warn};
48
49pub const CONTROLLER_ID: &str = "kanidm-restore";
50pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
51const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
52const DATA_VOLUME: &str = "kanidm-data";
53const SHARED_VOLUME: &str = "safety-backup-shared";
54const DATA_PATH: &str = "/data";
55const TLS_VOLUME: &str = "kanidm-certs";
56const TLS_PATH: &str = "/etc/kanidm/tls";
57const BACKUP_PATH: &str = "/data";
58const SHARED_VOL_PATH: &str = "/shared";
59const REQUEUE: Duration = Duration::from_secs(2);
60const CONDITION_TRUE: &str = "True";
61const CONDITION_FALSE: &str = "False";
62const CONDITION_PROGRESSING: &str = "Progressing";
63const CONDITION_READY: &str = "Ready";
64const CONDITION_FAILED: &str = "Failed";
65
66pub const BREAK_GLASS_REASON_ANNOTATION: &str = "backup.kaniop.rs/break-glass-reason";
67pub const BREAK_GLASS_APPROVED_BY_ANNOTATION: &str = "backup.kaniop.rs/break-glass-approved-by";
68const CONDITION_BREAK_GLASS: &str = "BreakGlassOverride";
69const TERMINATION_MESSAGE_PATH: &str = "/run/kaniop-result/termination-message";
70const SAFETY_BACKUP_RESULT_OPERATION: &str = "upload";
71const SOURCE_PREP_RESULT_OPERATION: &str = "download";
72
73#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
74#[cfg_attr(feature = "schemars", derive(JsonSchema))]
75#[cfg_attr(
76 feature = "schemars",
77 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
78)]
79#[kube(
80 category = "kaniop",
81 group = "kaniop.rs",
82 version = "v1beta1",
83 kind = "KanidmRestore",
84 plural = "kanidmrestores",
85 singular = "kanidmrestore",
86 shortname = "idmrestore",
87 namespaced,
88 status = "KanidmRestoreStatus",
89 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
90 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
91 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
92)]
93#[serde(rename_all = "camelCase")]
94pub struct KanidmRestoreSpec {
95 pub target_ref: KanidmRestoreTargetRef,
96 pub source: KanidmRestoreSource,
97 pub restore_image: String,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub safety_backup: Option<SafetyBackupConfig>,
100}
101
102#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
103#[cfg_attr(feature = "schemars", derive(JsonSchema))]
104#[serde(rename_all = "camelCase")]
105pub struct KanidmRestoreTargetRef {
106 pub name: String,
107 pub uid: String,
108}
109
110#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
111#[cfg_attr(feature = "schemars", derive(JsonSchema))]
112#[cfg_attr(
113 feature = "schemars",
114 schemars(extend("x-kubernetes-validations" = [
115 {
116 "message": "exactly one of local or backupRef must be set",
117 "rule": "(has(self.local) ? 1 : 0) + (has(self.backupRef) ? 1 : 0) == 1"
118 }
119 ]))
120)]
121#[serde(rename_all = "camelCase")]
122pub struct KanidmRestoreSource {
123 #[serde(skip_serializing_if = "Option::is_none")]
125 pub local: Option<KanidmRestoreLocalSource>,
126 #[serde(skip_serializing_if = "Option::is_none")]
128 pub backup_ref: Option<KanidmRestoreBackupRefSource>,
129}
130
131#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
132#[cfg_attr(feature = "schemars", derive(JsonSchema))]
133#[serde(rename_all = "camelCase")]
134pub struct KanidmRestoreLocalSource {
135 pub file_name: String,
137}
138
139#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
140#[cfg_attr(feature = "schemars", derive(JsonSchema))]
141#[serde(rename_all = "camelCase")]
142pub struct KanidmRestoreBackupRefSource {
143 pub name: String,
145}
146
147#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "schemars", derive(JsonSchema))]
149#[serde(rename_all = "camelCase")]
150pub struct SafetyBackupConfig {
151 #[serde(skip_serializing_if = "Option::is_none")]
153 pub repository_ref: Option<SafetyBackupRepositoryRef>,
154 #[serde(default)]
156 pub skip: bool,
157}
158
159#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
160#[cfg_attr(feature = "schemars", derive(JsonSchema))]
161#[serde(rename_all = "camelCase")]
162pub struct SafetyBackupRepositoryRef {
163 pub name: String,
164}
165
166#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
167#[cfg_attr(feature = "schemars", derive(JsonSchema))]
168pub enum KanidmRestorePhase {
169 #[default]
170 Pending,
171 Validating,
172 Quiescing,
173 SafetyBackup,
174 PreparingSource,
175 RestoringPrimary,
176 Verifying,
177 RebuildingReplicas,
178 Resuming,
179 Completed,
180 Failed,
181}
182
183#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
184#[cfg_attr(feature = "schemars", derive(JsonSchema))]
185#[serde(rename_all = "camelCase")]
186pub struct KanidmRestoreStatus {
187 #[serde(skip_serializing_if = "Option::is_none")]
188 pub observed_generation: Option<i64>,
189 #[serde(skip_serializing_if = "Option::is_none")]
190 pub observed_target_uid: Option<String>,
191 #[serde(default)]
192 pub phase: KanidmRestorePhase,
193 #[serde(skip_serializing_if = "Option::is_none")]
194 pub restore_job_name: Option<String>,
195 #[serde(skip_serializing_if = "Option::is_none")]
196 pub verify_job_name: Option<String>,
197 #[serde(skip_serializing_if = "Option::is_none")]
198 pub safety_backup_job_name: Option<String>,
199 #[serde(skip_serializing_if = "Option::is_none")]
200 pub source_prep_job_name: Option<String>,
201 #[serde(skip_serializing_if = "Option::is_none")]
202 pub safety_backup_ref: Option<String>,
203 #[serde(skip_serializing_if = "Option::is_none")]
204 pub safety_backup_expected_backup_id: Option<String>,
205 #[serde(skip_serializing_if = "Option::is_none")]
206 pub safety_backup_manifest_key: Option<String>,
207 #[serde(skip_serializing_if = "Option::is_none")]
208 pub safety_backup_payload_sha256: Option<String>,
209 #[serde(default)]
210 pub replicas_cleared: bool,
211 #[serde(default)]
212 pub certificates_cleared: bool,
213 #[serde(default)]
214 pub database_mutation_started: bool,
215 #[serde(default, skip_serializing_if = "Vec::is_empty")]
216 pub original_replicas: Vec<ReplicaCountEntry>,
217 #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")]
218 pub phase_timestamps: std::collections::BTreeMap<String, String>,
219 #[serde(default, skip_serializing_if = "Vec::is_empty")]
220 #[cfg_attr(
221 feature = "schemars",
222 schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))
223 )]
224 pub conditions: Vec<Condition>,
225 #[serde(skip_serializing_if = "Option::is_none")]
226 pub message: Option<String>,
227}
228
229#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
230#[cfg_attr(feature = "schemars", derive(JsonSchema))]
231#[serde(rename_all = "camelCase")]
232pub struct ReplicaCountEntry {
233 pub group: String,
234 pub replicas: i32,
235}
236
237#[derive(Clone)]
238struct RestoreMetrics {
239 attempts: Counter<u64>,
240 outcomes: Counter<u64>,
241 duration_seconds: Histogram<f64>,
242 break_glass_total: Counter<u64>,
243 safety_backup_duration_seconds: Histogram<f64>,
244}
245
246impl RestoreMetrics {
247 fn new() -> Self {
248 let meter = global::meter("kaniop");
249 Self {
250 attempts: meter
251 .u64_counter("kaniop_restore_attempts_total")
252 .with_description("Number of Kanidm restore attempts started")
253 .build(),
254 outcomes: meter
255 .u64_counter("kaniop_restore_outcomes_total")
256 .with_description("Number of terminal Kanidm restore outcomes")
257 .build(),
258 duration_seconds: meter
259 .f64_histogram("kaniop_restore_duration_seconds")
260 .with_description("Kanidm restore duration from object creation to terminal phase")
261 .with_unit("s")
262 .build(),
263 break_glass_total: meter
264 .u64_counter("kaniop_restore_break_glass_total")
265 .with_description("Number of break-glass safety backup overrides")
266 .build(),
267 safety_backup_duration_seconds: meter
268 .f64_histogram("kaniop_restore_safety_backup_duration_seconds")
269 .with_description("Duration of pre-restore safety backup creation")
270 .with_unit("s")
271 .build(),
272 }
273 }
274}
275
276#[derive(Clone)]
277struct RestoreContext {
278 client: Client,
279 recorder: Recorder,
280 metrics: RestoreMetrics,
281}
282
283async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
284 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
285 match phase {
286 KanidmRestorePhase::Pending => {
287 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
288 Ok(Action::requeue(REQUEUE))
289 }
290 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
291 Ok(target) => {
292 mark_restoring(&restore, &target, &ctx).await?;
293 let mut status = restore.status.clone().unwrap_or_default();
294 status.observed_target_uid = target.uid();
295 status.original_replicas = capture_original_replicas(&target);
296 status.phase = KanidmRestorePhase::Quiescing;
297 status.message = None;
298 patch_status(&restore, &ctx, status).await?;
299 Ok(Action::requeue(REQUEUE))
300 }
301 Err(error) => {
302 set_phase(
303 &restore,
304 &ctx,
305 KanidmRestorePhase::Failed,
306 Some(error.to_string()),
307 )
308 .await?;
309 Ok(Action::requeue(Duration::from_secs(300)))
310 }
311 },
312 KanidmRestorePhase::Quiescing => {
313 let target = get_target(&restore, &ctx).await?;
314 scale_all(&target, &ctx, 0).await?;
315 if target_pods_stopped(&target, &ctx).await?
316 && target_volumes_detached(&target, &ctx).await?
317 {
318 let next_phase = if requires_safety_backup(&restore) {
319 KanidmRestorePhase::SafetyBackup
320 } else {
321 record_break_glass(&restore, &ctx).await?;
322 KanidmRestorePhase::PreparingSource
323 };
324 if next_phase == KanidmRestorePhase::PreparingSource {
325 let refreshed = Api::<KanidmRestore>::namespaced(
326 ctx.client.clone(),
327 restore.namespace().as_deref().unwrap_or_default(),
328 )
329 .get(&restore.name_any())
330 .await
331 .map_err(|error| {
332 Error::kube_error(
333 "get",
334 "KanidmRestore",
335 restore.namespace().as_deref().unwrap_or_default(),
336 restore.name_any(),
337 error,
338 )
339 })?;
340 set_phase(&refreshed, &ctx, next_phase, None).await?;
341 } else {
342 set_phase(&restore, &ctx, next_phase, None).await?;
343 }
344 }
345 Ok(Action::requeue(REQUEUE))
346 }
347 KanidmRestorePhase::SafetyBackup => {
348 let target = get_target(&restore, &ctx).await?;
349 let name = safety_backup_job_name(&restore);
350 let expected_backup_id = compute_safety_backup_id(&restore)?;
351 let expected_manifest_key = compute_safety_manifest_key(&restore)?;
352 ensure_safety_backup_job(
353 &restore,
354 &target,
355 &ctx,
356 &name,
357 &expected_backup_id,
358 &expected_manifest_key,
359 )
360 .await?;
361 let mut status = restore.status.clone().unwrap_or_default();
362 status.safety_backup_expected_backup_id = Some(expected_backup_id.clone());
363 match job_state(&restore, &ctx, &name).await? {
364 JobState::Complete => {
365 let result =
366 read_safety_backup_result(&restore, &ctx, &name, &expected_backup_id).await;
367 match result {
368 Ok(verified) => {
369 status.safety_backup_job_name = Some(name);
370 status.safety_backup_ref =
371 Some(format!("safety-{}", restore.name_any()));
372 status.safety_backup_manifest_key = Some(verified.manifest_key);
373 status.safety_backup_payload_sha256 = Some(verified.payload_sha256);
374 record_safety_backup_duration(&restore, &ctx).await;
375 status.phase = KanidmRestorePhase::PreparingSource;
376 status.message = None;
377 patch_status(&restore, &ctx, status).await?;
378 }
379 Err(error) => {
380 resume_before_mutation(&restore, &ctx).await?;
381 set_phase(
382 &restore,
383 &ctx,
384 KanidmRestorePhase::Failed,
385 Some(format!("safety backup result verification failed: {error}")),
386 )
387 .await?;
388 }
389 }
390 }
391 JobState::Failed => {
392 resume_before_mutation(&restore, &ctx).await?;
393 set_phase(
394 &restore,
395 &ctx,
396 KanidmRestorePhase::Failed,
397 Some(
398 "safety backup job failed; target restored to original state"
399 .to_string(),
400 ),
401 )
402 .await?;
403 }
404 JobState::Running => {
405 patch_status(&restore, &ctx, status).await?;
406 }
407 }
408 Ok(Action::requeue(REQUEUE))
409 }
410 KanidmRestorePhase::PreparingSource => {
411 if is_remote_source(&restore) {
412 let target = get_target(&restore, &ctx).await?;
413 let name = source_prep_job_name(&restore);
414 ensure_source_prep_job(&restore, &target, &ctx, &name).await?;
415 match job_state(&restore, &ctx, &name).await? {
416 JobState::Complete => {
417 let backup_id = restore
418 .spec
419 .source
420 .backup_ref
421 .as_ref()
422 .map(|b| b.name.clone())
423 .unwrap_or_default();
424 let backup = {
425 let ns = restore.namespace().unwrap();
426 Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
427 .get(&backup_id)
428 .await
429 .map_err(|e| {
430 Error::kube_error("get", "KanidmBackup", &ns, &backup_id, e)
431 })?
432 };
433 let expected_backup_id = &backup.spec.backup_id;
434 let result =
435 read_source_prep_result(&restore, &ctx, &name, expected_backup_id)
436 .await;
437 match result {
438 Ok(verified) => {
439 if let Some(expected_sha256) = backup
440 .status
441 .as_ref()
442 .and_then(|s| s.payload_sha256.as_deref())
443 {
444 if verified.payload_sha256 != expected_sha256 {
445 resume_before_mutation(&restore, &ctx).await?;
446 set_phase(
447 &restore,
448 &ctx,
449 KanidmRestorePhase::Failed,
450 Some(format!(
451 "payload SHA256 mismatch: backup CR has '{expected_sha256}', downloaded payload has '{}'",
452 verified.payload_sha256
453 )),
454 )
455 .await?;
456 return Ok(Action::requeue(REQUEUE));
457 }
458 }
459 let mut status = restore.status.clone().unwrap_or_default();
460 status.source_prep_job_name = Some(name);
461 status.phase = KanidmRestorePhase::RestoringPrimary;
462 status.message = None;
463 patch_status(&restore, &ctx, status).await?;
464 }
465 Err(error) => {
466 resume_before_mutation(&restore, &ctx).await?;
467 set_phase(
468 &restore,
469 &ctx,
470 KanidmRestorePhase::Failed,
471 Some(format!(
472 "source preparation result verification failed: {error}"
473 )),
474 )
475 .await?;
476 }
477 }
478 }
479 JobState::Failed => {
480 resume_before_mutation(&restore, &ctx).await?;
481 set_phase(
482 &restore,
483 &ctx,
484 KanidmRestorePhase::Failed,
485 Some(
486 "source preparation job failed; target restored to original state"
487 .to_string(),
488 ),
489 )
490 .await?;
491 }
492 JobState::Running => {}
493 }
494 } else {
495 let target = get_target(&restore, &ctx).await?;
496 let name = source_check_job_name(&restore);
497 ensure_source_check_job(&restore, &target, &ctx, &name).await?;
498 match job_state(&restore, &ctx, &name).await? {
499 JobState::Complete => {
500 let mut status = restore.status.clone().unwrap_or_default();
501 status.source_prep_job_name = Some(name);
502 status.phase = KanidmRestorePhase::RestoringPrimary;
503 status.message = None;
504 patch_status(&restore, &ctx, status).await?;
505 }
506 JobState::Failed => {
507 resume_before_mutation(&restore, &ctx).await?;
508 set_phase(
509 &restore,
510 &ctx,
511 KanidmRestorePhase::Failed,
512 Some(
513 "local backup file check failed; target restored to original state"
514 .to_string(),
515 ),
516 )
517 .await?;
518 }
519 JobState::Running => {}
520 }
521 }
522 Ok(Action::requeue(REQUEUE))
523 }
524 KanidmRestorePhase::RestoringPrimary => {
525 let target = get_target(&restore, &ctx).await?;
526 let status = restore.status.as_ref().cloned().unwrap_or_default();
527 if requires_safety_backup(&restore) && status.safety_backup_ref.is_none() {
528 return Err(Error::MissingData(
529 "invariant violation: database mutation boundary attempted before safetyBackupRef was persisted".to_string(),
530 ));
531 }
532 if !status.database_mutation_started {
533 let mut status = status;
534 status.database_mutation_started = true;
535 status.message = Some(
536 "database mutation boundary persisted; starting primary restore".to_string(),
537 );
538 patch_status(&restore, &ctx, status).await?;
539 return Ok(Action::requeue(REQUEUE));
540 }
541 let name = restore_job_name(&restore);
542 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
543 match job_state(&restore, &ctx, &name).await? {
544 JobState::Complete => {
545 let mut status = restore.status.clone().unwrap_or_default();
546 status.restore_job_name = Some(name);
547 status.phase = KanidmRestorePhase::Verifying;
548 status.message = None;
549 patch_status(&restore, &ctx, status).await?;
550 }
551 JobState::Failed => {
552 let status = restore.status.clone().unwrap_or_default();
553 let failed_status = restore_primary_failed_status(&status, &name);
554 patch_status(&restore, &ctx, failed_status).await?;
555 }
556 JobState::Running => {}
557 }
558 Ok(Action::requeue(REQUEUE))
559 }
560 KanidmRestorePhase::Verifying => {
561 let target = get_target(&restore, &ctx).await?;
562 let name = verify_job_name(&restore);
563 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
564 match job_state(&restore, &ctx, &name).await? {
565 JobState::Complete => {
566 let mut status = restore.status.clone().unwrap_or_default();
567 status.verify_job_name = Some(name);
568 status.phase = KanidmRestorePhase::RebuildingReplicas;
569 status.message = None;
570 patch_status(&restore, &ctx, status).await?;
571 }
572 JobState::Failed => {
573 let status = restore.status.clone().unwrap_or_default();
574 let failed_status = verifying_failed_status(&status, &name);
575 patch_status(&restore, &ctx, failed_status).await?;
576 }
577 JobState::Running => {}
578 }
579 Ok(Action::requeue(REQUEUE))
580 }
581 KanidmRestorePhase::RebuildingReplicas => {
582 let target = get_target(&restore, &ctx).await?;
583 let mut status = restore.status.clone().unwrap_or_default();
584 if !status.replicas_cleared {
585 if delete_secondary_pvcs(&target, &ctx).await? {
586 status.replicas_cleared = true;
587 status.message = Some("secondary database state cleared".to_string());
588 patch_status(&restore, &ctx, status).await?;
589 }
590 return Ok(Action::requeue(REQUEUE));
591 }
592
593 scale_primary(&target, &ctx, 1).await?;
594 if !primary_ready(&target, &ctx).await? {
595 return Ok(Action::requeue(REQUEUE));
596 }
597 if !status.certificates_cleared {
598 delete_replica_cert_secrets(&target, &ctx).await?;
599 delete_admin_secret(&target, &ctx).await?;
600 status.certificates_cleared = true;
601 status.message = Some(
602 "replica certificates and admin secret cleared for regeneration".to_string(),
603 );
604 patch_status(&restore, &ctx, status).await?;
605 return Ok(Action::requeue(REQUEUE));
606 }
607 status.phase = KanidmRestorePhase::Resuming;
608 status.message = None;
609 patch_status(&restore, &ctx, status).await?;
610 Ok(Action::requeue(REQUEUE))
611 }
612 KanidmRestorePhase::Resuming => {
613 let target = get_target(&restore, &ctx).await?;
614 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
615 clear_restoring(&restore, &target, &ctx).await?;
616 Ok(Action::requeue(Duration::from_secs(3600)))
617 }
618 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
619 Ok(Action::requeue(Duration::from_secs(3600)))
620 }
621 }
622}
623
624async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
625 let target = get_target(restore, ctx).await?;
626 let actual_uid = target
627 .uid()
628 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
629 if actual_uid != restore.spec.target_ref.uid {
630 return Err(Error::MissingData(format!(
631 "target UID mismatch: expected {}, got {}",
632 restore.spec.target_ref.uid, actual_uid
633 )));
634 }
635 validate_source(restore)?;
636 if is_remote_source(restore) {
637 validate_backup_ref(restore, &target, ctx).await?;
638 }
639 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
640 {
641 return Err(Error::MissingData(format!(
642 "restoreImage must be the target's pinned Kanidm image (target image is {})",
643 target.spec.image
644 )));
645 }
646 let storage = target
647 .spec
648 .storage
649 .as_ref()
650 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
651 if storage.empty_dir.is_some()
652 || storage.ephemeral.is_some()
653 || storage.volume_claim_template.is_none()
654 {
655 return Err(Error::MissingData(
656 "restore requires PVC-backed Kanidm storage".to_string(),
657 ));
658 }
659 let primaries = target
660 .spec
661 .replica_groups
662 .iter()
663 .filter(|rg| rg.primary_node)
664 .count();
665 if primaries != 1 {
666 return Err(Error::MissingData(
667 "backup/restore requires exactly one primary replica group".to_string(),
668 ));
669 }
670 let ns = restore.namespace().unwrap();
671 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
672 .list(&ListParams::default())
673 .await
674 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
675 if restores.items.iter().any(|other| {
676 other.name_any() != restore.name_any()
677 && other.spec.target_ref.name == restore.spec.target_ref.name
678 && !matches!(
679 other.status.as_ref().map(|s| s.phase),
680 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
681 )
682 }) {
683 return Err(Error::MissingData(
684 "another active restore targets this Kanidm".to_string(),
685 ));
686 }
687 validate_safety_backup_config(restore)?;
688 Ok(target)
689}
690
691fn safe_basename(name: &str) -> bool {
692 !name.is_empty()
693 && name != "."
694 && name != ".."
695 && !name.contains('/')
696 && !name.contains('\\')
697 && !name.contains('\0')
698}
699
700fn mutable_image(image: &str) -> bool {
701 image == "kanidm/server:latest"
702 || image.ends_with(":latest")
703 || (!image.contains('@')
704 && !image
705 .rsplit('/')
706 .next()
707 .is_some_and(|part| part.contains(':')))
708}
709
710async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
711 let ns = restore
712 .namespace()
713 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
714 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
715 .get(&restore.spec.target_ref.name)
716 .await
717 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
718}
719
720async fn mark_restoring(
721 restore: &KanidmRestore,
722 target: &Kanidm,
723 ctx: &RestoreContext,
724) -> Result<()> {
725 let ns = target.namespace().unwrap();
726 let uid = restore
727 .uid()
728 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
729 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
730 .patch(
731 &target.name_any(),
732 &PatchParams::default(),
733 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
734 )
735 .await
736 .map(|_| ())
737 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
738}
739
740async fn clear_restoring(
741 restore: &KanidmRestore,
742 target: &Kanidm,
743 ctx: &RestoreContext,
744) -> Result<()> {
745 let ns = target.namespace().unwrap();
746 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
747 return Ok(());
748 }
749 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
750 .patch(
751 &target.name_any(),
752 &PatchParams::default(),
753 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
754 )
755 .await
756 .map(|_| ())
757 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
758}
759
760async fn patch_status(
761 restore: &KanidmRestore,
762 ctx: &RestoreContext,
763 mut status: KanidmRestoreStatus,
764) -> Result<()> {
765 let ns = restore.namespace().unwrap();
766 let previous_phase = restore.status.as_ref().map(|current| current.phase);
767 let next_phase = status.phase;
768 status.observed_generation = restore.metadata.generation;
769 update_restore_conditions(&mut status, restore.metadata.generation);
770
771 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
772 .patch_status(
773 &restore.name_any(),
774 &PatchParams::default(),
775 &Patch::Merge(json!({"status": &status})),
776 )
777 .await
778 .map_err(|e| {
779 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
780 })?;
781
782 if previous_phase != Some(next_phase) {
783 record_restore_transition(
784 restore,
785 ctx,
786 previous_phase,
787 next_phase,
788 status.message.as_deref(),
789 )
790 .await;
791 }
792 Ok(())
793}
794
795fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
796 let previous = status.conditions.clone();
797 let phase = status.phase;
798 let phase_reason = format!("{phase:?}");
799 let message = status
800 .message
801 .clone()
802 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
803 let terminal_success = phase == KanidmRestorePhase::Completed;
804 let terminal_failure = phase == KanidmRestorePhase::Failed;
805 let progressing = !terminal_success && !terminal_failure;
806
807 let break_glass = previous
808 .iter()
809 .find(|condition| condition.type_ == CONDITION_BREAK_GLASS)
810 .cloned();
811 status.conditions = vec![
812 restore_condition(
813 &previous,
814 CONDITION_PROGRESSING,
815 if progressing {
816 CONDITION_TRUE
817 } else {
818 CONDITION_FALSE
819 },
820 &phase_reason,
821 &message,
822 generation,
823 ),
824 restore_condition(
825 &previous,
826 CONDITION_READY,
827 if terminal_success {
828 CONDITION_TRUE
829 } else {
830 CONDITION_FALSE
831 },
832 if terminal_success {
833 "RestoreCompleted"
834 } else {
835 "RestoreNotCompleted"
836 },
837 if terminal_success {
838 "Kanidm restore completed successfully."
839 } else {
840 "Kanidm restore has not completed successfully."
841 },
842 generation,
843 ),
844 restore_condition(
845 &previous,
846 CONDITION_FAILED,
847 if terminal_failure {
848 CONDITION_TRUE
849 } else {
850 CONDITION_FALSE
851 },
852 if terminal_failure {
853 "RestoreFailed"
854 } else {
855 "NoRestoreFailure"
856 },
857 if terminal_failure {
858 status
859 .message
860 .as_deref()
861 .unwrap_or("Kanidm restore failed.")
862 } else {
863 "No terminal restore failure has been recorded."
864 },
865 generation,
866 ),
867 ];
868 if let Some(condition) = break_glass {
869 status.conditions.push(condition);
870 }
871}
872
873fn restore_condition(
874 previous: &[Condition],
875 condition_type: &str,
876 condition_status: &str,
877 reason: &str,
878 message: &str,
879 generation: Option<i64>,
880) -> Condition {
881 let last_transition_time = previous
882 .iter()
883 .find(|condition| {
884 condition.type_ == condition_type
885 && condition.status == condition_status
886 && condition.reason == reason
887 })
888 .map(|condition| condition.last_transition_time.clone())
889 .unwrap_or_else(|| Time(Timestamp::now()));
890 Condition {
891 type_: condition_type.to_string(),
892 status: condition_status.to_string(),
893 reason: reason.to_string(),
894 message: message.to_string(),
895 last_transition_time,
896 observed_generation: generation,
897 }
898}
899
900async fn record_restore_transition(
901 restore: &KanidmRestore,
902 ctx: &RestoreContext,
903 previous_phase: Option<KanidmRestorePhase>,
904 phase: KanidmRestorePhase,
905 message: Option<&str>,
906) {
907 if phase == KanidmRestorePhase::Validating {
908 ctx.metrics.attempts.add(1, &[]);
909 }
910
911 let result = match phase {
912 KanidmRestorePhase::Completed => Some("success"),
913 KanidmRestorePhase::Failed => Some("failure"),
914 _ => None,
915 };
916 if let Some(result) = result {
917 let attributes = [KeyValue::new("result", result)];
918 ctx.metrics.outcomes.add(1, &attributes);
919 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
920 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
921 ctx.metrics.duration_seconds.record(elapsed, &attributes);
922 }
923 }
924
925 let reason = if phase == KanidmRestorePhase::Failed {
926 "RestoreFailed"
927 } else {
928 "RestorePhaseChanged"
929 };
930 let note = message.map(str::to_string).or_else(|| {
931 Some(format!(
932 "Kanidm restore phase changed from {} to {phase:?}.",
933 previous_phase
934 .map(|previous| format!("{previous:?}"))
935 .unwrap_or_else(|| "None".to_string())
936 ))
937 });
938 if let Err(error) = ctx
939 .recorder
940 .publish(
941 &Event {
942 type_: if phase == KanidmRestorePhase::Failed {
943 EventType::Warning
944 } else {
945 EventType::Normal
946 },
947 reason: reason.to_string(),
948 note,
949 action: "Restore".to_string(),
950 secondary: None,
951 },
952 &restore.object_ref(&()),
953 )
954 .await
955 {
956 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
957 }
958}
959
960async fn set_phase(
961 restore: &KanidmRestore,
962 ctx: &RestoreContext,
963 phase: KanidmRestorePhase,
964 message: Option<String>,
965) -> Result<()> {
966 let mut status = restore.status.clone().unwrap_or_default();
967 status.phase = phase;
968 status.message = message;
969 patch_status(restore, ctx, status).await
970}
971
972async fn fail_after_mutation(
973 restore: &KanidmRestore,
974 ctx: &RestoreContext,
975 message: &str,
976) -> Result<()> {
977 set_phase(
978 restore,
979 ctx,
980 KanidmRestorePhase::Failed,
981 Some(message.to_string()),
982 )
983 .await
984}
985
986fn restore_primary_failed_status(
987 existing: &KanidmRestoreStatus,
988 job_name: &str,
989) -> KanidmRestoreStatus {
990 let mut status = existing.clone();
991 status.restore_job_name = Some(job_name.to_string());
992 status.phase = KanidmRestorePhase::Failed;
993 status.message = Some("database restore job failed".to_string());
994 status
995}
996
997fn verifying_failed_status(existing: &KanidmRestoreStatus, job_name: &str) -> KanidmRestoreStatus {
998 let mut status = existing.clone();
999 status.verify_job_name = Some(job_name.to_string());
1000 status.phase = KanidmRestorePhase::Failed;
1001 status.message = Some("database verification failed".to_string());
1002 status
1003}
1004
1005fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
1006 target
1007 .spec
1008 .replica_groups
1009 .iter()
1010 .find(|rg| rg.primary_node)
1011 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
1012}
1013
1014async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1015 let ns = target.namespace().unwrap();
1016 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1017 for rg in &target.spec.replica_groups {
1018 let name = target.statefulset_name(&rg.name);
1019 api.patch(
1020 &name,
1021 &PatchParams::default(),
1022 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1023 )
1024 .await
1025 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1026 }
1027 Ok(())
1028}
1029
1030async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
1031 let ns = target.namespace().unwrap();
1032 let rg = primary_group(target)?;
1033 let name = target.statefulset_name(&rg.name);
1034 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1035 .patch(
1036 &name,
1037 &PatchParams::default(),
1038 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
1039 )
1040 .await
1041 .map(|_| ())
1042 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
1043}
1044
1045async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
1046 let ns = target.namespace().unwrap();
1047 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
1048 for rg in &target.spec.replica_groups {
1049 let name = target.statefulset_name(&rg.name);
1050 api.patch(
1051 &name,
1052 &PatchParams::default(),
1053 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
1054 )
1055 .await
1056 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
1057 }
1058 Ok(())
1059}
1060
1061async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1062 let ns = target.namespace().unwrap();
1063 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
1064 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
1065 .await
1066 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
1067 Ok(pods.items.is_empty())
1068}
1069
1070async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1071 let ns = target.namespace().unwrap();
1072 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1073 let mut pv_names = BTreeSet::new();
1074 let primary_name = primary_pvc_name(target)?;
1075 let primary_pvc = pvc_api
1076 .get(&primary_name)
1077 .await
1078 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
1079 let primary_pv = primary_pvc
1080 .spec
1081 .as_ref()
1082 .and_then(|spec| spec.volume_name.clone())
1083 .ok_or_else(|| {
1084 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
1085 })?;
1086 pv_names.insert(primary_pv);
1087
1088 for rg in &target.spec.replica_groups {
1089 let sts = target.statefulset_name(&rg.name);
1090 for ordinal in 0..rg.replicas {
1091 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1092 if name == primary_name {
1093 continue;
1094 }
1095 if let Some(pvc) = pvc_api
1096 .get_opt(&name)
1097 .await
1098 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1099 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
1100 {
1101 pv_names.insert(pv_name);
1102 }
1103 }
1104 }
1105
1106 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
1107 .list(&ListParams::default())
1108 .await
1109 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
1110 Ok(!attachments.items.iter().any(|attachment| {
1111 attachment
1112 .spec
1113 .source
1114 .persistent_volume_name
1115 .as_ref()
1116 .is_some_and(|pv_name| pv_names.contains(pv_name))
1117 }))
1118}
1119
1120async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1121 let ns = target.namespace().unwrap();
1122 let name = target.statefulset_name(&primary_group(target)?.name);
1123 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
1124 .get(&name)
1125 .await
1126 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
1127 Ok(sts
1128 .status
1129 .as_ref()
1130 .and_then(|s| s.ready_replicas)
1131 .unwrap_or(0)
1132 >= 1)
1133}
1134
1135fn primary_pvc_name(target: &Kanidm) -> Result<String> {
1136 let rg = primary_group(target)?;
1137 Ok(format!(
1138 "{DATA_VOLUME}-{}-0",
1139 target.statefulset_name(&rg.name)
1140 ))
1141}
1142
1143async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1144 let ns = target.namespace().unwrap();
1145 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
1146 let mut all_absent = true;
1147 for rg in &target.spec.replica_groups {
1148 let sts = target.statefulset_name(&rg.name);
1149 for ordinal in 0..rg.replicas {
1150 if rg.primary_node && ordinal == 0 {
1151 continue;
1152 }
1153 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
1154 match api
1155 .get_opt(&name)
1156 .await
1157 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
1158 {
1159 None => {}
1160 Some(pvc) => {
1161 all_absent = false;
1162 if pvc.metadata.deletion_timestamp.is_none() {
1163 match api.delete(&name, &DeleteParams::default()).await {
1164 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
1165 Err(kube::Error::Api(status)) if status.code == 404 => {}
1166 Err(error) => {
1167 return Err(Error::kube_error(
1168 "delete",
1169 "PersistentVolumeClaim",
1170 &ns,
1171 &name,
1172 error,
1173 ));
1174 }
1175 }
1176 }
1177 }
1178 }
1179 }
1180 }
1181 Ok(all_absent)
1182}
1183
1184async fn delete_replica_cert_secrets(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1185 let ns = target.namespace().unwrap();
1186 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1187 let label_selector = format!(
1188 "{}={},{}={}",
1189 CLUSTER_LABEL,
1190 target.name_any(),
1191 SECRET_TYPE_LABEL,
1192 serde_plain::to_string(&SecretType::ReplicaCert).unwrap()
1193 );
1194 let secrets = api
1195 .list(&ListParams::default().labels(&label_selector))
1196 .await
1197 .map_err(|e| Error::kube_error("list", "Secret", &ns, &label_selector, e))?;
1198 let mut all_absent = secrets.items.is_empty();
1199 for secret in secrets.items {
1200 let name = secret.name_any();
1201 if secret.metadata.deletion_timestamp.is_none() {
1202 match api.delete(&name, &DeleteParams::default()).await {
1203 Ok(_) => debug!(secret = %name, "deleting stale replica certificate secret"),
1204 Err(kube::Error::Api(status)) if status.code == 404 => {}
1205 Err(error) => {
1206 return Err(Error::kube_error("delete", "Secret", &ns, &name, error));
1207 }
1208 }
1209 all_absent = false;
1210 }
1211 }
1212 Ok(all_absent)
1213}
1214
1215async fn delete_admin_secret(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
1216 let ns = target.namespace().unwrap();
1217 let api = Api::<Secret>::namespaced(ctx.client.clone(), &ns);
1218 let name = format!("{}-admin-passwords", target.name_any());
1219 match api
1220 .get_opt(&name)
1221 .await
1222 .map_err(|e| Error::kube_error("get", "Secret", &ns, &name, e))?
1223 {
1224 None => Ok(true),
1225 Some(secret) => {
1226 if secret.metadata.deletion_timestamp.is_some() {
1227 return Ok(false);
1228 }
1229 match api.delete(&name, &DeleteParams::default()).await {
1230 Ok(_) => {
1231 debug!(secret = %name, "deleting stale admin secret");
1232 Ok(false)
1233 }
1234 Err(kube::Error::Api(status)) if status.code == 404 => Ok(true),
1235 Err(error) => Err(Error::kube_error("delete", "Secret", &ns, &name, error)),
1236 }
1237 }
1238 }
1239}
1240
1241fn restore_job_name(restore: &KanidmRestore) -> String {
1242 format!("{}-restore", restore.name_any())
1243}
1244fn verify_job_name(restore: &KanidmRestore) -> String {
1245 format!("{}-verify", restore.name_any())
1246}
1247
1248fn build_kanidm_db_env_vars(target: &Kanidm) -> Vec<EnvVar> {
1249 let origin = target
1250 .spec
1251 .origin
1252 .clone()
1253 .unwrap_or_else(|| format!("https://{}", target.spec.domain));
1254 vec![
1255 EnvVar {
1256 name: "KANIDM_DB_PATH".to_string(),
1257 value: Some("/data/kanidm.db".to_string()),
1258 ..Default::default()
1259 },
1260 EnvVar {
1261 name: "KANIDM_DOMAIN".to_string(),
1262 value: Some(target.spec.domain.clone()),
1263 ..Default::default()
1264 },
1265 EnvVar {
1266 name: "KANIDM_ORIGIN".to_string(),
1267 value: Some(origin),
1268 ..Default::default()
1269 },
1270 EnvVar {
1271 name: "KANIDM_TLS_CHAIN".to_string(),
1272 value: Some(format!("{TLS_PATH}/tls.crt")),
1273 ..Default::default()
1274 },
1275 EnvVar {
1276 name: "KANIDM_TLS_KEY".to_string(),
1277 value: Some(format!("{TLS_PATH}/tls.key")),
1278 ..Default::default()
1279 },
1280 ]
1281}
1282
1283async fn ensure_database_job(
1284 restore: &KanidmRestore,
1285 target: &Kanidm,
1286 ctx: &RestoreContext,
1287 name: &str,
1288 verify: bool,
1289) -> Result<()> {
1290 let ns = restore.namespace().unwrap();
1291 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1292 if api
1293 .get_opt(name)
1294 .await
1295 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1296 .is_some()
1297 {
1298 return Ok(());
1299 }
1300 let mut command = vec![
1301 "kanidmd".to_string(),
1302 "database".to_string(),
1303 if verify { "backup" } else { "restore" }.to_string(),
1304 ];
1305 if verify {
1306 command.push("/verify/verification.json.gz".to_string());
1307 } else {
1308 if is_remote_source(restore) {
1309 command.push(format!("{DATA_PATH}/source-payload.json.gz"));
1310 } else {
1311 let file_name = restore
1312 .spec
1313 .source
1314 .local
1315 .as_ref()
1316 .map(|l| l.file_name.clone())
1317 .unwrap_or_default();
1318 command.push(format!("{BACKUP_PATH}/{file_name}"));
1319 }
1320 }
1321 let job = Job {
1322 metadata: ObjectMeta {
1323 name: Some(name.to_string()),
1324 namespace: Some(ns.clone()),
1325 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1326 ..Default::default()
1327 },
1328 spec: Some(JobSpec {
1329 backoff_limit: Some(0),
1330 template: PodTemplateSpec {
1331 metadata: Some(ObjectMeta::default()),
1332 spec: Some(PodSpec {
1333 automount_service_account_token: Some(false),
1334 restart_policy: Some("Never".to_string()),
1335 security_context: Some(source_job_pod_security_context(target)),
1336 containers: vec![Container {
1337 name: if verify { "verify" } else { "restore" }.to_string(),
1338 image: Some(restore.spec.restore_image.clone()),
1339 command: Some(command),
1340 env: Some(build_kanidm_db_env_vars(target)),
1341 security_context: Some(kanidm_job_security_context()),
1342 resources: Some(job_resource_requirements()),
1343 volume_mounts: Some(vec![
1344 VolumeMount {
1345 name: DATA_VOLUME.to_string(),
1346 mount_path: DATA_PATH.to_string(),
1347 ..Default::default()
1348 },
1349 VolumeMount {
1350 name: TLS_VOLUME.to_string(),
1351 mount_path: TLS_PATH.to_string(),
1352 read_only: Some(true),
1353 ..Default::default()
1354 },
1355 VolumeMount {
1356 name: "verification-output".to_string(),
1357 mount_path: "/verify".to_string(),
1358 ..Default::default()
1359 },
1360 ]),
1361 ..Default::default()
1362 }],
1363 volumes: Some(vec![
1364 Volume {
1365 name: DATA_VOLUME.to_string(),
1366 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1367 claim_name: primary_pvc_name(target)?,
1368 read_only: Some(false),
1369 }),
1370 ..Default::default()
1371 },
1372 Volume {
1373 name: TLS_VOLUME.to_string(),
1374 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
1375 secret_name: Some(target.effective_tls_secret_name()),
1376 ..Default::default()
1377 }),
1378 ..Default::default()
1379 },
1380 Volume {
1381 name: "verification-output".to_string(),
1382 empty_dir: Some(Default::default()),
1383 ..Default::default()
1384 },
1385 ]),
1386 ..Default::default()
1387 }),
1388 },
1389 ..Default::default()
1390 }),
1391 ..Default::default()
1392 };
1393 api.create(&PostParams::default(), &job)
1394 .await
1395 .map(|_| ())
1396 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1397}
1398
1399enum JobState {
1400 Running,
1401 Complete,
1402 Failed,
1403}
1404
1405async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1406 let ns = restore.namespace().unwrap();
1407 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1408 .get(name)
1409 .await
1410 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1411 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1412 return Ok(JobState::Failed);
1413 }
1414 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1415 return Ok(JobState::Complete);
1416 }
1417 Ok(JobState::Running)
1418}
1419
1420fn is_remote_source(restore: &KanidmRestore) -> bool {
1421 restore.spec.source.backup_ref.is_some()
1422}
1423
1424fn requires_safety_backup(restore: &KanidmRestore) -> bool {
1425 let safety = restore.spec.safety_backup.as_ref();
1426 let skip = safety.is_some_and(|s| s.skip);
1427 !skip
1428}
1429
1430fn validate_source(restore: &KanidmRestore) -> Result<()> {
1431 let has_local = restore.spec.source.local.is_some();
1432 let has_backup_ref = restore.spec.source.backup_ref.is_some();
1433 if has_local && has_backup_ref {
1434 return Err(Error::MissingData(
1435 "source.local and source.backupRef are mutually exclusive".to_string(),
1436 ));
1437 }
1438 if !has_local && !has_backup_ref {
1439 return Err(Error::MissingData(
1440 "source must specify either local or backupRef".to_string(),
1441 ));
1442 }
1443 if let Some(local) = &restore.spec.source.local {
1444 if !safe_basename(&local.file_name) {
1445 return Err(Error::MissingData(
1446 "restore source fileName must be a safe basename".to_string(),
1447 ));
1448 }
1449 }
1450 if let Some(backup_ref) = &restore.spec.source.backup_ref {
1451 if backup_ref.name.is_empty() {
1452 return Err(Error::MissingData(
1453 "source.backupRef.name must not be empty".to_string(),
1454 ));
1455 }
1456 }
1457 Ok(())
1458}
1459
1460fn validate_safety_backup_config(restore: &KanidmRestore) -> Result<()> {
1461 let safety = restore.spec.safety_backup.as_ref();
1462 let skip = safety.is_some_and(|s| s.skip);
1463 if skip {
1464 let annotations = restore.metadata.annotations.as_ref();
1465 let reason = annotations
1466 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1467 .map(|s| s.trim())
1468 .unwrap_or("");
1469 let approver = annotations
1470 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1471 .map(|s| s.trim())
1472 .unwrap_or("");
1473 if reason.is_empty() {
1474 return Err(Error::MissingData(format!(
1475 "break-glass requires non-empty annotation '{BREAK_GLASS_REASON_ANNOTATION}'"
1476 )));
1477 }
1478 if approver.is_empty() {
1479 return Err(Error::MissingData(format!(
1480 "break-glass requires non-empty annotation '{BREAK_GLASS_APPROVED_BY_ANNOTATION}'"
1481 )));
1482 }
1483 }
1484 if !skip {
1485 let has_repo = safety
1486 .and_then(|s| s.repository_ref.as_ref())
1487 .is_some_and(|r| !r.name.is_empty());
1488 if !has_repo {
1489 return Err(Error::MissingData(
1490 "safety backup requires safetyBackup.repositoryRef when skip is not set"
1491 .to_string(),
1492 ));
1493 }
1494 }
1495 Ok(())
1496}
1497
1498fn capture_original_replicas(target: &Kanidm) -> Vec<ReplicaCountEntry> {
1499 target
1500 .spec
1501 .replica_groups
1502 .iter()
1503 .map(|rg| ReplicaCountEntry {
1504 group: rg.name.clone(),
1505 replicas: rg.replicas,
1506 })
1507 .collect()
1508}
1509
1510async fn resume_before_mutation(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1511 if let Ok(target) = get_target(restore, ctx).await {
1512 let owns_maintenance =
1513 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
1514 if owns_maintenance {
1515 scale_desired(&target, ctx).await?;
1516 clear_restoring(restore, &target, ctx).await?;
1517 }
1518 }
1519 Ok(())
1520}
1521
1522async fn record_break_glass(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<()> {
1523 ctx.metrics.break_glass_total.add(1, &[]);
1524 let reason = restore
1525 .metadata
1526 .annotations
1527 .as_ref()
1528 .and_then(|a| a.get(BREAK_GLASS_REASON_ANNOTATION))
1529 .cloned()
1530 .unwrap_or_default();
1531 let approver = restore
1532 .metadata
1533 .annotations
1534 .as_ref()
1535 .and_then(|a| a.get(BREAK_GLASS_APPROVED_BY_ANNOTATION))
1536 .cloned()
1537 .unwrap_or_default();
1538 let mut status = restore.status.clone().unwrap_or_default();
1539 let break_glass_condition = restore_condition(
1540 &status.conditions,
1541 CONDITION_BREAK_GLASS,
1542 CONDITION_TRUE,
1543 "BreakGlassOverride",
1544 &format!("Safety backup skipped via break-glass. reason={reason}, approvedBy={approver}"),
1545 restore.metadata.generation,
1546 );
1547 status
1548 .conditions
1549 .retain(|c| c.type_ != CONDITION_BREAK_GLASS);
1550 status.conditions.push(break_glass_condition);
1551 patch_status(restore, ctx, status).await?;
1552 if let Err(error) = ctx
1553 .recorder
1554 .publish(
1555 &Event {
1556 type_: EventType::Warning,
1557 reason: "SafetyBackupSkipped".to_string(),
1558 note: Some(format!(
1559 "Break-glass: safety backup skipped. reason={reason}, approvedBy={approver}"
1560 )),
1561 action: "Restore".to_string(),
1562 secondary: None,
1563 },
1564 &restore.object_ref(&()),
1565 )
1566 .await
1567 {
1568 warn!(restore = %restore.name_any(), %error, "failed to publish break-glass event");
1569 }
1570 warn!(
1571 restore = %restore.name_any(),
1572 reason = %reason,
1573 approver = %approver,
1574 "break-glass safety backup override activated"
1575 );
1576 Ok(())
1577}
1578
1579async fn record_safety_backup_duration(restore: &KanidmRestore, ctx: &RestoreContext) {
1580 let status = restore.status.as_ref().cloned().unwrap_or_default();
1581 if let Some(start) = status.phase_timestamps.get("SafetyBackup") {
1582 if let Ok(start_ts) = start.parse::<Timestamp>() {
1583 let elapsed = (Timestamp::now().as_second() - start_ts.as_second()).max(0) as f64;
1584 ctx.metrics
1585 .safety_backup_duration_seconds
1586 .record(elapsed, &[]);
1587 }
1588 }
1589}
1590
1591fn safety_backup_job_name(restore: &KanidmRestore) -> String {
1592 format!("{}-safety-backup", restore.name_any())
1593}
1594
1595fn source_prep_job_name(restore: &KanidmRestore) -> String {
1596 format!("{}-source-prep", restore.name_any())
1597}
1598
1599fn source_check_job_name(restore: &KanidmRestore) -> String {
1600 format!("{}-source-check", restore.name_any())
1601}
1602
1603async fn ensure_source_check_job(
1604 restore: &KanidmRestore,
1605 target: &Kanidm,
1606 ctx: &RestoreContext,
1607 name: &str,
1608) -> Result<()> {
1609 let ns = restore.namespace().unwrap();
1610 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1611 if api
1612 .get_opt(name)
1613 .await
1614 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1615 .is_some()
1616 {
1617 return Ok(());
1618 }
1619 let file_name = restore
1620 .spec
1621 .source
1622 .local
1623 .as_ref()
1624 .map(|l| l.file_name.clone())
1625 .unwrap_or_default();
1626 let operation_doc = serde_json::json!({
1627 "apiVersion": "backup.kaniop.rs/v1alpha1",
1628 "kind": "OperationDocument",
1629 "operation": "check",
1630 "path": format!("{BACKUP_PATH}/{file_name}"),
1631 "resultPath": "/run/kaniop-result/result.json",
1632 "format": "kanidmJsonGzip",
1633 })
1634 .to_string();
1635 let operation_cm_name = format!("{}-source-check-op", restore.name_any());
1636 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
1637 let data_mover_image = data_mover_image();
1638 let job = Job {
1639 metadata: ObjectMeta {
1640 name: Some(name.to_string()),
1641 namespace: Some(ns.clone()),
1642 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1643 ..Default::default()
1644 },
1645 spec: Some(JobSpec {
1646 backoff_limit: Some(0),
1647 template: PodTemplateSpec {
1648 metadata: Some(ObjectMeta::default()),
1649 spec: Some(PodSpec {
1650 automount_service_account_token: Some(false),
1651 restart_policy: Some("Never".to_string()),
1652 security_context: Some(source_job_pod_security_context(target)),
1653 containers: vec![Container {
1654 name: "source-check".to_string(),
1655 image: Some(data_mover_image),
1656 command: Some(vec![
1657 "/bin/kaniop-data-mover".to_string(),
1658 "check".to_string(),
1659 "--operation-doc".to_string(),
1660 "/run/kaniop/operation.json".to_string(),
1661 ]),
1662 security_context: Some(hardened_security_context()),
1663 resources: Some(job_resource_requirements()),
1664 env: Some(vec![EnvVar {
1665 name: "RUST_LOG".to_string(),
1666 value: Some("info".to_string()),
1667 ..Default::default()
1668 }]),
1669 volume_mounts: Some(vec![
1670 VolumeMount {
1671 name: DATA_VOLUME.to_string(),
1672 mount_path: DATA_PATH.to_string(),
1673 read_only: Some(true),
1674 ..Default::default()
1675 },
1676 VolumeMount {
1677 name: "operation".to_string(),
1678 mount_path: "/run/kaniop".to_string(),
1679 read_only: Some(true),
1680 ..Default::default()
1681 },
1682 VolumeMount {
1683 name: "result".to_string(),
1684 mount_path: "/run/kaniop-result".to_string(),
1685 ..Default::default()
1686 },
1687 ]),
1688 ..Default::default()
1689 }],
1690 volumes: Some(vec![
1691 Volume {
1692 name: DATA_VOLUME.to_string(),
1693 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1694 claim_name: primary_pvc_name(target)?,
1695 read_only: Some(true),
1696 }),
1697 ..Default::default()
1698 },
1699 Volume {
1700 name: "operation".to_string(),
1701 config_map: Some(k8s_openapi::api::core::v1::ConfigMapVolumeSource {
1702 name: operation_cm_name,
1703 ..Default::default()
1704 }),
1705 ..Default::default()
1706 },
1707 Volume {
1708 name: "result".to_string(),
1709 empty_dir: Some(Default::default()),
1710 ..Default::default()
1711 },
1712 ]),
1713 ..Default::default()
1714 }),
1715 },
1716 ..Default::default()
1717 }),
1718 ..Default::default()
1719 };
1720 api.create(&PostParams::default(), &job)
1721 .await
1722 .map(|_| ())
1723 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1724}
1725
1726const SAFETY_BACKUP_NAMESPACE: &str = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
1727
1728fn compute_safety_backup_id(restore: &KanidmRestore) -> Result<String> {
1729 let ns = restore.namespace().unwrap_or_default();
1730 let uid = restore.uid().ok_or_else(|| {
1731 Error::MissingData(
1732 "restore CR has no UID; cannot derive safety backup ID without a persisted UID"
1733 .to_string(),
1734 )
1735 })?;
1736 let namespace_uuid =
1737 uuid::Uuid::parse_str(SAFETY_BACKUP_NAMESPACE).expect("valid UUID for namespace");
1738 let name_uuid = uuid::Uuid::new_v5(&namespace_uuid, ns.as_bytes());
1739 Ok(uuid::Uuid::new_v5(&name_uuid, uid.as_bytes()).to_string())
1740}
1741
1742fn compute_safety_manifest_key(restore: &KanidmRestore) -> Result<String> {
1743 let ns = restore.namespace().unwrap_or_default();
1744 let kanidm_uid = &restore.spec.target_ref.uid;
1745 let backup_id = compute_safety_backup_id(restore)?;
1746 Ok(format!(
1747 "v1/tenants/{ns}/clusters/{kanidm_uid}/backups/{backup_id}/manifest.json"
1748 ))
1749}
1750
1751struct VerifiedSafetyBackup {
1752 manifest_key: String,
1753 payload_sha256: String,
1754}
1755
1756async fn read_safety_backup_result(
1757 restore: &KanidmRestore,
1758 ctx: &RestoreContext,
1759 job_name: &str,
1760 expected_backup_id: &str,
1761) -> Result<VerifiedSafetyBackup> {
1762 let ns = restore.namespace().unwrap();
1763 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1764 let pod_list = pods
1765 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1766 .await
1767 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1768 let pod = pod_list.items.first().ok_or_else(|| {
1769 Error::MissingData(format!("no pod found for safety backup job {job_name}"))
1770 })?;
1771 let pod_name = pod.name_any();
1772 let container_status = pod
1773 .status
1774 .as_ref()
1775 .and_then(|s| s.container_statuses.as_ref())
1776 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "safety-upload"))
1777 .ok_or_else(|| {
1778 Error::MissingData(format!(
1779 "safety-upload container status not found in pod {pod_name}"
1780 ))
1781 })?;
1782 let termination_message = container_status
1783 .state
1784 .as_ref()
1785 .and_then(|state| state.terminated.as_ref())
1786 .and_then(|t| t.message.as_ref())
1787 .ok_or_else(|| {
1788 Error::MissingData(format!(
1789 "termination message not found in pod {pod_name}; result transport failed"
1790 ))
1791 })?;
1792 if termination_message.is_empty() {
1793 return Err(Error::MissingData(format!(
1794 "termination message is empty in pod {pod_name}; result document was not written"
1795 )));
1796 }
1797 let result_doc = parse_result_document(termination_message).map_err(|e| {
1798 Error::ParseError(format!(
1799 "failed to parse result document from pod {pod_name}: {e}"
1800 ))
1801 })?;
1802 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1803 return Err(Error::ParseError(format!(
1804 "unsupported result document version: {}",
1805 result_doc.api_version
1806 )));
1807 }
1808 if result_doc.kind != "ResultDocument" {
1809 return Err(Error::ParseError(format!(
1810 "invalid result document kind: {}",
1811 result_doc.kind
1812 )));
1813 }
1814 if result_doc.operation != SAFETY_BACKUP_RESULT_OPERATION {
1815 return Err(Error::ParseError(format!(
1816 "result document operation mismatch: expected '{}', got '{}'",
1817 SAFETY_BACKUP_RESULT_OPERATION, result_doc.operation
1818 )));
1819 }
1820 if !result_doc.success {
1821 return Err(Error::ParseError(format!(
1822 "safety backup reported failure with exit code {:?}",
1823 result_doc.exit_code
1824 )));
1825 }
1826 if result_doc.exit_code != ExitCode::Success {
1827 return Err(Error::ParseError(format!(
1828 "safety backup exit code is not Success: {:?}",
1829 result_doc.exit_code
1830 )));
1831 }
1832 let doc_backup_id = result_doc
1833 .backup_id
1834 .as_deref()
1835 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1836 if doc_backup_id != expected_backup_id {
1837 return Err(Error::ParseError(format!(
1838 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1839 )));
1840 }
1841 let manifest_key = result_doc
1842 .manifest_key
1843 .clone()
1844 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1845 if manifest_key.is_empty() {
1846 return Err(Error::ParseError(
1847 "result document manifestKey is empty".to_string(),
1848 ));
1849 }
1850 let payload_sha256 = result_doc
1851 .payload_sha256
1852 .clone()
1853 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1854 if payload_sha256.is_empty() {
1855 return Err(Error::ParseError(
1856 "result document payloadSha256 is empty".to_string(),
1857 ));
1858 }
1859 info!(
1860 restore = %restore.name_any(),
1861 backup_id = %expected_backup_id,
1862 manifest_key = %manifest_key,
1863 "safety backup result verified"
1864 );
1865 Ok(VerifiedSafetyBackup {
1866 manifest_key,
1867 payload_sha256,
1868 })
1869}
1870
1871struct VerifiedSourcePrepResult {
1872 #[allow(dead_code)]
1873 manifest_key: String,
1874 payload_sha256: String,
1875}
1876
1877async fn read_source_prep_result(
1878 restore: &KanidmRestore,
1879 ctx: &RestoreContext,
1880 job_name: &str,
1881 expected_backup_id: &str,
1882) -> Result<VerifiedSourcePrepResult> {
1883 let ns = restore.namespace().unwrap();
1884 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
1885 let pod_list = pods
1886 .list(&ListParams::default().labels(&format!("job-name={job_name}")))
1887 .await
1888 .map_err(|e| Error::kube_error("list", "Pod", &ns, job_name, e))?;
1889 let pod = pod_list.items.first().ok_or_else(|| {
1890 Error::MissingData(format!("no pod found for source prep job {job_name}"))
1891 })?;
1892 let pod_name = pod.name_any();
1893 let container_status = pod
1894 .status
1895 .as_ref()
1896 .and_then(|s| s.container_statuses.as_ref())
1897 .and_then(|statuses| statuses.iter().find(|cs| cs.name == "source-prep"))
1898 .ok_or_else(|| {
1899 Error::MissingData(format!(
1900 "source-prep container status not found in pod {pod_name}"
1901 ))
1902 })?;
1903 let termination_message = container_status
1904 .state
1905 .as_ref()
1906 .and_then(|state| state.terminated.as_ref())
1907 .and_then(|t| t.message.as_ref())
1908 .ok_or_else(|| {
1909 Error::MissingData(format!(
1910 "termination message not found in pod {pod_name}; result transport failed"
1911 ))
1912 })?;
1913 if termination_message.is_empty() {
1914 return Err(Error::MissingData(format!(
1915 "termination message is empty in pod {pod_name}; result document was not written"
1916 )));
1917 }
1918 let result_doc = parse_result_document(termination_message).map_err(|e| {
1919 Error::ParseError(format!(
1920 "failed to parse result document from pod {pod_name}: {e}"
1921 ))
1922 })?;
1923 if result_doc.api_version != kaniop_backup_core::result::RESULT_DOC_VERSION {
1924 return Err(Error::ParseError(format!(
1925 "unsupported result document version: {}",
1926 result_doc.api_version
1927 )));
1928 }
1929 if result_doc.kind != "ResultDocument" {
1930 return Err(Error::ParseError(format!(
1931 "invalid result document kind: {}",
1932 result_doc.kind
1933 )));
1934 }
1935 if result_doc.operation != SOURCE_PREP_RESULT_OPERATION {
1936 return Err(Error::ParseError(format!(
1937 "result document operation mismatch: expected '{}', got '{}'",
1938 SOURCE_PREP_RESULT_OPERATION, result_doc.operation
1939 )));
1940 }
1941 if !result_doc.success {
1942 return Err(Error::ParseError(format!(
1943 "source prep reported failure with exit code {:?}",
1944 result_doc.exit_code
1945 )));
1946 }
1947 if result_doc.exit_code != ExitCode::Success {
1948 return Err(Error::ParseError(format!(
1949 "source prep exit code is not Success: {:?}",
1950 result_doc.exit_code
1951 )));
1952 }
1953 let doc_backup_id = result_doc
1954 .backup_id
1955 .as_deref()
1956 .ok_or_else(|| Error::ParseError("result document missing backupId".to_string()))?;
1957 if doc_backup_id != expected_backup_id {
1958 return Err(Error::ParseError(format!(
1959 "backup ID mismatch: expected '{expected_backup_id}', got '{doc_backup_id}'"
1960 )));
1961 }
1962 let manifest_key = result_doc
1963 .manifest_key
1964 .clone()
1965 .ok_or_else(|| Error::ParseError("result document missing manifestKey".to_string()))?;
1966 if manifest_key.is_empty() {
1967 return Err(Error::ParseError(
1968 "result document manifestKey is empty".to_string(),
1969 ));
1970 }
1971 let payload_sha256 = result_doc
1972 .payload_sha256
1973 .clone()
1974 .ok_or_else(|| Error::ParseError("result document missing payloadSha256".to_string()))?;
1975 if payload_sha256.is_empty() {
1976 return Err(Error::ParseError(
1977 "result document payloadSha256 is empty".to_string(),
1978 ));
1979 }
1980 info!(
1981 restore = %restore.name_any(),
1982 backup_id = %expected_backup_id,
1983 manifest_key = %manifest_key,
1984 payload_sha256 = %payload_sha256,
1985 "source prep result verified"
1986 );
1987 Ok(VerifiedSourcePrepResult {
1988 manifest_key,
1989 payload_sha256,
1990 })
1991}
1992
1993fn source_job_pod_security_context(
1994 target: &Kanidm,
1995) -> k8s_openapi::api::core::v1::PodSecurityContext {
1996 let mut security_context = target.spec.security_context.clone().unwrap_or_default();
1997 security_context.seccomp_profile = Some(SeccompProfile {
1998 type_: "RuntimeDefault".to_string(),
1999 ..Default::default()
2000 });
2001 security_context
2002}
2003
2004fn hardened_security_context() -> SecurityContext {
2005 SecurityContext {
2006 allow_privilege_escalation: Some(false),
2007 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2008 drop: Some(vec!["ALL".to_string()]),
2009 ..Default::default()
2010 }),
2011 read_only_root_filesystem: Some(true),
2012 run_as_non_root: Some(true),
2013 seccomp_profile: Some(SeccompProfile {
2014 type_: "RuntimeDefault".to_string(),
2015 ..Default::default()
2016 }),
2017 ..Default::default()
2018 }
2019}
2020
2021fn kanidm_job_security_context() -> SecurityContext {
2022 SecurityContext {
2023 allow_privilege_escalation: Some(false),
2024 capabilities: Some(k8s_openapi::api::core::v1::Capabilities {
2025 drop: Some(vec!["ALL".to_string()]),
2026 ..Default::default()
2027 }),
2028 seccomp_profile: Some(SeccompProfile {
2029 type_: "RuntimeDefault".to_string(),
2030 ..Default::default()
2031 }),
2032 ..Default::default()
2033 }
2034}
2035
2036fn job_resource_requirements() -> ResourceRequirements {
2037 ResourceRequirements {
2038 requests: Some(std::collections::BTreeMap::from([
2039 ("cpu".to_string(), Quantity("100m".to_string())),
2040 ("memory".to_string(), Quantity("128Mi".to_string())),
2041 ])),
2042 limits: Some(std::collections::BTreeMap::from([
2043 ("cpu".to_string(), Quantity("2".to_string())),
2044 ("memory".to_string(), Quantity("2Gi".to_string())),
2045 ])),
2046 ..Default::default()
2047 }
2048}
2049
2050async fn ensure_safety_backup_job(
2051 restore: &KanidmRestore,
2052 target: &Kanidm,
2053 ctx: &RestoreContext,
2054 name: &str,
2055 backup_id: &str,
2056 manifest_key: &str,
2057) -> Result<()> {
2058 let ns = restore.namespace().unwrap();
2059 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2060 if api
2061 .get_opt(name)
2062 .await
2063 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2064 .is_some()
2065 {
2066 return Ok(());
2067 }
2068 let safety = restore
2069 .spec
2070 .safety_backup
2071 .as_ref()
2072 .ok_or_else(|| Error::MissingData("safetyBackup config required".to_string()))?;
2073 let repo_ref = safety
2074 .repository_ref
2075 .as_ref()
2076 .ok_or_else(|| Error::MissingData("safetyBackup.repositoryRef required".to_string()))?;
2077 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2078 .get(&repo_ref.name)
2079 .await
2080 .map_err(|e| Error::kube_error("get", "KanidmBackupRepository", &ns, &repo_ref.name, e))?;
2081 let _namespace_uid = &ns;
2082 let _kanidm_uid = &restore.spec.target_ref.uid;
2083 let endpoint = &repo.spec.s3.endpoint;
2084 let region = &repo.spec.s3.region;
2085 let operation_doc = build_safety_upload_operation_doc(
2086 restore,
2087 target,
2088 backup_id,
2089 manifest_key,
2090 &repo.spec.s3.bucket,
2091 &repo.spec.s3.prefix,
2092 endpoint,
2093 region,
2094 repo.spec.s3.force_path_style,
2095 repo.spec.s3.insecure,
2096 repo.spec
2097 .s3
2098 .ca_bundle_ref
2099 .as_deref()
2100 .map(|_| ca_bundle_path())
2101 .as_deref(),
2102 repo.spec.encryption.as_ref(),
2103 )?;
2104 let operation_cm_name = format!("{name}-op");
2105 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2106 let auth_method = &repo.spec.authentication.writer;
2107 let data_mover_image = data_mover_image();
2108 let init_command = vec![
2109 "kanidmd".to_string(),
2110 "database".to_string(),
2111 "backup".to_string(),
2112 format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2113 ];
2114 let upload_script = format!(
2115 r#"set -eu
2116/bin/kaniop-data-mover upload --operation-doc /run/kaniop/operation.json
2117RESULT_FILE="/run/kaniop-result/result.json"
2118TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2119if [ ! -f "$RESULT_FILE" ]; then
2120 echo "ERROR: result document not found at $RESULT_FILE"
2121 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2122 exit 1
2123fi
2124if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2125 echo "ERROR: safety backup upload reported failure"
2126 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2127 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2128 exit 1
2129fi
2130dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2131echo "safety backup upload completed successfully"
2132"#
2133 );
2134 let job = Job {
2135 metadata: ObjectMeta {
2136 name: Some(name.to_string()),
2137 namespace: Some(ns.clone()),
2138 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2139 ..Default::default()
2140 },
2141 spec: Some(JobSpec {
2142 backoff_limit: Some(0),
2143 template: PodTemplateSpec {
2144 metadata: Some(ObjectMeta::default()),
2145 spec: Some(PodSpec {
2146 automount_service_account_token: Some(false),
2147 restart_policy: Some("Never".to_string()),
2148 security_context: Some(source_job_pod_security_context(target)),
2149 init_containers: Some(vec![Container {
2150 name: "safety-backup".to_string(),
2151 image: Some(restore.spec.restore_image.clone()),
2152 command: Some(init_command),
2153 security_context: Some(SecurityContext {
2154 allow_privilege_escalation: Some(false),
2155 capabilities: Some(Capabilities {
2156 drop: Some(vec!["ALL".to_string()]),
2157 ..Default::default()
2158 }),
2159 read_only_root_filesystem: Some(true),
2160 seccomp_profile: Some(SeccompProfile {
2161 type_: "RuntimeDefault".to_string(),
2162 ..Default::default()
2163 }),
2164 ..Default::default()
2165 }),
2166 resources: Some(job_resource_requirements()),
2167 env: Some(build_kanidm_db_env_vars(target)),
2168 volume_mounts: Some(vec![
2169 VolumeMount {
2170 name: DATA_VOLUME.to_string(),
2171 mount_path: DATA_PATH.to_string(),
2172 read_only: Some(false),
2173 ..Default::default()
2174 },
2175 VolumeMount {
2176 name: TLS_VOLUME.to_string(),
2177 mount_path: TLS_PATH.to_string(),
2178 read_only: Some(true),
2179 ..Default::default()
2180 },
2181 VolumeMount {
2182 name: SHARED_VOLUME.to_string(),
2183 mount_path: SHARED_VOL_PATH.to_string(),
2184 ..Default::default()
2185 },
2186 ]),
2187 ..Default::default()
2188 }]),
2189 containers: vec![Container {
2190 name: "safety-upload".to_string(),
2191 image: Some(data_mover_image),
2192 command: Some(vec!["/bin/sh".to_string()]),
2193 args: Some(vec!["-c".to_string(), upload_script]),
2194 security_context: Some(hardened_security_context()),
2195 resources: Some(job_resource_requirements()),
2196 env: {
2197 let mut env_vars =
2198 build_auth_env_vars(auth_method, &repo_ref.name, AuthRole::Writer);
2199 env_vars.push(EnvVar {
2200 name: "RUST_LOG".to_string(),
2201 value: Some("info".to_string()),
2202 ..Default::default()
2203 });
2204 if repo.spec.s3.ca_bundle_ref.is_some() {
2205 env_vars.push(ca_bundle_env_var());
2206 }
2207 env_vars.extend(build_encryption_env_vars(
2208 repo.spec
2209 .encryption
2210 .as_ref()
2211 .and_then(|e| e.key_ref.as_ref()),
2212 ));
2213 Some(env_vars)
2214 },
2215 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2216 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2217 volume_mounts: {
2218 let mut mounts = vec![
2219 VolumeMount {
2220 name: SHARED_VOLUME.to_string(),
2221 mount_path: SHARED_VOL_PATH.to_string(),
2222 read_only: Some(true),
2223 ..Default::default()
2224 },
2225 VolumeMount {
2226 name: "operation".to_string(),
2227 mount_path: "/run/kaniop".to_string(),
2228 read_only: Some(true),
2229 ..Default::default()
2230 },
2231 VolumeMount {
2232 name: "result".to_string(),
2233 mount_path: "/run/kaniop-result".to_string(),
2234 ..Default::default()
2235 },
2236 ];
2237 mounts.extend(build_auth_volume_mounts(auth_method));
2238 if repo.spec.s3.ca_bundle_ref.is_some() {
2239 mounts.push(build_ca_bundle_volume_mount());
2240 }
2241 Some(mounts)
2242 },
2243 ..Default::default()
2244 }],
2245 volumes: {
2246 let mut vols = vec![
2247 Volume {
2248 name: DATA_VOLUME.to_string(),
2249 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2250 claim_name: primary_pvc_name(target)?,
2251 read_only: Some(false),
2252 }),
2253 ..Default::default()
2254 },
2255 Volume {
2256 name: TLS_VOLUME.to_string(),
2257 secret: Some(k8s_openapi::api::core::v1::SecretVolumeSource {
2258 secret_name: Some(target.effective_tls_secret_name()),
2259 ..Default::default()
2260 }),
2261 ..Default::default()
2262 },
2263 Volume {
2264 name: SHARED_VOLUME.to_string(),
2265 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2266 size_limit: Some(crate::controller::backup_job_volume_size()),
2267 ..Default::default()
2268 }),
2269 ..Default::default()
2270 },
2271 Volume {
2272 name: "operation".to_string(),
2273 config_map: Some(ConfigMapVolumeSource {
2274 name: operation_cm_name,
2275 ..Default::default()
2276 }),
2277 ..Default::default()
2278 },
2279 Volume {
2280 name: "result".to_string(),
2281 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2282 size_limit: Some(Quantity("16Mi".to_string())),
2283 ..Default::default()
2284 }),
2285 ..Default::default()
2286 },
2287 ];
2288 vols.extend(build_auth_volumes(auth_method));
2289 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2290 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2291 }
2292 Some(vols)
2293 },
2294 ..Default::default()
2295 }),
2296 },
2297 ..Default::default()
2298 }),
2299 ..Default::default()
2300 };
2301 api.create(&PostParams::default(), &job)
2302 .await
2303 .map(|_| ())
2304 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2305}
2306
2307async fn ensure_source_prep_job(
2308 restore: &KanidmRestore,
2309 target: &Kanidm,
2310 ctx: &RestoreContext,
2311 name: &str,
2312) -> Result<()> {
2313 let ns = restore.namespace().unwrap();
2314 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
2315 if api
2316 .get_opt(name)
2317 .await
2318 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
2319 .is_some()
2320 {
2321 return Ok(());
2322 }
2323 let backup_name = restore
2324 .spec
2325 .source
2326 .backup_ref
2327 .as_ref()
2328 .map(|b| b.name.clone())
2329 .unwrap_or_default();
2330 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2331 .get(&backup_name)
2332 .await
2333 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_name, e))?;
2334 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2335 .get(&backup.spec.repository_ref.name)
2336 .await
2337 .map_err(|e| {
2338 Error::kube_error(
2339 "get",
2340 "KanidmBackupRepository",
2341 &ns,
2342 &backup.spec.repository_ref.name,
2343 e,
2344 )
2345 })?;
2346 let endpoint = &repo.spec.s3.endpoint;
2347 let region = &repo.spec.s3.region;
2348 let operation_doc = build_download_operation_doc(
2349 restore,
2350 target,
2351 &backup.spec.manifest_key,
2352 &backup.spec.backup_id,
2353 &repo.spec.s3.bucket,
2354 &repo.spec.s3.prefix,
2355 endpoint,
2356 region,
2357 repo.spec.s3.force_path_style,
2358 repo.spec.s3.insecure,
2359 repo.spec
2360 .s3
2361 .ca_bundle_ref
2362 .as_deref()
2363 .map(|_| ca_bundle_path())
2364 .as_deref(),
2365 repo.spec.encryption.as_ref(),
2366 );
2367 let operation_cm_name = format!("{}-source-prep-op", restore.name_any());
2368 ensure_operation_configmap(restore, &operation_cm_name, &operation_doc, &ns, ctx).await?;
2369 let auth_method = &repo.spec.authentication.reader;
2370 let data_mover_image = data_mover_image();
2371 let download_script = format!(
2372 r#"set -eu
2373/bin/kaniop-data-mover download --operation-doc /run/kaniop/operation.json
2374RESULT_FILE="/run/kaniop-result/result.json"
2375TERM_MSG="{TERMINATION_MESSAGE_PATH}"
2376if [ ! -f "$RESULT_FILE" ]; then
2377 echo "ERROR: result document not found at $RESULT_FILE"
2378 printf '%.4096s' "result document not found at $RESULT_FILE" > "$TERM_MSG"
2379 exit 1
2380fi
2381if ! grep -q '"success": true' "$RESULT_FILE" || ! grep -q '"exitCode": "success"' "$RESULT_FILE"; then
2382 echo "ERROR: source preparation download reported failure"
2383 dd if="$RESULT_FILE" bs=4000 count=1 2>/dev/null
2384 dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2385 exit 1
2386fi
2387dd if="$RESULT_FILE" of="$TERM_MSG" bs=4096 count=1 2>/dev/null
2388echo "source preparation download completed successfully"
2389"#
2390 );
2391 let job = Job {
2392 metadata: ObjectMeta {
2393 name: Some(name.to_string()),
2394 namespace: Some(ns.clone()),
2395 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2396 ..Default::default()
2397 },
2398 spec: Some(JobSpec {
2399 backoff_limit: Some(0),
2400 template: PodTemplateSpec {
2401 metadata: Some(ObjectMeta::default()),
2402 spec: Some(PodSpec {
2403 automount_service_account_token: Some(false),
2404 restart_policy: Some("Never".to_string()),
2405 security_context: Some(source_job_pod_security_context(target)),
2406 containers: vec![Container {
2407 name: "source-prep".to_string(),
2408 image: Some(data_mover_image),
2409 command: Some(vec!["/bin/sh".to_string()]),
2410 args: Some(vec!["-c".to_string(), download_script]),
2411 security_context: Some(hardened_security_context()),
2412 resources: Some(job_resource_requirements()),
2413 termination_message_path: Some(TERMINATION_MESSAGE_PATH.to_string()),
2414 termination_message_policy: Some("FallbackToLogsOnError".to_string()),
2415 env: {
2416 let mut env_vars = build_auth_env_vars(
2417 auth_method,
2418 &backup.spec.repository_ref.name,
2419 AuthRole::Reader,
2420 );
2421 env_vars.push(EnvVar {
2422 name: "RUST_LOG".to_string(),
2423 value: Some("info".to_string()),
2424 ..Default::default()
2425 });
2426 if repo.spec.s3.ca_bundle_ref.is_some() {
2427 env_vars.push(ca_bundle_env_var());
2428 }
2429 env_vars.extend(build_encryption_env_vars(
2430 repo.spec
2431 .encryption
2432 .as_ref()
2433 .and_then(|e| e.key_ref.as_ref()),
2434 ));
2435 Some(env_vars)
2436 },
2437 volume_mounts: {
2438 let mut mounts = vec![
2439 VolumeMount {
2440 name: DATA_VOLUME.to_string(),
2441 mount_path: DATA_PATH.to_string(),
2442 ..Default::default()
2443 },
2444 VolumeMount {
2445 name: "operation".to_string(),
2446 mount_path: "/run/kaniop".to_string(),
2447 read_only: Some(true),
2448 ..Default::default()
2449 },
2450 VolumeMount {
2451 name: "result".to_string(),
2452 mount_path: "/run/kaniop-result".to_string(),
2453 ..Default::default()
2454 },
2455 ];
2456 mounts.extend(build_auth_volume_mounts(auth_method));
2457 if repo.spec.s3.ca_bundle_ref.is_some() {
2458 mounts.push(build_ca_bundle_volume_mount());
2459 }
2460 Some(mounts)
2461 },
2462 ..Default::default()
2463 }],
2464 volumes: {
2465 let mut vols = vec![
2466 Volume {
2467 name: DATA_VOLUME.to_string(),
2468 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
2469 claim_name: primary_pvc_name(target)?,
2470 read_only: Some(false),
2471 }),
2472 ..Default::default()
2473 },
2474 Volume {
2475 name: "operation".to_string(),
2476 config_map: Some(ConfigMapVolumeSource {
2477 name: operation_cm_name,
2478 ..Default::default()
2479 }),
2480 ..Default::default()
2481 },
2482 Volume {
2483 name: "result".to_string(),
2484 empty_dir: Some(k8s_openapi::api::core::v1::EmptyDirVolumeSource {
2485 size_limit: Some(Quantity("16Mi".to_string())),
2486 ..Default::default()
2487 }),
2488 ..Default::default()
2489 },
2490 ];
2491 vols.extend(build_auth_volumes(auth_method));
2492 if let Some(ca_bundle_ref) = &repo.spec.s3.ca_bundle_ref {
2493 vols.push(build_ca_bundle_volume(ca_bundle_ref));
2494 }
2495 Some(vols)
2496 },
2497 ..Default::default()
2498 }),
2499 },
2500 ..Default::default()
2501 }),
2502 ..Default::default()
2503 };
2504 api.create(&PostParams::default(), &job)
2505 .await
2506 .map(|_| ())
2507 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
2508}
2509
2510#[allow(clippy::too_many_arguments)]
2511fn build_safety_upload_operation_doc(
2512 restore: &KanidmRestore,
2513 target: &Kanidm,
2514 backup_id: &str,
2515 manifest_key: &str,
2516 bucket: &str,
2517 prefix: &str,
2518 endpoint: &str,
2519 region: &str,
2520 force_path_style: bool,
2521 insecure: bool,
2522 ca_bundle_path: Option<&str>,
2523 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2524) -> Result<String> {
2525 let enc_mode =
2526 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2527 let enc_key_id = encryption
2528 .and_then(|e| e.key_id.as_ref())
2529 .map(|s| serde_json::Value::String(s.clone()));
2530 let op = UploadOperation {
2531 payload_path: format!("{SHARED_VOL_PATH}/safety-backup.json.gz"),
2532 bucket: bucket.to_string(),
2533 prefix: prefix.to_string(),
2534 endpoint: endpoint.to_string(),
2535 region: region.to_string(),
2536 force_path_style,
2537 insecure,
2538 ca_bundle_path: ca_bundle_path.map(str::to_string),
2539 backup_id: backup_id.to_string(),
2540 namespace_uid: restore.namespace().unwrap_or_default(),
2541 kanidm_uid: restore.spec.target_ref.uid.clone(),
2542 kanidm_name: restore.spec.target_ref.name.clone(),
2543 domain: target.spec.domain.clone(),
2544 kanidm_version: target
2545 .status
2546 .as_ref()
2547 .and_then(|s| s.version.as_ref())
2548 .map(|v| v.image_tag.clone())
2549 .unwrap_or_default(),
2550 image_digest: None,
2551 consistency: "kanidm-offline".to_string(),
2552 reason: "restore-safety".to_string(),
2553 encryption_mode: enc_mode.and_then(|v| v.as_str().map(String::from)),
2554 encryption_key_id: enc_key_id.and_then(|v| v.as_str().map(String::from)),
2555 result_path: "/run/kaniop-result/result.json".to_string(),
2556 max_concurrent_parts: 4,
2557 max_retries: 3,
2558 };
2559 let _ = manifest_key;
2560 let doc = OperationDocument {
2561 api_version: OPERATION_DOC_VERSION.to_string(),
2562 kind: "OperationDocument".to_string(),
2563 spec: OperationSpec::Upload(op),
2564 };
2565 serde_json::to_string(&doc)
2566 .map_err(|e| Error::MissingData(format!("failed to serialize operation document: {e}")))
2567}
2568
2569async fn validate_backup_ref(
2570 restore: &KanidmRestore,
2571 target: &Kanidm,
2572 ctx: &RestoreContext,
2573) -> Result<()> {
2574 let backup_ref = restore
2575 .spec
2576 .source
2577 .backup_ref
2578 .as_ref()
2579 .ok_or_else(|| Error::MissingData("backupRef source not set".to_string()))?;
2580 let ns = restore.namespace().unwrap();
2581 let backup = Api::<KanidmBackup>::namespaced(ctx.client.clone(), &ns)
2582 .get(&backup_ref.name)
2583 .await
2584 .map_err(|e| Error::kube_error("get", "KanidmBackup", &ns, &backup_ref.name, e))?;
2585 if backup.status.as_ref().map(|s| &s.phase) != Some(&KanidmBackupPhase::Ready) {
2586 return Err(Error::MissingData(format!(
2587 "KanidmBackup '{}' is not Ready (phase: {:?})",
2588 backup_ref.name,
2589 backup.status.as_ref().map(|s| &s.phase)
2590 )));
2591 }
2592 if backup.spec.kanidm_ref.uid != restore.spec.target_ref.uid {
2593 return Err(Error::MissingData(format!(
2594 "KanidmBackup kanidmRef.uid '{}' does not match target UID '{}'",
2595 backup.spec.kanidm_ref.uid, restore.spec.target_ref.uid
2596 )));
2597 }
2598 if backup.spec.kanidm_ref.name != restore.spec.target_ref.name {
2599 return Err(Error::MissingData(format!(
2600 "KanidmBackup kanidmRef.name '{}' does not match target name '{}'",
2601 backup.spec.kanidm_ref.name, restore.spec.target_ref.name
2602 )));
2603 }
2604 let repo = Api::<KanidmBackupRepository>::namespaced(ctx.client.clone(), &ns)
2605 .get(&backup.spec.repository_ref.name)
2606 .await
2607 .map_err(|e| {
2608 Error::kube_error(
2609 "get",
2610 "KanidmBackupRepository",
2611 &ns,
2612 &backup.spec.repository_ref.name,
2613 e,
2614 )
2615 })?;
2616 if !has_accepted_condition(
2617 &repo
2618 .status
2619 .as_ref()
2620 .map(|s| &s.conditions)
2621 .cloned()
2622 .unwrap_or_default(),
2623 ) {
2624 return Err(Error::MissingData(format!(
2625 "KanidmBackupRepository '{}' configuration has not been accepted",
2626 backup.spec.repository_ref.name
2627 )));
2628 }
2629 validate_backup_compatibility(restore, target, &backup)?;
2630 Ok(())
2631}
2632
2633fn has_accepted_condition(conditions: &[Condition]) -> bool {
2634 conditions
2635 .iter()
2636 .any(|c| c.type_ == "Ready" && c.status == "True" && c.reason == "Accepted")
2637}
2638
2639fn validate_backup_compatibility(
2640 restore: &KanidmRestore,
2641 target: &Kanidm,
2642 backup: &KanidmBackup,
2643) -> Result<()> {
2644 let status = backup.status.as_ref();
2645 if let Some(backup_version) = status.and_then(|s| s.kanidm_version.as_ref()) {
2646 let target_version = target
2647 .status
2648 .as_ref()
2649 .and_then(|s| s.version.as_ref())
2650 .map(|v| v.image_tag.as_str());
2651 if let Some(tv) = target_version {
2652 if !tv.is_empty() && !backup_version.is_empty() && tv != backup_version {
2653 return Err(Error::MissingData(format!(
2654 "backup Kanidm version '{backup_version}' does not match target version '{tv}'"
2655 )));
2656 }
2657 }
2658 }
2659 if let Some(backup_digest) = status.and_then(|s| s.image_digest.as_ref()) {
2660 if !backup_digest.is_empty()
2661 && restore.spec.restore_image.contains('@')
2662 && !restore.spec.restore_image.contains(backup_digest)
2663 {
2664 return Err(Error::MissingData(format!(
2665 "restore image digest does not match backup image digest '{backup_digest}'"
2666 )));
2667 }
2668 }
2669 Ok(())
2670}
2671
2672#[allow(clippy::too_many_arguments)]
2673fn build_download_operation_doc(
2674 restore: &KanidmRestore,
2675 target: &Kanidm,
2676 manifest_key: &str,
2677 expected_backup_id: &str,
2678 bucket: &str,
2679 prefix: &str,
2680 endpoint: &str,
2681 region: &str,
2682 force_path_style: bool,
2683 insecure: bool,
2684 ca_bundle_path: Option<&str>,
2685 encryption: Option<&kaniop_backup_core::crd::RepositoryEncryption>,
2686) -> String {
2687 let enc_mode =
2688 encryption.map(|e| serde_json::to_value(&e.mode).unwrap_or(serde_json::Value::Null));
2689 let enc_key_id = encryption
2690 .and_then(|e| e.key_id.as_ref())
2691 .map(|s| serde_json::Value::String(s.clone()));
2692 serde_json::json!({
2693 "apiVersion": "backup.kaniop.rs/v1alpha1",
2694 "kind": "OperationDocument",
2695 "operation": "download",
2696 "manifestKey": manifest_key,
2697 "bucket": bucket,
2698 "prefix": prefix,
2699 "endpoint": endpoint,
2700 "region": region,
2701 "forcePathStyle": force_path_style,
2702 "insecure": insecure,
2703 "caBundlePath": ca_bundle_path,
2704 "expectedBackupId": expected_backup_id,
2705 "expectedKanidmUid": restore.spec.target_ref.uid,
2706 "expectedDomain": target.spec.domain,
2707 "outputPath": format!("{DATA_PATH}/source-payload.json.gz"),
2708 "resultPath": "/run/kaniop-result/result.json",
2709 "maxRetries": 3,
2710 "encryptionMode": enc_mode,
2711 "encryptionKeyId": enc_key_id,
2712 })
2713 .to_string()
2714}
2715
2716async fn ensure_operation_configmap(
2717 restore: &KanidmRestore,
2718 name: &str,
2719 data: &str,
2720 ns: &str,
2721 ctx: &RestoreContext,
2722) -> Result<()> {
2723 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), ns);
2724 if api
2725 .get_opt(name)
2726 .await
2727 .map_err(|e| Error::kube_error("get", "ConfigMap", ns, name, e))?
2728 .is_some()
2729 {
2730 return Ok(());
2731 }
2732 let cm = ConfigMap {
2733 metadata: ObjectMeta {
2734 name: Some(name.to_string()),
2735 namespace: Some(ns.to_string()),
2736 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
2737 ..Default::default()
2738 },
2739 data: Some(std::collections::BTreeMap::from([(
2740 "operation.json".to_string(),
2741 data.to_string(),
2742 )])),
2743 ..Default::default()
2744 };
2745 api.create(&PostParams::default(), &cm)
2746 .await
2747 .map(|_| ())
2748 .map_err(|e| Error::kube_error("create", "ConfigMap", ns, name, e))
2749}
2750
2751#[cfg(test)]
2752mod tests {
2753 use super::{
2754 BREAK_GLASS_APPROVED_BY_ANNOTATION, BREAK_GLASS_REASON_ANNOTATION, CONDITION_BREAK_GLASS,
2755 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestore, KanidmRestorePhase,
2756 KanidmRestoreSource, KanidmRestoreSpec, KanidmRestoreStatus, KanidmRestoreTargetRef,
2757 ReplicaCountEntry, SafetyBackupConfig, SafetyBackupRepositoryRef,
2758 hardened_security_context, has_accepted_condition, is_remote_source,
2759 kanidm_job_security_context, mutable_image, requires_safety_backup, restore_job_name,
2760 safe_basename, validate_safety_backup_config, validate_source, verify_job_name,
2761 };
2762 use k8s_openapi::apimachinery::pkg::api::resource::Quantity;
2763 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition;
2764 use kube::api::ObjectMeta as ApiObjectMeta;
2765 use std::collections::BTreeMap;
2766
2767 fn make_restore(
2768 source: KanidmRestoreSource,
2769 safety: Option<SafetyBackupConfig>,
2770 ) -> KanidmRestore {
2771 KanidmRestore {
2772 metadata: ApiObjectMeta {
2773 name: Some("test-restore".to_string()),
2774 namespace: Some("default".to_string()),
2775 uid: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()),
2776 ..Default::default()
2777 },
2778 spec: KanidmRestoreSpec {
2779 target_ref: KanidmRestoreTargetRef {
2780 name: "corp-idm".to_string(),
2781 uid: "test-uid-123".to_string(),
2782 },
2783 source,
2784 restore_image: "kanidm/server@sha256:abc".to_string(),
2785 safety_backup: safety,
2786 },
2787 status: None,
2788 }
2789 }
2790
2791 fn kanidm_with_version(image_tag: &str) -> super::super::crd::Kanidm {
2792 super::super::crd::Kanidm {
2793 status: Some(super::super::crd::KanidmStatus {
2794 available_replicas: 1,
2795 replicas: 1,
2796 unavailable_replicas: 0,
2797 updated_replicas: 1,
2798 replica_statuses: vec![],
2799 replica_column: "1/1".to_string(),
2800 secret_name: None,
2801 version: Some(super::super::crd::KanidmVersionStatus {
2802 image_tag: image_tag.to_string(),
2803 upgrade_check_result: super::super::crd::KanidmUpgradeCheckResult::Passed,
2804 compatibility_result: super::super::crd::VersionCompatibilityResult::Compatible,
2805 }),
2806 domain_appearance_image: None,
2807 mail_sender: None,
2808 conditions: None,
2809 }),
2810 ..Default::default()
2811 }
2812 }
2813
2814 #[test]
2815 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
2816 assert!(!KanidmRestoreStatus::default().database_mutation_started);
2817 }
2818
2819 #[test]
2820 fn completed_restore_sets_ready_condition() {
2821 let mut status = KanidmRestoreStatus {
2822 phase: KanidmRestorePhase::Completed,
2823 ..Default::default()
2824 };
2825 super::update_restore_conditions(&mut status, Some(7));
2826 assert!(status.conditions.iter().any(|condition| {
2827 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
2828 }));
2829 assert!(!status.conditions.iter().any(|condition| {
2830 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2831 }));
2832 }
2833
2834 #[test]
2835 fn failed_restore_sets_failed_condition() {
2836 let mut status = KanidmRestoreStatus {
2837 phase: KanidmRestorePhase::Failed,
2838 message: Some("verification failed".to_string()),
2839 ..Default::default()
2840 };
2841 super::update_restore_conditions(&mut status, Some(8));
2842 assert!(status.conditions.iter().any(|condition| {
2843 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
2844 }));
2845 }
2846
2847 #[test]
2848 fn restore_filename_is_confined_to_backup_directory() {
2849 assert!(safe_basename("backup.json.gz"));
2850 assert!(!safe_basename("../backup.json.gz"));
2851 assert!(!safe_basename("a/b"));
2852 assert!(!safe_basename(""));
2853 }
2854
2855 #[test]
2856 fn restore_image_rejects_latest_and_unpinned_names() {
2857 assert!(mutable_image("kanidm/server:latest"));
2858 assert!(mutable_image("kanidm/server"));
2859 assert!(!mutable_image("kanidm/server:1.10.0"));
2860 assert!(!mutable_image("kanidm/server@sha256:abc"));
2861 }
2862
2863 #[test]
2864 fn source_validation_rejects_both_local_and_backup_ref() {
2865 let source = KanidmRestoreSource {
2866 local: Some(super::KanidmRestoreLocalSource {
2867 file_name: "backup.json".to_string(),
2868 }),
2869 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2870 name: "some-backup".to_string(),
2871 }),
2872 };
2873 let restore = make_restore(source, None);
2874 assert!(validate_source(&restore).is_err());
2875 }
2876
2877 #[test]
2878 fn source_validation_rejects_empty_source() {
2879 let source = KanidmRestoreSource {
2880 local: None,
2881 backup_ref: None,
2882 };
2883 let restore = make_restore(source, None);
2884 assert!(validate_source(&restore).is_err());
2885 }
2886
2887 #[test]
2888 fn source_validation_accepts_local_source() {
2889 let source = KanidmRestoreSource {
2890 local: Some(super::KanidmRestoreLocalSource {
2891 file_name: "backup.json".to_string(),
2892 }),
2893 backup_ref: None,
2894 };
2895 let restore = make_restore(source, None);
2896 assert!(validate_source(&restore).is_ok());
2897 }
2898
2899 #[test]
2900 fn source_validation_accepts_backup_ref_source() {
2901 let source = KanidmRestoreSource {
2902 local: None,
2903 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2904 name: "corp-idm-019c7c76".to_string(),
2905 }),
2906 };
2907 let restore = make_restore(source, None);
2908 assert!(validate_source(&restore).is_ok());
2909 }
2910
2911 #[test]
2912 fn source_validation_rejects_empty_backup_ref_name() {
2913 let source = KanidmRestoreSource {
2914 local: None,
2915 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2916 name: "".to_string(),
2917 }),
2918 };
2919 let restore = make_restore(source, None);
2920 assert!(validate_source(&restore).is_err());
2921 }
2922
2923 #[test]
2924 fn is_remote_source_true_for_backup_ref() {
2925 let source = KanidmRestoreSource {
2926 local: None,
2927 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2928 name: "backup-1".to_string(),
2929 }),
2930 };
2931 let restore = make_restore(source, None);
2932 assert!(is_remote_source(&restore));
2933 }
2934
2935 #[test]
2936 fn is_remote_source_false_for_local() {
2937 let source = KanidmRestoreSource {
2938 local: Some(super::KanidmRestoreLocalSource {
2939 file_name: "backup.json".to_string(),
2940 }),
2941 backup_ref: None,
2942 };
2943 let restore = make_restore(source, None);
2944 assert!(!is_remote_source(&restore));
2945 }
2946
2947 #[test]
2948 fn requires_safety_backup_default_true() {
2949 let source = KanidmRestoreSource {
2950 local: None,
2951 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2952 name: "backup-1".to_string(),
2953 }),
2954 };
2955 let restore = make_restore(source, None);
2956 assert!(requires_safety_backup(&restore));
2957 }
2958
2959 #[test]
2960 fn requires_safety_backup_false_when_skip() {
2961 let source = KanidmRestoreSource {
2962 local: None,
2963 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2964 name: "backup-1".to_string(),
2965 }),
2966 };
2967 let safety = SafetyBackupConfig {
2968 repository_ref: Some(SafetyBackupRepositoryRef {
2969 name: "offsite".to_string(),
2970 }),
2971 skip: true,
2972 };
2973 let restore = make_restore(source, Some(safety));
2974 assert!(!requires_safety_backup(&restore));
2975 }
2976
2977 #[test]
2978 fn break_glass_validation_rejects_skip_without_reason() {
2979 let source = KanidmRestoreSource {
2980 local: None,
2981 backup_ref: Some(super::KanidmRestoreBackupRefSource {
2982 name: "backup-1".to_string(),
2983 }),
2984 };
2985 let safety = SafetyBackupConfig {
2986 repository_ref: Some(SafetyBackupRepositoryRef {
2987 name: "offsite".to_string(),
2988 }),
2989 skip: true,
2990 };
2991 let mut restore = make_restore(source, Some(safety));
2992 restore.metadata.annotations = Some(BTreeMap::from([(
2993 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
2994 "commander@example.com".to_string(),
2995 )]));
2996 assert!(validate_safety_backup_config(&restore).is_err());
2997 }
2998
2999 #[test]
3000 fn break_glass_validation_rejects_skip_without_approver() {
3001 let source = KanidmRestoreSource {
3002 local: None,
3003 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3004 name: "backup-1".to_string(),
3005 }),
3006 };
3007 let safety = SafetyBackupConfig {
3008 repository_ref: Some(SafetyBackupRepositoryRef {
3009 name: "offsite".to_string(),
3010 }),
3011 skip: true,
3012 };
3013 let mut restore = make_restore(source, Some(safety));
3014 restore.metadata.annotations = Some(BTreeMap::from([(
3015 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3016 "PVC is unreadable".to_string(),
3017 )]));
3018 assert!(validate_safety_backup_config(&restore).is_err());
3019 }
3020
3021 #[test]
3022 fn break_glass_validation_accepts_valid_annotations() {
3023 let source = KanidmRestoreSource {
3024 local: None,
3025 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3026 name: "backup-1".to_string(),
3027 }),
3028 };
3029 let safety = SafetyBackupConfig {
3030 repository_ref: Some(SafetyBackupRepositoryRef {
3031 name: "offsite".to_string(),
3032 }),
3033 skip: true,
3034 };
3035 let mut restore = make_restore(source, Some(safety));
3036 restore.metadata.annotations = Some(BTreeMap::from([
3037 (
3038 BREAK_GLASS_REASON_ANNOTATION.to_string(),
3039 "PVC is unreadable".to_string(),
3040 ),
3041 (
3042 BREAK_GLASS_APPROVED_BY_ANNOTATION.to_string(),
3043 "commander@example.com".to_string(),
3044 ),
3045 ]));
3046 assert!(validate_safety_backup_config(&restore).is_ok());
3047 }
3048
3049 #[test]
3050 fn remote_source_requires_repository_ref_when_not_skipping() {
3051 let source = KanidmRestoreSource {
3052 local: None,
3053 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3054 name: "backup-1".to_string(),
3055 }),
3056 };
3057 let safety = SafetyBackupConfig {
3058 repository_ref: None,
3059 skip: false,
3060 };
3061 let restore = make_restore(source, Some(safety));
3062 assert!(validate_safety_backup_config(&restore).is_err());
3063 }
3064
3065 #[test]
3066 fn remote_source_accepts_repository_ref() {
3067 let source = KanidmRestoreSource {
3068 local: None,
3069 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3070 name: "backup-1".to_string(),
3071 }),
3072 };
3073 let safety = SafetyBackupConfig {
3074 repository_ref: Some(SafetyBackupRepositoryRef {
3075 name: "offsite".to_string(),
3076 }),
3077 skip: false,
3078 };
3079 let restore = make_restore(source, Some(safety));
3080 assert!(validate_safety_backup_config(&restore).is_ok());
3081 }
3082
3083 #[test]
3084 fn phase_ordering_includes_safety_backup_and_preparing_source() {
3085 let phases = vec![
3086 KanidmRestorePhase::Pending,
3087 KanidmRestorePhase::Validating,
3088 KanidmRestorePhase::Quiescing,
3089 KanidmRestorePhase::SafetyBackup,
3090 KanidmRestorePhase::PreparingSource,
3091 KanidmRestorePhase::RestoringPrimary,
3092 KanidmRestorePhase::Verifying,
3093 KanidmRestorePhase::RebuildingReplicas,
3094 KanidmRestorePhase::Resuming,
3095 KanidmRestorePhase::Completed,
3096 ];
3097 assert_eq!(phases.len(), 10);
3098 assert_eq!(phases[3], KanidmRestorePhase::SafetyBackup);
3099 assert_eq!(phases[4], KanidmRestorePhase::PreparingSource);
3100 }
3101
3102 #[test]
3103 fn status_default_has_empty_phase_timestamps() {
3104 let status = KanidmRestoreStatus::default();
3105 assert!(status.phase_timestamps.is_empty());
3106 }
3107
3108 #[test]
3109 fn status_default_has_empty_original_replicas() {
3110 let status = KanidmRestoreStatus::default();
3111 assert!(status.original_replicas.is_empty());
3112 }
3113
3114 #[test]
3115 fn source_job_security_context_inherits_target_identity_and_volume_ownership() {
3116 let mut target = super::super::crd::Kanidm::default();
3117 target.spec.security_context = Some(k8s_openapi::api::core::v1::PodSecurityContext {
3118 run_as_non_root: Some(false),
3119 run_as_user: Some(389),
3120 run_as_group: Some(389),
3121 fs_group: Some(389),
3122 fs_group_change_policy: Some("OnRootMismatch".to_string()),
3123 supplemental_groups: Some(vec![390, 391]),
3124 ..Default::default()
3125 });
3126
3127 let ctx = super::source_job_pod_security_context(&target);
3128
3129 assert_eq!(ctx.run_as_non_root, Some(false));
3130 assert_eq!(ctx.run_as_user, Some(389));
3131 assert_eq!(ctx.run_as_group, Some(389));
3132 assert_eq!(ctx.fs_group, Some(389));
3133 assert_eq!(
3134 ctx.fs_group_change_policy.as_deref(),
3135 Some("OnRootMismatch")
3136 );
3137 assert_eq!(ctx.supplemental_groups, Some(vec![390, 391]));
3138 assert_eq!(
3139 ctx.seccomp_profile
3140 .as_ref()
3141 .map(|profile| profile.type_.as_str()),
3142 Some("RuntimeDefault")
3143 );
3144 }
3145
3146 #[test]
3147 fn hardened_security_context_drops_all_capabilities() {
3148 let ctx = hardened_security_context();
3149 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3150 assert_eq!(ctx.read_only_root_filesystem, Some(true));
3151 assert_eq!(ctx.run_as_non_root, Some(true));
3152 let caps = ctx.capabilities.unwrap();
3153 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3154 assert_eq!(
3155 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3156 Some("RuntimeDefault")
3157 );
3158 }
3159
3160 #[test]
3161 fn kanidm_job_security_context_drops_caps_without_run_as_non_root() {
3162 let ctx = kanidm_job_security_context();
3163 assert_eq!(ctx.allow_privilege_escalation, Some(false));
3164 assert!(ctx.run_as_non_root.is_none());
3165 assert!(ctx.read_only_root_filesystem.is_none());
3166 let caps = ctx.capabilities.unwrap();
3167 assert_eq!(caps.drop, Some(vec!["ALL".to_string()]));
3168 assert_eq!(
3169 ctx.seccomp_profile.as_ref().map(|s| s.type_.as_str()),
3170 Some("RuntimeDefault")
3171 );
3172 }
3173
3174 #[test]
3175 fn safety_backup_job_name_is_deterministic() {
3176 let source = KanidmRestoreSource {
3177 local: Some(super::KanidmRestoreLocalSource {
3178 file_name: "backup.json".to_string(),
3179 }),
3180 backup_ref: None,
3181 };
3182 let restore = make_restore(source, None);
3183 let name1 = super::safety_backup_job_name(&restore);
3184 let name2 = super::safety_backup_job_name(&restore);
3185 assert_eq!(name1, name2);
3186 assert_eq!(name1, "test-restore-safety-backup");
3187 }
3188
3189 #[test]
3190 fn source_prep_job_name_is_deterministic() {
3191 let source = KanidmRestoreSource {
3192 local: None,
3193 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3194 name: "backup-1".to_string(),
3195 }),
3196 };
3197 let restore = make_restore(source, None);
3198 let name = super::source_prep_job_name(&restore);
3199 assert_eq!(name, "test-restore-source-prep");
3200 }
3201
3202 #[test]
3203 fn break_glass_condition_is_recorded_in_status() {
3204 let mut status = KanidmRestoreStatus::default();
3205 let condition = super::restore_condition(
3206 &status.conditions,
3207 CONDITION_BREAK_GLASS,
3208 CONDITION_TRUE,
3209 "BreakGlassOverride",
3210 "Safety backup skipped",
3211 Some(1),
3212 );
3213 status.conditions.push(condition);
3214 assert!(status.conditions.iter().any(|c| {
3215 c.type_ == CONDITION_BREAK_GLASS
3216 && c.status == CONDITION_TRUE
3217 && c.reason == "BreakGlassOverride"
3218 }));
3219 }
3220
3221 #[test]
3222 fn replica_count_entry_serialization() {
3223 let entry = ReplicaCountEntry {
3224 group: "primary".to_string(),
3225 replicas: 3,
3226 };
3227 let json = serde_json::to_string(&entry).unwrap();
3228 assert!(json.contains("\"group\":\"primary\""));
3229 assert!(json.contains("\"replicas\":3"));
3230 }
3231
3232 #[test]
3233 fn local_source_requires_repository_ref_when_not_skipping() {
3234 let source = KanidmRestoreSource {
3235 local: Some(super::KanidmRestoreLocalSource {
3236 file_name: "backup.json".to_string(),
3237 }),
3238 backup_ref: None,
3239 };
3240 let safety = SafetyBackupConfig {
3241 repository_ref: None,
3242 skip: false,
3243 };
3244 let restore = make_restore(source, Some(safety));
3245 assert!(validate_safety_backup_config(&restore).is_err());
3246 }
3247
3248 #[test]
3249 fn local_source_accepts_repository_ref_when_not_skipping() {
3250 let source = KanidmRestoreSource {
3251 local: Some(super::KanidmRestoreLocalSource {
3252 file_name: "backup.json".to_string(),
3253 }),
3254 backup_ref: None,
3255 };
3256 let safety = SafetyBackupConfig {
3257 repository_ref: Some(SafetyBackupRepositoryRef {
3258 name: "offsite".to_string(),
3259 }),
3260 skip: false,
3261 };
3262 let restore = make_restore(source, Some(safety));
3263 assert!(validate_safety_backup_config(&restore).is_ok());
3264 }
3265
3266 #[test]
3267 fn skip_without_break_glass_annotations_is_rejected() {
3268 let source = KanidmRestoreSource {
3269 local: Some(super::KanidmRestoreLocalSource {
3270 file_name: "backup.json".to_string(),
3271 }),
3272 backup_ref: None,
3273 };
3274 let safety = SafetyBackupConfig {
3275 repository_ref: None,
3276 skip: true,
3277 };
3278 let restore = make_restore(source, Some(safety));
3279 assert!(validate_safety_backup_config(&restore).is_err());
3280 }
3281
3282 #[test]
3283 fn has_accepted_condition_true() {
3284 let conditions = vec![Condition {
3285 type_: "Ready".to_string(),
3286 status: "True".to_string(),
3287 reason: "Accepted".to_string(),
3288 message: String::new(),
3289 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3290 k8s_openapi::jiff::Timestamp::now(),
3291 ),
3292 observed_generation: None,
3293 }];
3294 assert!(has_accepted_condition(&conditions));
3295 }
3296
3297 #[test]
3298 fn has_accepted_condition_false_when_not_accepted_reason() {
3299 let conditions = vec![Condition {
3300 type_: "Ready".to_string(),
3301 status: "True".to_string(),
3302 reason: "SomeOtherReason".to_string(),
3303 message: String::new(),
3304 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3305 k8s_openapi::jiff::Timestamp::now(),
3306 ),
3307 observed_generation: None,
3308 }];
3309 assert!(!has_accepted_condition(&conditions));
3310 }
3311
3312 #[test]
3313 fn has_accepted_condition_false_when_status_false() {
3314 let conditions = vec![Condition {
3315 type_: "Ready".to_string(),
3316 status: "False".to_string(),
3317 reason: "Accepted".to_string(),
3318 message: String::new(),
3319 last_transition_time: k8s_openapi::apimachinery::pkg::apis::meta::v1::Time(
3320 k8s_openapi::jiff::Timestamp::now(),
3321 ),
3322 observed_generation: None,
3323 }];
3324 assert!(!has_accepted_condition(&conditions));
3325 }
3326
3327 #[test]
3328 fn has_accepted_condition_empty() {
3329 assert!(!has_accepted_condition(&[]));
3330 }
3331
3332 #[test]
3333 fn safety_backup_upload_operation_doc_is_valid() {
3334 let source = KanidmRestoreSource {
3335 local: None,
3336 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3337 name: "backup-1".to_string(),
3338 }),
3339 };
3340 let mut restore = make_restore(source, None);
3341 restore.metadata.namespace = Some("test-ns".to_string());
3342 let target = super::super::crd::Kanidm::default();
3343 let doc_str = super::build_safety_upload_operation_doc(
3344 &restore,
3345 &target,
3346 "test-backup-id",
3347 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3348 "my-bucket",
3349 "prod",
3350 "https://s3.example.com",
3351 "us-east-1",
3352 false,
3353 false,
3354 None,
3355 None,
3356 )
3357 .unwrap();
3358 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3359 assert_eq!(parsed["apiVersion"], "backup.kaniop.rs/v1alpha1");
3360 assert_eq!(parsed["kind"], "OperationDocument");
3361 assert_eq!(parsed["operation"], "upload");
3362 assert_eq!(parsed["bucket"], "my-bucket");
3363 assert_eq!(parsed["prefix"], "prod");
3364 assert_eq!(parsed["endpoint"], "https://s3.example.com");
3365 assert_eq!(parsed["region"], "us-east-1");
3366 assert_eq!(parsed["backupId"], "test-backup-id");
3367 assert_eq!(parsed["consistency"], "kanidm-offline");
3368 assert_eq!(parsed["reason"], "restore-safety");
3369 assert_eq!(
3370 parsed["payloadPath"],
3371 format!("{}/safety-backup.json.gz", super::SHARED_VOL_PATH)
3372 );
3373 assert_eq!(parsed["resultPath"], "/run/kaniop-result/result.json");
3374 }
3375
3376 #[test]
3377 fn download_operation_doc_uses_real_repository_data() {
3378 let source = KanidmRestoreSource {
3379 local: None,
3380 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3381 name: "backup-1".to_string(),
3382 }),
3383 };
3384 let restore = make_restore(source, None);
3385 let target = super::super::crd::Kanidm::default();
3386 let doc_str = super::build_download_operation_doc(
3387 &restore,
3388 &target,
3389 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3390 "019c7c76-f423-7a12-8f41-2bea7588a303",
3391 "real-bucket",
3392 "real-prefix",
3393 "https://real-endpoint.com",
3394 "eu-west-1",
3395 true,
3396 false,
3397 None,
3398 None,
3399 );
3400 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3401 assert_eq!(parsed["bucket"], "real-bucket");
3402 assert_eq!(parsed["prefix"], "real-prefix");
3403 assert_eq!(parsed["endpoint"], "https://real-endpoint.com");
3404 assert_eq!(parsed["region"], "eu-west-1");
3405 assert_eq!(parsed["forcePathStyle"], true);
3406 assert_eq!(
3407 parsed["manifestKey"],
3408 "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3409 );
3410 assert_eq!(
3411 parsed["expectedBackupId"],
3412 "019c7c76-f423-7a12-8f41-2bea7588a303"
3413 );
3414 }
3415
3416 #[test]
3417 fn safety_backup_ref_is_set_after_verification() {
3418 let status = KanidmRestoreStatus {
3419 phase: KanidmRestorePhase::SafetyBackup,
3420 safety_backup_job_name: Some("test-safety-backup".to_string()),
3421 safety_backup_ref: Some("safety-test-restore".to_string()),
3422 safety_backup_expected_backup_id: Some(
3423 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3424 ),
3425 safety_backup_manifest_key: Some(
3426 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3427 ),
3428 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3429 ..Default::default()
3430 };
3431 assert!(status.safety_backup_ref.is_some());
3432 assert!(
3433 status
3434 .safety_backup_ref
3435 .as_ref()
3436 .unwrap()
3437 .starts_with("safety-")
3438 );
3439 assert!(status.safety_backup_expected_backup_id.is_some());
3440 assert!(status.safety_backup_manifest_key.is_some());
3441 assert!(status.safety_backup_payload_sha256.is_some());
3442 }
3443
3444 #[test]
3445 fn database_mutation_not_started_without_safety_backup_ref() {
3446 let status = KanidmRestoreStatus {
3447 phase: KanidmRestorePhase::PreparingSource,
3448 safety_backup_ref: None,
3449 database_mutation_started: false,
3450 ..Default::default()
3451 };
3452 assert!(!status.database_mutation_started);
3453 assert!(status.safety_backup_ref.is_none());
3454 }
3455
3456 #[test]
3457 fn pre_boundary_failure_does_not_set_mutation_started() {
3458 let status = KanidmRestoreStatus {
3459 phase: KanidmRestorePhase::Failed,
3460 message: Some(
3461 "safety backup job failed; target restored to original state".to_string(),
3462 ),
3463 database_mutation_started: false,
3464 ..Default::default()
3465 };
3466 assert!(!status.database_mutation_started);
3467 }
3468
3469 #[test]
3470 fn shared_volume_constants_are_distinct() {
3471 assert_ne!(super::SHARED_VOLUME, super::DATA_VOLUME);
3472 assert_ne!(super::SHARED_VOL_PATH, super::DATA_PATH);
3473 }
3474
3475 #[test]
3476 fn validate_backup_compatibility_matching_versions_passes() {
3477 let source = KanidmRestoreSource {
3478 local: None,
3479 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3480 name: "backup-1".to_string(),
3481 }),
3482 };
3483 let restore = make_restore(source, None);
3484 let target = kanidm_with_version("1.10.4");
3485 let backup = kaniop_backup_core::crd::KanidmBackup {
3486 metadata: kube::api::ObjectMeta::default(),
3487 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3488 backup_id: "id".to_string(),
3489 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3490 name: "k".to_string(),
3491 uid: "uid".to_string(),
3492 },
3493 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3494 name: "repo".to_string(),
3495 },
3496 manifest_key: "key".to_string(),
3497 },
3498 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3499 kanidm_version: Some("1.10.4".to_string()),
3500 ..Default::default()
3501 }),
3502 };
3503 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3504 }
3505
3506 #[test]
3507 fn validate_backup_compatibility_version_mismatch_fails() {
3508 let source = KanidmRestoreSource {
3509 local: None,
3510 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3511 name: "backup-1".to_string(),
3512 }),
3513 };
3514 let restore = make_restore(source, None);
3515 let target = kanidm_with_version("1.11.0");
3516 let backup = kaniop_backup_core::crd::KanidmBackup {
3517 metadata: kube::api::ObjectMeta::default(),
3518 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3519 backup_id: "id".to_string(),
3520 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3521 name: "k".to_string(),
3522 uid: "uid".to_string(),
3523 },
3524 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3525 name: "repo".to_string(),
3526 },
3527 manifest_key: "key".to_string(),
3528 },
3529 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3530 kanidm_version: Some("1.10.4".to_string()),
3531 ..Default::default()
3532 }),
3533 };
3534 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3535 }
3536
3537 #[test]
3538 fn validate_backup_compatibility_no_backup_version_passes() {
3539 let source = KanidmRestoreSource {
3540 local: None,
3541 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3542 name: "backup-1".to_string(),
3543 }),
3544 };
3545 let restore = make_restore(source, None);
3546 let target = super::super::crd::Kanidm::default();
3547 let backup = kaniop_backup_core::crd::KanidmBackup {
3548 metadata: kube::api::ObjectMeta::default(),
3549 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3550 backup_id: "id".to_string(),
3551 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3552 name: "k".to_string(),
3553 uid: "uid".to_string(),
3554 },
3555 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3556 name: "repo".to_string(),
3557 },
3558 manifest_key: "key".to_string(),
3559 },
3560 status: None,
3561 };
3562 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3563 }
3564
3565 #[test]
3566 fn validate_backup_compatibility_image_digest_mismatch_fails() {
3567 let source = KanidmRestoreSource {
3568 local: None,
3569 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3570 name: "backup-1".to_string(),
3571 }),
3572 };
3573 let mut restore = make_restore(source, None);
3574 restore.spec.restore_image = "kanidm/server@sha256:differentdigest".to_string();
3575 let target = super::super::crd::Kanidm::default();
3576 let backup = kaniop_backup_core::crd::KanidmBackup {
3577 metadata: kube::api::ObjectMeta::default(),
3578 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3579 backup_id: "id".to_string(),
3580 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3581 name: "k".to_string(),
3582 uid: "uid".to_string(),
3583 },
3584 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3585 name: "repo".to_string(),
3586 },
3587 manifest_key: "key".to_string(),
3588 },
3589 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3590 image_digest: Some("sha256:originaldigest".to_string()),
3591 ..Default::default()
3592 }),
3593 };
3594 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_err());
3595 }
3596
3597 #[test]
3598 fn validate_backup_compatibility_image_digest_match_passes() {
3599 let source = KanidmRestoreSource {
3600 local: None,
3601 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3602 name: "backup-1".to_string(),
3603 }),
3604 };
3605 let mut restore = make_restore(source, None);
3606 restore.spec.restore_image = "kanidm/server@sha256:abc123".to_string();
3607 let target = super::super::crd::Kanidm::default();
3608 let backup = kaniop_backup_core::crd::KanidmBackup {
3609 metadata: kube::api::ObjectMeta::default(),
3610 spec: kaniop_backup_core::crd::KanidmBackupSpec {
3611 backup_id: "id".to_string(),
3612 kanidm_ref: kaniop_backup_core::crd::BackupKanidmRef {
3613 name: "k".to_string(),
3614 uid: "uid".to_string(),
3615 },
3616 repository_ref: kaniop_backup_core::crd::BackupRepositoryRef {
3617 name: "repo".to_string(),
3618 },
3619 manifest_key: "key".to_string(),
3620 },
3621 status: Some(kaniop_backup_core::crd::KanidmBackupStatus {
3622 image_digest: Some("sha256:abc123".to_string()),
3623 ..Default::default()
3624 }),
3625 };
3626 assert!(super::validate_backup_compatibility(&restore, &target, &backup).is_ok());
3627 }
3628
3629 #[test]
3630 fn safety_backup_id_is_deterministic() {
3631 let source = KanidmRestoreSource {
3632 local: None,
3633 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3634 name: "backup-1".to_string(),
3635 }),
3636 };
3637 let restore = make_restore(source, None);
3638 let id1 = super::compute_safety_backup_id(&restore).unwrap();
3639 let id2 = super::compute_safety_backup_id(&restore).unwrap();
3640 assert_eq!(id1, id2);
3641 assert!(uuid::Uuid::parse_str(&id1).is_ok());
3642 }
3643
3644 #[test]
3645 fn safety_manifest_key_is_deterministic() {
3646 let source = KanidmRestoreSource {
3647 local: None,
3648 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3649 name: "backup-1".to_string(),
3650 }),
3651 };
3652 let restore = make_restore(source, None);
3653 let key1 = super::compute_safety_manifest_key(&restore).unwrap();
3654 let key2 = super::compute_safety_manifest_key(&restore).unwrap();
3655 assert_eq!(key1, key2);
3656 assert!(key1.contains("manifest.json"));
3657 assert!(key1.starts_with("v1/tenants/"));
3658 }
3659
3660 #[test]
3661 fn safety_backup_id_survives_restart() {
3662 let source = KanidmRestoreSource {
3663 local: None,
3664 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3665 name: "backup-1".to_string(),
3666 }),
3667 };
3668 let mut restore = make_restore(source, None);
3669 let original_id = super::compute_safety_backup_id(&restore).unwrap();
3670 restore.status = Some(KanidmRestoreStatus {
3671 phase: KanidmRestorePhase::SafetyBackup,
3672 safety_backup_expected_backup_id: Some(original_id.clone()),
3673 ..Default::default()
3674 });
3675 let recomputed_id = super::compute_safety_backup_id(&restore).unwrap();
3676 assert_eq!(original_id, recomputed_id);
3677 }
3678
3679 #[test]
3680 fn safety_backup_id_derived_from_uid_not_name() {
3681 let source = KanidmRestoreSource {
3682 local: None,
3683 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3684 name: "backup-1".to_string(),
3685 }),
3686 };
3687 let restore1 = make_restore(source.clone(), None);
3688 let mut restore2 = make_restore(source, None);
3689 restore2.metadata.name = Some("different-name-same-uid".to_string());
3690 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3691 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3692 assert_eq!(id1, id2);
3693 }
3694
3695 #[test]
3696 fn safety_backup_id_differs_for_different_uids() {
3697 let source = KanidmRestoreSource {
3698 local: None,
3699 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3700 name: "backup-1".to_string(),
3701 }),
3702 };
3703 let restore1 = make_restore(source.clone(), None);
3704 let mut restore2 = make_restore(source, None);
3705 restore2.metadata.uid = Some("ffffffff-eeee-dddd-cccc-bbbbbbbbbbbb".to_string());
3706 let id1 = super::compute_safety_backup_id(&restore1).unwrap();
3707 let id2 = super::compute_safety_backup_id(&restore2).unwrap();
3708 assert_ne!(id1, id2);
3709 }
3710
3711 #[test]
3712 fn safety_backup_id_errors_without_uid() {
3713 let source = KanidmRestoreSource {
3714 local: None,
3715 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3716 name: "backup-1".to_string(),
3717 }),
3718 };
3719 let mut restore = make_restore(source, None);
3720 restore.metadata.uid = None;
3721 assert!(super::compute_safety_backup_id(&restore).is_err());
3722 }
3723
3724 #[test]
3725 fn mutation_boundary_requires_safety_backup_ref_when_safety_required() {
3726 let source = KanidmRestoreSource {
3727 local: None,
3728 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3729 name: "backup-1".to_string(),
3730 }),
3731 };
3732 let restore = make_restore(source, None);
3733 assert!(super::requires_safety_backup(&restore));
3734 let status_without_ref = KanidmRestoreStatus {
3735 phase: KanidmRestorePhase::RestoringPrimary,
3736 safety_backup_ref: None,
3737 database_mutation_started: false,
3738 ..Default::default()
3739 };
3740 assert!(status_without_ref.safety_backup_ref.is_none());
3741 assert!(!status_without_ref.database_mutation_started);
3742 }
3743
3744 #[test]
3745 fn mutation_boundary_allows_progression_when_safety_ref_set() {
3746 let status_with_ref = KanidmRestoreStatus {
3747 phase: KanidmRestorePhase::RestoringPrimary,
3748 safety_backup_ref: Some("safety-test-restore".to_string()),
3749 safety_backup_manifest_key: Some(
3750 "v1/tenants/ns/clusters/k/backups/b/manifest.json".to_string(),
3751 ),
3752 safety_backup_payload_sha256: Some("abc123".to_string()),
3753 database_mutation_started: false,
3754 ..Default::default()
3755 };
3756 assert!(status_with_ref.safety_backup_ref.is_some());
3757 assert!(!status_with_ref.database_mutation_started);
3758 }
3759
3760 #[test]
3761 fn result_document_verification_rejects_missing_backup_id() {
3762 let result_json = r#"{
3763 "apiVersion": "backup.kaniop.rs/v1alpha1",
3764 "kind": "ResultDocument",
3765 "operation": "upload",
3766 "success": true,
3767 "exitCode": "success",
3768 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3769 "payloadSha256": "abc123"
3770 }"#;
3771 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3772 assert!(result.backup_id.is_none());
3773 }
3774
3775 #[test]
3776 fn result_document_verification_rejects_missing_manifest_key() {
3777 let result_json = r#"{
3778 "apiVersion": "backup.kaniop.rs/v1alpha1",
3779 "kind": "ResultDocument",
3780 "operation": "upload",
3781 "success": true,
3782 "exitCode": "success",
3783 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3784 "payloadSha256": "abc123"
3785 }"#;
3786 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3787 assert!(result.manifest_key.is_none());
3788 }
3789
3790 #[test]
3791 fn result_document_verification_rejects_missing_payload_sha256() {
3792 let result_json = r#"{
3793 "apiVersion": "backup.kaniop.rs/v1alpha1",
3794 "kind": "ResultDocument",
3795 "operation": "upload",
3796 "success": true,
3797 "exitCode": "success",
3798 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3799 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json"
3800 }"#;
3801 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3802 assert!(result.payload_sha256.is_none());
3803 }
3804
3805 #[test]
3806 fn result_document_verification_rejects_failure_exit_code() {
3807 let result_json = r#"{
3808 "apiVersion": "backup.kaniop.rs/v1alpha1",
3809 "kind": "ResultDocument",
3810 "operation": "upload",
3811 "success": false,
3812 "exitCode": "integrity",
3813 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3814 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3815 "payloadSha256": "abc123"
3816 }"#;
3817 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3818 assert!(!result.success);
3819 assert_eq!(
3820 result.exit_code,
3821 kaniop_backup_core::result::ExitCode::Integrity
3822 );
3823 }
3824
3825 #[test]
3826 fn result_document_verification_rejects_wrong_operation() {
3827 let result_json = r#"{
3828 "apiVersion": "backup.kaniop.rs/v1alpha1",
3829 "kind": "ResultDocument",
3830 "operation": "download",
3831 "success": true,
3832 "exitCode": "success",
3833 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3834 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3835 "payloadSha256": "abc123"
3836 }"#;
3837 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3838 assert_ne!(result.operation, super::SAFETY_BACKUP_RESULT_OPERATION);
3839 }
3840
3841 #[test]
3842 fn result_document_verification_accepts_valid_document() {
3843 let result_json = r#"{
3844 "apiVersion": "backup.kaniop.rs/v1alpha1",
3845 "kind": "ResultDocument",
3846 "operation": "upload",
3847 "success": true,
3848 "exitCode": "success",
3849 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
3850 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3851 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
3852 }"#;
3853 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
3854 assert!(result.success);
3855 assert_eq!(
3856 result.exit_code,
3857 kaniop_backup_core::result::ExitCode::Success
3858 );
3859 assert_eq!(result.operation, "upload");
3860 assert!(result.backup_id.is_some());
3861 assert!(result.manifest_key.is_some());
3862 assert!(result.payload_sha256.is_some());
3863 }
3864
3865 #[test]
3866 fn termination_message_path_constant_is_valid() {
3867 assert!(super::TERMINATION_MESSAGE_PATH.starts_with('/'));
3868 assert!(super::TERMINATION_MESSAGE_PATH.contains("kaniop-result"));
3869 }
3870
3871 #[test]
3872 fn status_tracks_safety_backup_evidence() {
3873 let status = KanidmRestoreStatus {
3874 phase: KanidmRestorePhase::PreparingSource,
3875 safety_backup_ref: Some("safety-test-restore".to_string()),
3876 safety_backup_expected_backup_id: Some(
3877 "019c7c76-f423-7a12-8f41-2bea7588a303".to_string(),
3878 ),
3879 safety_backup_manifest_key: Some(
3880 "v1/tenants/ns/clusters/k/backups/019c7c76/manifest.json".to_string(),
3881 ),
3882 safety_backup_payload_sha256: Some("e3b0c44298fc1c149afbf4c8996fb924".to_string()),
3883 ..Default::default()
3884 };
3885 assert!(status.safety_backup_ref.is_some());
3886 assert!(status.safety_backup_expected_backup_id.is_some());
3887 assert!(status.safety_backup_manifest_key.is_some());
3888 assert!(status.safety_backup_payload_sha256.is_some());
3889 }
3890
3891 #[test]
3892 fn fail_closed_semantics_preserved_on_result_parse_failure() {
3893 let invalid_json = "not valid json";
3894 let result = kaniop_backup_core::result::parse_result_document(invalid_json);
3895 assert!(result.is_err());
3896 }
3897
3898 #[test]
3899 fn fail_closed_semantics_preserved_on_oversized_result() {
3900 let oversized_json = format!(
3901 "{{\"apiVersion\":\"{}\",\"kind\":\"ResultDocument\",\"operation\":\"upload\",\"success\":true,\"exitCode\":\"success\",\"padding\":\"{}\"}}",
3902 kaniop_backup_core::result::RESULT_DOC_VERSION,
3903 "x".repeat(kaniop_backup_core::result::MAX_RESULT_DOC_SIZE + 1)
3904 );
3905 let result = kaniop_backup_core::result::parse_result_document(&oversized_json);
3906 assert!(result.is_err());
3907 }
3908
3909 #[test]
3910 fn safety_upload_operation_doc_includes_ca_bundle_path_when_set() {
3911 let source = KanidmRestoreSource {
3912 local: None,
3913 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3914 name: "backup-1".to_string(),
3915 }),
3916 };
3917 let mut restore = make_restore(source, None);
3918 restore.metadata.namespace = Some("test-ns".to_string());
3919 let target = super::super::crd::Kanidm::default();
3920 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3921 let doc_str = super::build_safety_upload_operation_doc(
3922 &restore,
3923 &target,
3924 "test-backup-id",
3925 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3926 "my-bucket",
3927 "prod",
3928 "https://s3.example.com",
3929 "us-east-1",
3930 false,
3931 false,
3932 Some(&ca_path),
3933 None,
3934 )
3935 .unwrap();
3936 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3937 assert_eq!(parsed["caBundlePath"], ca_path);
3938 }
3939
3940 #[test]
3941 fn safety_upload_operation_doc_omits_ca_bundle_path_when_absent() {
3942 let source = KanidmRestoreSource {
3943 local: None,
3944 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3945 name: "backup-1".to_string(),
3946 }),
3947 };
3948 let mut restore = make_restore(source, None);
3949 restore.metadata.namespace = Some("test-ns".to_string());
3950 let target = super::super::crd::Kanidm::default();
3951 let doc_str = super::build_safety_upload_operation_doc(
3952 &restore,
3953 &target,
3954 "test-backup-id",
3955 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
3956 "my-bucket",
3957 "prod",
3958 "https://s3.example.com",
3959 "us-east-1",
3960 false,
3961 false,
3962 None,
3963 None,
3964 )
3965 .unwrap();
3966 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3967 assert!(parsed["caBundlePath"].is_null());
3968 }
3969
3970 #[test]
3971 fn download_operation_doc_includes_ca_bundle_path_when_set() {
3972 let source = KanidmRestoreSource {
3973 local: None,
3974 backup_ref: Some(super::KanidmRestoreBackupRefSource {
3975 name: "backup-1".to_string(),
3976 }),
3977 };
3978 let restore = make_restore(source, None);
3979 let target = super::super::crd::Kanidm::default();
3980 let ca_path = kaniop_backup_core::auth::ca_bundle_path();
3981 let doc_str = super::build_download_operation_doc(
3982 &restore,
3983 &target,
3984 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
3985 "019c7c76-f423-7a12-8f41-2bea7588a303",
3986 "real-bucket",
3987 "real-prefix",
3988 "https://real-endpoint.com",
3989 "eu-west-1",
3990 true,
3991 false,
3992 Some(&ca_path),
3993 None,
3994 );
3995 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
3996 assert_eq!(parsed["caBundlePath"], ca_path);
3997 }
3998
3999 #[test]
4000 fn download_operation_doc_omits_ca_bundle_path_when_absent() {
4001 let source = KanidmRestoreSource {
4002 local: None,
4003 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4004 name: "backup-1".to_string(),
4005 }),
4006 };
4007 let restore = make_restore(source, None);
4008 let target = super::super::crd::Kanidm::default();
4009 let doc_str = super::build_download_operation_doc(
4010 &restore,
4011 &target,
4012 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4013 "019c7c76-f423-7a12-8f41-2bea7588a303",
4014 "real-bucket",
4015 "real-prefix",
4016 "https://real-endpoint.com",
4017 "eu-west-1",
4018 true,
4019 false,
4020 None,
4021 None,
4022 );
4023 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4024 assert!(parsed["caBundlePath"].is_null());
4025 }
4026
4027 #[test]
4028 fn source_prep_result_operation_is_download() {
4029 assert_eq!(super::SOURCE_PREP_RESULT_OPERATION, "download");
4030 assert_ne!(
4031 super::SOURCE_PREP_RESULT_OPERATION,
4032 super::SAFETY_BACKUP_RESULT_OPERATION
4033 );
4034 }
4035
4036 #[test]
4037 fn download_result_document_verification_accepts_valid_document() {
4038 let result_json = r#"{
4039 "apiVersion": "backup.kaniop.rs/v1alpha1",
4040 "kind": "ResultDocument",
4041 "operation": "download",
4042 "success": true,
4043 "exitCode": "success",
4044 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4045 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4046 "payloadSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
4047 }"#;
4048 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4049 assert!(result.success);
4050 assert_eq!(
4051 result.exit_code,
4052 kaniop_backup_core::result::ExitCode::Success
4053 );
4054 assert_eq!(result.operation, "download");
4055 assert_eq!(
4056 result.backup_id.as_deref(),
4057 Some("019c7c76-f423-7a12-8f41-2bea7588a303")
4058 );
4059 assert!(result.manifest_key.is_some());
4060 assert!(result.payload_sha256.is_some());
4061 }
4062
4063 #[test]
4064 fn download_result_document_rejects_wrong_operation() {
4065 let result_json = r#"{
4066 "apiVersion": "backup.kaniop.rs/v1alpha1",
4067 "kind": "ResultDocument",
4068 "operation": "upload",
4069 "success": true,
4070 "exitCode": "success",
4071 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4072 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4073 "payloadSha256": "abc123"
4074 }"#;
4075 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4076 assert_ne!(result.operation, super::SOURCE_PREP_RESULT_OPERATION);
4077 }
4078
4079 #[test]
4080 fn download_result_document_rejects_failure_success_field() {
4081 let result_json = r#"{
4082 "apiVersion": "backup.kaniop.rs/v1alpha1",
4083 "kind": "ResultDocument",
4084 "operation": "download",
4085 "success": false,
4086 "exitCode": "retryable",
4087 "backupId": "019c7c76-f423-7a12-8f41-2bea7588a303",
4088 "manifestKey": "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4089 "payloadSha256": "abc123"
4090 }"#;
4091 let result = kaniop_backup_core::result::parse_result_document(result_json).unwrap();
4092 assert!(!result.success);
4093 assert_ne!(
4094 result.exit_code,
4095 kaniop_backup_core::result::ExitCode::Success
4096 );
4097 }
4098
4099 #[test]
4100 fn pre_mutation_boundary_blocks_transition_on_corrupt_download() {
4101 let status = KanidmRestoreStatus {
4102 phase: KanidmRestorePhase::PreparingSource,
4103 database_mutation_started: false,
4104 ..Default::default()
4105 };
4106 assert!(!status.database_mutation_started);
4107 assert_eq!(status.phase, KanidmRestorePhase::PreparingSource);
4108 }
4109
4110 #[test]
4111 fn backup_job_volume_size_default_is_10gi() {
4112 let qty = crate::controller::backup_job_volume_size();
4113 assert_eq!(qty, Quantity("10Gi".to_string()));
4114 }
4115
4116 #[test]
4117 fn encryption_env_vars_present_when_client_side_key_ref_set() {
4118 use kaniop_backup_core::auth::build_encryption_env_vars;
4119 use kaniop_backup_core::crd::SecretRef;
4120
4121 let key_ref = SecretRef {
4122 name: "kek-secret".to_string(),
4123 };
4124 let env_vars = build_encryption_env_vars(Some(&key_ref));
4125 assert_eq!(env_vars.len(), 1);
4126 assert_eq!(env_vars[0].name, "KANIOP_ENCRYPTION_KEY");
4127 let secret_ref = env_vars[0]
4128 .value_from
4129 .as_ref()
4130 .unwrap()
4131 .secret_key_ref
4132 .as_ref()
4133 .unwrap();
4134 assert_eq!(secret_ref.name, "kek-secret");
4135 assert_eq!(secret_ref.key, "encryption-key");
4136 }
4137
4138 #[test]
4139 fn encryption_env_vars_absent_when_no_key_ref() {
4140 use kaniop_backup_core::auth::build_encryption_env_vars;
4141
4142 let env_vars = build_encryption_env_vars(None);
4143 assert!(
4144 env_vars.is_empty(),
4145 "encryption env vars must be empty when key_ref is None"
4146 );
4147 }
4148
4149 #[test]
4150 fn safety_upload_operation_doc_includes_encryption_fields_when_client_side() {
4151 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption, SecretRef};
4152
4153 let source = KanidmRestoreSource {
4154 local: None,
4155 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4156 name: "backup-1".to_string(),
4157 }),
4158 };
4159 let mut restore = make_restore(source, None);
4160 restore.metadata.namespace = Some("test-ns".to_string());
4161 let target = super::super::crd::Kanidm::default();
4162 let encryption = RepositoryEncryption {
4163 mode: EncryptionMode::ClientSide,
4164 key_id: None,
4165 key_ref: Some(SecretRef {
4166 name: "kek-secret".to_string(),
4167 }),
4168 };
4169 let doc_str = super::build_safety_upload_operation_doc(
4170 &restore,
4171 &target,
4172 "test-backup-id",
4173 "v1/tenants/test-ns/clusters/test-uid/backups/test-backup-id/manifest.json",
4174 "my-bucket",
4175 "prod",
4176 "https://s3.example.com",
4177 "us-east-1",
4178 false,
4179 false,
4180 None,
4181 Some(&encryption),
4182 )
4183 .unwrap();
4184 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4185 assert_eq!(parsed["encryptionMode"], "clientSide");
4186 assert!(parsed["encryptionKeyId"].is_null());
4187 }
4188
4189 #[test]
4190 fn download_operation_doc_includes_encryption_fields_when_provider_kms() {
4191 use kaniop_backup_core::crd::{EncryptionMode, RepositoryEncryption};
4192
4193 let source = KanidmRestoreSource {
4194 local: None,
4195 backup_ref: Some(super::KanidmRestoreBackupRefSource {
4196 name: "backup-1".to_string(),
4197 }),
4198 };
4199 let restore = make_restore(source, None);
4200 let target = super::super::crd::Kanidm::default();
4201 let encryption = RepositoryEncryption {
4202 mode: EncryptionMode::ProviderKms,
4203 key_id: Some("alias/kaniop-backups".to_string()),
4204 key_ref: None,
4205 };
4206 let doc_str = super::build_download_operation_doc(
4207 &restore,
4208 &target,
4209 "v1/tenants/ns/clusters/k/backups/b/manifest.json",
4210 "019c7c76-f423-7a12-8f41-2bea7588a303",
4211 "real-bucket",
4212 "real-prefix",
4213 "https://real-endpoint.com",
4214 "eu-west-1",
4215 true,
4216 false,
4217 None,
4218 Some(&encryption),
4219 );
4220 let parsed: serde_json::Value = serde_json::from_str(&doc_str).unwrap();
4221 assert_eq!(parsed["encryptionMode"], "providerKms");
4222 assert_eq!(parsed["encryptionKeyId"], "alias/kaniop-backups");
4223 }
4224
4225 #[test]
4226 fn post_mutation_failure_records_job_names() {
4227 let existing = KanidmRestoreStatus {
4228 phase: KanidmRestorePhase::RestoringPrimary,
4229 database_mutation_started: true,
4230 ..Default::default()
4231 };
4232
4233 let restore_failed = super::restore_primary_failed_status(&existing, "my-restore-job");
4234 assert_eq!(restore_failed.phase, KanidmRestorePhase::Failed);
4235 assert_eq!(
4236 restore_failed.restore_job_name.as_deref(),
4237 Some("my-restore-job")
4238 );
4239 assert!(restore_failed.verify_job_name.is_none());
4240 assert!(restore_failed.database_mutation_started);
4241 assert_eq!(
4242 restore_failed.message.as_deref(),
4243 Some("database restore job failed")
4244 );
4245
4246 let verify_failed = super::verifying_failed_status(&existing, "my-verify-job");
4247 assert_eq!(verify_failed.phase, KanidmRestorePhase::Failed);
4248 assert_eq!(
4249 verify_failed.verify_job_name.as_deref(),
4250 Some("my-verify-job")
4251 );
4252 assert!(verify_failed.restore_job_name.is_none());
4253 assert!(verify_failed.database_mutation_started);
4254 assert_eq!(
4255 verify_failed.message.as_deref(),
4256 Some("database verification failed")
4257 );
4258 }
4259
4260 #[test]
4261 fn restore_and_verify_job_names_are_deterministic() {
4262 let source = KanidmRestoreSource {
4263 local: Some(super::KanidmRestoreLocalSource {
4264 file_name: "backup.json".to_string(),
4265 }),
4266 backup_ref: None,
4267 };
4268 let restore = make_restore(source, None);
4269 assert_eq!(restore_job_name(&restore), "test-restore-restore");
4270 assert_eq!(verify_job_name(&restore), "test-restore-verify");
4271 }
4272}