1use super::crd::Kanidm;
2use super::reconcile::{CLUSTER_LABEL, statefulset::StatefulSetExt};
3
4use kaniop_k8s_util::client::get_output;
5use kaniop_k8s_util::error::{Error, Result};
6
7use std::collections::BTreeSet;
8use std::sync::Arc;
9use std::time::Duration;
10
11use futures::StreamExt;
12use k8s_openapi::api::apps::v1::StatefulSet;
13use k8s_openapi::api::batch::v1::{Job, JobSpec};
14use k8s_openapi::api::core::v1::{
15 ConfigMap, ConfigMapVolumeSource, Container, EnvVar, PersistentVolumeClaim,
16 PersistentVolumeClaimVolumeSource, Pod, PodSpec, PodTemplateSpec, Volume, VolumeMount,
17};
18use k8s_openapi::api::storage::v1::VolumeAttachment;
19use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time};
20use k8s_openapi::jiff::Timestamp;
21use kube::api::{
22 AttachParams, DeleteParams, ListParams, ObjectMeta, Patch, PatchParams, PostParams,
23};
24use kube::runtime::controller::{Action, Controller};
25use kube::runtime::events::{Event, EventType, Recorder, Reporter};
26use kube::runtime::finalizer::{Event as Finalizer, finalizer};
27use kube::runtime::watcher;
28use kube::{Api, Client, CustomResource, Resource, ResourceExt};
29use opentelemetry::metrics::{Counter, Histogram};
30use opentelemetry::{KeyValue, global};
31#[cfg(feature = "schemars")]
32use schemars::JsonSchema;
33use serde::{Deserialize, Serialize};
34use serde_json::json;
35use tracing::{debug, error, info, warn};
36
37pub const CONTROLLER_ID: &str = "kanidm-restore";
38pub const RESTORE_ANNOTATION: &str = "kanidm.kaniop.rs/restore-in-progress";
39const RESTORE_FINALIZER: &str = "kanidmrestores.kaniop.rs/finalizer";
40const DATA_VOLUME: &str = "kanidm-data";
41const CONFIG_VOLUME: &str = "kanidm-config";
42const DATA_PATH: &str = "/data";
43const CONFIG_PATH: &str = "/run/kanidm";
44const BACKUP_PATH: &str = "/data/backups";
45const REQUEUE: Duration = Duration::from_secs(2);
46const CONDITION_TRUE: &str = "True";
47const CONDITION_FALSE: &str = "False";
48const CONDITION_PROGRESSING: &str = "Progressing";
49const CONDITION_READY: &str = "Ready";
50const CONDITION_FAILED: &str = "Failed";
51
52#[derive(CustomResource, Serialize, Deserialize, Clone, Debug)]
53#[cfg_attr(feature = "schemars", derive(JsonSchema))]
54#[cfg_attr(
55 feature = "schemars",
56 schemars(extend("x-kubernetes-validations" = [{"message": "KanidmRestore spec is immutable", "rule": "self == oldSelf"}]))
57)]
58#[kube(
59 category = "kaniop",
60 group = "kaniop.rs",
61 version = "v1beta1",
62 kind = "KanidmRestore",
63 plural = "kanidmrestores",
64 singular = "kanidmrestore",
65 shortname = "idmrestore",
66 namespaced,
67 status = "KanidmRestoreStatus",
68 printcolumn = r#"{"name":"Target","type":"string","jsonPath":".spec.targetRef.name"}"#,
69 printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"#,
70 printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#
71)]
72#[serde(rename_all = "camelCase")]
73pub struct KanidmRestoreSpec {
74 pub target_ref: KanidmRestoreTargetRef,
75 pub source: KanidmRestoreSource,
76 pub restore_image: String,
78}
79
80#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
81#[cfg_attr(feature = "schemars", derive(JsonSchema))]
82#[serde(rename_all = "camelCase")]
83pub struct KanidmRestoreTargetRef {
84 pub name: String,
85 pub uid: String,
86}
87
88#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
89#[cfg_attr(feature = "schemars", derive(JsonSchema))]
90#[serde(rename_all = "camelCase")]
91pub struct KanidmRestoreSource {
92 pub local: KanidmRestoreLocalSource,
93}
94
95#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
96#[cfg_attr(feature = "schemars", derive(JsonSchema))]
97#[serde(rename_all = "camelCase")]
98pub struct KanidmRestoreLocalSource {
99 pub file_name: String,
101}
102
103#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
104#[cfg_attr(feature = "schemars", derive(JsonSchema))]
105pub enum KanidmRestorePhase {
106 #[default]
107 Pending,
108 Validating,
109 Quiescing,
110 RestoringPrimary,
111 Verifying,
112 RebuildingReplicas,
113 Resuming,
114 Completed,
115 Failed,
116}
117
118#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
119#[cfg_attr(feature = "schemars", derive(JsonSchema))]
120#[serde(rename_all = "camelCase")]
121pub struct KanidmRestoreStatus {
122 #[serde(skip_serializing_if = "Option::is_none")]
123 pub observed_generation: Option<i64>,
124 #[serde(skip_serializing_if = "Option::is_none")]
125 pub observed_target_uid: Option<String>,
126 #[serde(default)]
127 pub phase: KanidmRestorePhase,
128 #[serde(skip_serializing_if = "Option::is_none")]
129 pub restore_job_name: Option<String>,
130 #[serde(skip_serializing_if = "Option::is_none")]
131 pub verify_job_name: Option<String>,
132 #[serde(default)]
133 pub replicas_cleared: bool,
134 #[serde(default)]
137 pub database_mutation_started: bool,
138 #[serde(default, skip_serializing_if = "Vec::is_empty")]
139 pub conditions: Vec<Condition>,
140 #[serde(skip_serializing_if = "Option::is_none")]
141 pub message: Option<String>,
142}
143
144#[derive(Clone)]
145struct RestoreMetrics {
146 attempts: Counter<u64>,
147 outcomes: Counter<u64>,
148 duration_seconds: Histogram<f64>,
149}
150
151impl RestoreMetrics {
152 fn new() -> Self {
153 let meter = global::meter("kaniop");
154 Self {
155 attempts: meter
156 .u64_counter("kanidm_restore_attempts")
157 .with_description("Number of Kanidm restore attempts started")
158 .build(),
159 outcomes: meter
160 .u64_counter("kanidm_restore_outcomes")
161 .with_description("Number of terminal Kanidm restore outcomes")
162 .build(),
163 duration_seconds: meter
164 .f64_histogram("kanidm_restore_duration_seconds")
165 .with_description("Kanidm restore duration from object creation to terminal phase")
166 .with_unit("s")
167 .build(),
168 }
169 }
170}
171
172#[derive(Clone)]
173struct RestoreContext {
174 client: Client,
175 recorder: Recorder,
176 metrics: RestoreMetrics,
177}
178
179pub async fn run(client: Client) {
180 let api = Api::<KanidmRestore>::all(client.clone());
181 let recorder = Recorder::new(
182 client.clone(),
183 Reporter {
184 controller: CONTROLLER_ID.into(),
185 instance: None,
186 },
187 );
188 let ctx = Arc::new(RestoreContext {
189 client,
190 recorder,
191 metrics: RestoreMetrics::new(),
192 });
193 info!("starting {CONTROLLER_ID} controller");
194 Controller::new(api, watcher::Config::default().any_semantic())
195 .shutdown_on_signal()
196 .run(reconcile_restore, error_policy, ctx)
197 .for_each(|result| async move {
198 if let Err(error) = result {
199 error!(%error, "KanidmRestore reconciliation failed");
200 }
201 })
202 .await;
203}
204
205fn error_policy(restore: Arc<KanidmRestore>, error: &Error, _ctx: Arc<RestoreContext>) -> Action {
206 warn!(restore = %restore.name_any(), %error, "restore reconciliation error");
207 Action::requeue(Duration::from_secs(5))
208}
209
210async fn reconcile_restore(
211 restore: Arc<KanidmRestore>,
212 ctx: Arc<RestoreContext>,
213) -> Result<Action> {
214 let namespace = restore
215 .namespace()
216 .ok_or_else(|| Error::MissingData("KanidmRestore has no namespace".to_string()))?;
217 let api = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &namespace);
218 finalizer(&api, RESTORE_FINALIZER, restore, |event| {
219 let ctx = ctx.clone();
220 async move {
221 match event {
222 Finalizer::Apply(restore) => reconcile_apply(restore, ctx).await,
223 Finalizer::Cleanup(restore) => cleanup(restore, ctx).await,
224 }
225 }
226 })
227 .await
228 .map_err(|error| {
229 Error::FinalizerError(
230 "failed on KanidmRestore finalizer".to_string(),
231 Box::new(error),
232 )
233 })
234}
235
236async fn reconcile_apply(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
237 let phase = restore.status.as_ref().map(|s| s.phase).unwrap_or_default();
238 match phase {
239 KanidmRestorePhase::Pending => {
240 set_phase(&restore, &ctx, KanidmRestorePhase::Validating, None).await?;
241 Ok(Action::requeue(REQUEUE))
242 }
243 KanidmRestorePhase::Validating => match validate(&restore, &ctx).await {
244 Ok(target) => {
245 mark_restoring(&restore, &target, &ctx).await?;
246 let mut status = restore.status.clone().unwrap_or_default();
247 status.observed_target_uid = target.uid();
248 status.phase = KanidmRestorePhase::Quiescing;
249 status.message = None;
250 patch_status(&restore, &ctx, status).await?;
251 Ok(Action::requeue(REQUEUE))
252 }
253 Err(error) => {
254 set_phase(
255 &restore,
256 &ctx,
257 KanidmRestorePhase::Failed,
258 Some(error.to_string()),
259 )
260 .await?;
261 Ok(Action::requeue(Duration::from_secs(300)))
262 }
263 },
264 KanidmRestorePhase::Quiescing => {
265 let target = get_target(&restore, &ctx).await?;
266 scale_all(&target, &ctx, 0).await?;
267 if target_pods_stopped(&target, &ctx).await?
268 && target_volumes_detached(&target, &ctx).await?
269 {
270 set_phase(&restore, &ctx, KanidmRestorePhase::RestoringPrimary, None).await?;
271 }
272 Ok(Action::requeue(REQUEUE))
273 }
274 KanidmRestorePhase::RestoringPrimary => {
275 let target = get_target(&restore, &ctx).await?;
276 ensure_restore_config(&restore, &target, &ctx).await?;
277 if !restore
278 .status
279 .as_ref()
280 .is_some_and(|status| status.database_mutation_started)
281 {
282 let mut status = restore.status.clone().unwrap_or_default();
283 status.database_mutation_started = true;
284 status.message = Some(
285 "database mutation boundary persisted; starting primary restore".to_string(),
286 );
287 patch_status(&restore, &ctx, status).await?;
288 return Ok(Action::requeue(REQUEUE));
289 }
290 let name = restore_job_name(&restore);
291 ensure_database_job(&restore, &target, &ctx, &name, false).await?;
292 match job_state(&restore, &ctx, &name).await? {
293 JobState::Complete => {
294 let mut status = restore.status.clone().unwrap_or_default();
295 status.restore_job_name = Some(name);
296 status.phase = KanidmRestorePhase::Verifying;
297 status.message = None;
298 patch_status(&restore, &ctx, status).await?;
299 }
300 JobState::Failed => {
301 fail_after_mutation(&restore, &ctx, "database restore job failed").await?
302 }
303 JobState::Running => {}
304 }
305 Ok(Action::requeue(REQUEUE))
306 }
307 KanidmRestorePhase::Verifying => {
308 let target = get_target(&restore, &ctx).await?;
309 let name = verify_job_name(&restore);
310 ensure_database_job(&restore, &target, &ctx, &name, true).await?;
311 match job_state(&restore, &ctx, &name).await? {
312 JobState::Complete => {
313 let mut status = restore.status.clone().unwrap_or_default();
314 status.verify_job_name = Some(name);
315 status.phase = KanidmRestorePhase::RebuildingReplicas;
316 status.message = None;
317 patch_status(&restore, &ctx, status).await?;
318 }
319 JobState::Failed => {
320 fail_after_mutation(&restore, &ctx, "database verification failed").await?
321 }
322 JobState::Running => {}
323 }
324 Ok(Action::requeue(REQUEUE))
325 }
326 KanidmRestorePhase::RebuildingReplicas => {
327 let target = get_target(&restore, &ctx).await?;
328 let mut status = restore.status.clone().unwrap_or_default();
329 if !status.replicas_cleared {
330 if delete_secondary_pvcs(&target, &ctx).await? {
331 status.replicas_cleared = true;
332 status.message = Some("secondary database state cleared".to_string());
333 patch_status(&restore, &ctx, status).await?;
334 }
335 return Ok(Action::requeue(REQUEUE));
336 }
337
338 scale_primary(&target, &ctx, 1).await?;
339 if !primary_ready(&target, &ctx).await? {
340 return Ok(Action::requeue(REQUEUE));
341 }
342 scale_desired(&target, &ctx).await?;
343 if all_desired_ready(&target, &ctx).await? {
344 set_phase(&restore, &ctx, KanidmRestorePhase::Resuming, None).await?;
345 }
346 Ok(Action::requeue(REQUEUE))
347 }
348 KanidmRestorePhase::Resuming => {
349 let target = get_target(&restore, &ctx).await?;
350 clear_restoring(&restore, &target, &ctx).await?;
351 set_phase(&restore, &ctx, KanidmRestorePhase::Completed, None).await?;
352 Ok(Action::requeue(Duration::from_secs(3600)))
353 }
354 KanidmRestorePhase::Completed | KanidmRestorePhase::Failed => {
355 Ok(Action::requeue(Duration::from_secs(3600)))
356 }
357 }
358}
359
360async fn cleanup(restore: Arc<KanidmRestore>, ctx: Arc<RestoreContext>) -> Result<Action> {
361 let status = restore.status.clone().unwrap_or_default();
362 if status.database_mutation_started && status.phase != KanidmRestorePhase::Completed {
363 return Err(Error::MissingData(format!(
364 "refusing to remove KanidmRestore after database mutation started in phase {:?}; recover the target or remove the finalizer explicitly",
365 status.phase
366 )));
367 }
368 if let Ok(target) = get_target(&restore, &ctx).await {
369 let owns_maintenance =
370 target.annotations().get(RESTORE_ANNOTATION) == restore.uid().as_ref();
371 if owns_maintenance && !status.database_mutation_started {
372 scale_desired(&target, &ctx).await?;
373 }
374 clear_restoring(&restore, &target, &ctx).await?;
375 }
376 Ok(Action::await_change())
377}
378
379async fn validate(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
380 let target = get_target(restore, ctx).await?;
381 let actual_uid = target
382 .uid()
383 .ok_or_else(|| Error::MissingData("target Kanidm has no UID".to_string()))?;
384 if actual_uid != restore.spec.target_ref.uid {
385 return Err(Error::MissingData(format!(
386 "target UID mismatch: expected {}, got {}",
387 restore.spec.target_ref.uid, actual_uid
388 )));
389 }
390 if !safe_basename(&restore.spec.source.local.file_name) {
391 return Err(Error::MissingData(
392 "restore source fileName must be a safe basename".to_string(),
393 ));
394 }
395 if target.spec.image != restore.spec.restore_image || mutable_image(&restore.spec.restore_image)
396 {
397 return Err(Error::MissingData(format!(
398 "restoreImage must be the target's pinned Kanidm image (target image is {})",
399 target.spec.image
400 )));
401 }
402 let storage = target
403 .spec
404 .storage
405 .as_ref()
406 .ok_or_else(|| Error::MissingData("restore requires persistent storage".to_string()))?;
407 if storage.empty_dir.is_some()
408 || storage.ephemeral.is_some()
409 || storage.volume_claim_template.is_none()
410 {
411 return Err(Error::MissingData(
412 "restore requires PVC-backed Kanidm storage".to_string(),
413 ));
414 }
415 let primaries = target
416 .spec
417 .replica_groups
418 .iter()
419 .filter(|rg| rg.primary_node)
420 .count();
421 if primaries != 1 {
422 return Err(Error::MissingData(
423 "backup/restore requires exactly one primary replica group".to_string(),
424 ));
425 }
426 validate_backup_source(restore, &target, ctx).await?;
427
428 let ns = restore.namespace().unwrap();
429 let restores = Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
430 .list(&ListParams::default())
431 .await
432 .map_err(|e| Error::kube_error("list", "KanidmRestore", &ns, "*", e))?;
433 if restores.items.iter().any(|other| {
434 other.name_any() != restore.name_any()
435 && other.spec.target_ref.name == restore.spec.target_ref.name
436 && !matches!(
437 other.status.as_ref().map(|s| s.phase),
438 Some(KanidmRestorePhase::Completed | KanidmRestorePhase::Failed)
439 )
440 }) {
441 return Err(Error::MissingData(
442 "another active restore targets this Kanidm".to_string(),
443 ));
444 }
445 Ok(target)
446}
447
448fn safe_basename(name: &str) -> bool {
449 !name.is_empty()
450 && name != "."
451 && name != ".."
452 && !name.contains('/')
453 && !name.contains('\\')
454 && !name.contains('\0')
455}
456
457fn mutable_image(image: &str) -> bool {
458 image == "kanidm/server:latest"
459 || image.ends_with(":latest")
460 || (!image.contains('@')
461 && !image
462 .rsplit('/')
463 .next()
464 .is_some_and(|part| part.contains(':')))
465}
466
467async fn validate_backup_source(
468 restore: &KanidmRestore,
469 target: &Kanidm,
470 ctx: &RestoreContext,
471) -> Result<()> {
472 let ns = target.namespace().unwrap();
473 let primary = primary_group(target)?;
474 let pod_name = format!("{}-0", target.statefulset_name(&primary.name));
475 let backup_file = format!("{BACKUP_PATH}/{}", restore.spec.source.local.file_name);
476 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns);
477 let attached = pods
478 .exec(
479 &pod_name,
480 vec![
481 "/bin/sh".to_string(),
482 "-c".to_string(),
483 r#"test -f "$1""#.to_string(),
484 "kaniop-backup-preflight".to_string(),
485 backup_file.clone(),
486 ],
487 &AttachParams::default().container("kanidm"),
488 )
489 .await
490 .map_err(|e| Error::kube_error("exec backup preflight in", "Pod", &ns, &pod_name, e))?;
491 get_output(attached).await.map(|_| ()).map_err(|error| {
492 Error::MissingData(format!(
493 "restore source {backup_file} is not accessible on primary pod {pod_name}: {error}"
494 ))
495 })
496}
497
498async fn get_target(restore: &KanidmRestore, ctx: &RestoreContext) -> Result<Kanidm> {
499 let ns = restore
500 .namespace()
501 .ok_or_else(|| Error::MissingData("restore has no namespace".to_string()))?;
502 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
503 .get(&restore.spec.target_ref.name)
504 .await
505 .map_err(|e| Error::kube_error("get", "Kanidm", &ns, &restore.spec.target_ref.name, e))
506}
507
508async fn mark_restoring(
509 restore: &KanidmRestore,
510 target: &Kanidm,
511 ctx: &RestoreContext,
512) -> Result<()> {
513 let ns = target.namespace().unwrap();
514 let uid = restore
515 .uid()
516 .ok_or_else(|| Error::MissingData("restore has no UID".to_string()))?;
517 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
518 .patch(
519 &target.name_any(),
520 &PatchParams::default(),
521 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:uid}}})),
522 )
523 .await
524 .map(|_| ())
525 .map_err(|e| Error::kube_error("mark restoring", "Kanidm", &ns, target.name_any(), e))
526}
527
528async fn clear_restoring(
529 restore: &KanidmRestore,
530 target: &Kanidm,
531 ctx: &RestoreContext,
532) -> Result<()> {
533 let ns = target.namespace().unwrap();
534 if target.annotations().get(RESTORE_ANNOTATION) != restore.uid().as_ref() {
535 return Ok(());
536 }
537 Api::<Kanidm>::namespaced(ctx.client.clone(), &ns)
538 .patch(
539 &target.name_any(),
540 &PatchParams::default(),
541 &Patch::Merge(json!({"metadata":{"annotations":{RESTORE_ANNOTATION:null}}})),
542 )
543 .await
544 .map(|_| ())
545 .map_err(|e| Error::kube_error("clear restoring", "Kanidm", &ns, target.name_any(), e))
546}
547
548async fn patch_status(
549 restore: &KanidmRestore,
550 ctx: &RestoreContext,
551 mut status: KanidmRestoreStatus,
552) -> Result<()> {
553 let ns = restore.namespace().unwrap();
554 let previous_phase = restore.status.as_ref().map(|current| current.phase);
555 let next_phase = status.phase;
556 status.observed_generation = restore.metadata.generation;
557 update_restore_conditions(&mut status, restore.metadata.generation);
558
559 Api::<KanidmRestore>::namespaced(ctx.client.clone(), &ns)
560 .patch_status(
561 &restore.name_any(),
562 &PatchParams::default(),
563 &Patch::Merge(json!({"status": &status})),
564 )
565 .await
566 .map_err(|e| {
567 Error::kube_error("patch status", "KanidmRestore", &ns, restore.name_any(), e)
568 })?;
569
570 if previous_phase != Some(next_phase) {
571 record_restore_transition(
572 restore,
573 ctx,
574 previous_phase,
575 next_phase,
576 status.message.as_deref(),
577 )
578 .await;
579 }
580 Ok(())
581}
582
583fn update_restore_conditions(status: &mut KanidmRestoreStatus, generation: Option<i64>) {
584 let previous = status.conditions.clone();
585 let phase = status.phase;
586 let phase_reason = format!("{phase:?}");
587 let message = status
588 .message
589 .clone()
590 .unwrap_or_else(|| format!("Kanidm restore is in phase {phase_reason}."));
591 let terminal_success = phase == KanidmRestorePhase::Completed;
592 let terminal_failure = phase == KanidmRestorePhase::Failed;
593 let progressing = !terminal_success && !terminal_failure;
594
595 status.conditions = vec![
596 restore_condition(
597 &previous,
598 CONDITION_PROGRESSING,
599 if progressing {
600 CONDITION_TRUE
601 } else {
602 CONDITION_FALSE
603 },
604 &phase_reason,
605 &message,
606 generation,
607 ),
608 restore_condition(
609 &previous,
610 CONDITION_READY,
611 if terminal_success {
612 CONDITION_TRUE
613 } else {
614 CONDITION_FALSE
615 },
616 if terminal_success {
617 "RestoreCompleted"
618 } else {
619 "RestoreNotCompleted"
620 },
621 if terminal_success {
622 "Kanidm restore completed successfully."
623 } else {
624 "Kanidm restore has not completed successfully."
625 },
626 generation,
627 ),
628 restore_condition(
629 &previous,
630 CONDITION_FAILED,
631 if terminal_failure {
632 CONDITION_TRUE
633 } else {
634 CONDITION_FALSE
635 },
636 if terminal_failure {
637 "RestoreFailed"
638 } else {
639 "NoRestoreFailure"
640 },
641 if terminal_failure {
642 status
643 .message
644 .as_deref()
645 .unwrap_or("Kanidm restore failed.")
646 } else {
647 "No terminal restore failure has been recorded."
648 },
649 generation,
650 ),
651 ];
652}
653
654fn restore_condition(
655 previous: &[Condition],
656 condition_type: &str,
657 condition_status: &str,
658 reason: &str,
659 message: &str,
660 generation: Option<i64>,
661) -> Condition {
662 let last_transition_time = previous
663 .iter()
664 .find(|condition| {
665 condition.type_ == condition_type
666 && condition.status == condition_status
667 && condition.reason == reason
668 })
669 .map(|condition| condition.last_transition_time.clone())
670 .unwrap_or_else(|| Time(Timestamp::now()));
671 Condition {
672 type_: condition_type.to_string(),
673 status: condition_status.to_string(),
674 reason: reason.to_string(),
675 message: message.to_string(),
676 last_transition_time,
677 observed_generation: generation,
678 }
679}
680
681async fn record_restore_transition(
682 restore: &KanidmRestore,
683 ctx: &RestoreContext,
684 previous_phase: Option<KanidmRestorePhase>,
685 phase: KanidmRestorePhase,
686 message: Option<&str>,
687) {
688 if phase == KanidmRestorePhase::Validating {
689 ctx.metrics.attempts.add(1, &[]);
690 }
691
692 let result = match phase {
693 KanidmRestorePhase::Completed => Some("success"),
694 KanidmRestorePhase::Failed => Some("failure"),
695 _ => None,
696 };
697 if let Some(result) = result {
698 let attributes = [KeyValue::new("result", result)];
699 ctx.metrics.outcomes.add(1, &attributes);
700 if let Some(created) = restore.metadata.creation_timestamp.as_ref() {
701 let elapsed = (Timestamp::now().as_second() - created.0.as_second()).max(0) as f64;
702 ctx.metrics.duration_seconds.record(elapsed, &attributes);
703 }
704 }
705
706 let reason = if phase == KanidmRestorePhase::Failed {
707 "RestoreFailed"
708 } else {
709 "RestorePhaseChanged"
710 };
711 let note = message.map(str::to_string).or_else(|| {
712 Some(format!(
713 "Kanidm restore phase changed from {} to {phase:?}.",
714 previous_phase
715 .map(|previous| format!("{previous:?}"))
716 .unwrap_or_else(|| "None".to_string())
717 ))
718 });
719 if let Err(error) = ctx
720 .recorder
721 .publish(
722 &Event {
723 type_: if phase == KanidmRestorePhase::Failed {
724 EventType::Warning
725 } else {
726 EventType::Normal
727 },
728 reason: reason.to_string(),
729 note,
730 action: "Restore".to_string(),
731 secondary: None,
732 },
733 &restore.object_ref(&()),
734 )
735 .await
736 {
737 warn!(restore = %restore.name_any(), %error, "failed to publish restore event");
738 }
739}
740
741async fn set_phase(
742 restore: &KanidmRestore,
743 ctx: &RestoreContext,
744 phase: KanidmRestorePhase,
745 message: Option<String>,
746) -> Result<()> {
747 let mut status = restore.status.clone().unwrap_or_default();
748 status.phase = phase;
749 status.message = message;
750 patch_status(restore, ctx, status).await
751}
752
753async fn fail_after_mutation(
754 restore: &KanidmRestore,
755 ctx: &RestoreContext,
756 message: &str,
757) -> Result<()> {
758 set_phase(
759 restore,
760 ctx,
761 KanidmRestorePhase::Failed,
762 Some(message.to_string()),
763 )
764 .await
765}
766
767fn primary_group(target: &Kanidm) -> Result<&super::crd::ReplicaGroup> {
768 target
769 .spec
770 .replica_groups
771 .iter()
772 .find(|rg| rg.primary_node)
773 .ok_or_else(|| Error::MissingData("primary replica group not found".to_string()))
774}
775
776async fn scale_all(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
777 let ns = target.namespace().unwrap();
778 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
779 for rg in &target.spec.replica_groups {
780 let name = target.statefulset_name(&rg.name);
781 api.patch(
782 &name,
783 &PatchParams::default(),
784 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
785 )
786 .await
787 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
788 }
789 Ok(())
790}
791
792async fn scale_primary(target: &Kanidm, ctx: &RestoreContext, replicas: i32) -> Result<()> {
793 let ns = target.namespace().unwrap();
794 let rg = primary_group(target)?;
795 let name = target.statefulset_name(&rg.name);
796 Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
797 .patch(
798 &name,
799 &PatchParams::default(),
800 &Patch::Merge(json!({"spec":{"replicas":replicas}})),
801 )
802 .await
803 .map(|_| ())
804 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))
805}
806
807async fn scale_desired(target: &Kanidm, ctx: &RestoreContext) -> Result<()> {
808 let ns = target.namespace().unwrap();
809 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
810 for rg in &target.spec.replica_groups {
811 let name = target.statefulset_name(&rg.name);
812 api.patch(
813 &name,
814 &PatchParams::default(),
815 &Patch::Merge(json!({"spec":{"replicas":rg.replicas}})),
816 )
817 .await
818 .map_err(|e| Error::kube_error("scale", "StatefulSet", &ns, &name, e))?;
819 }
820 Ok(())
821}
822
823async fn target_pods_stopped(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
824 let ns = target.namespace().unwrap();
825 let pods = Api::<Pod>::namespaced(ctx.client.clone(), &ns)
826 .list(&ListParams::default().labels(&format!("{CLUSTER_LABEL}={}", target.name_any())))
827 .await
828 .map_err(|e| Error::kube_error("list", "Pod", &ns, target.name_any(), e))?;
829 Ok(pods.items.is_empty())
830}
831
832async fn target_volumes_detached(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
833 let ns = target.namespace().unwrap();
834 let pvc_api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
835 let mut pv_names = BTreeSet::new();
836 let primary_name = primary_pvc_name(target)?;
837 let primary_pvc = pvc_api
838 .get(&primary_name)
839 .await
840 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &primary_name, e))?;
841 let primary_pv = primary_pvc
842 .spec
843 .as_ref()
844 .and_then(|spec| spec.volume_name.clone())
845 .ok_or_else(|| {
846 Error::MissingData(format!("primary PVC {ns}/{primary_name} has no bound PV"))
847 })?;
848 pv_names.insert(primary_pv);
849
850 for rg in &target.spec.replica_groups {
851 let sts = target.statefulset_name(&rg.name);
852 for ordinal in 0..rg.replicas {
853 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
854 if name == primary_name {
855 continue;
856 }
857 if let Some(pvc) = pvc_api
858 .get_opt(&name)
859 .await
860 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
861 && let Some(pv_name) = pvc.spec.as_ref().and_then(|spec| spec.volume_name.clone())
862 {
863 pv_names.insert(pv_name);
864 }
865 }
866 }
867
868 let attachments = Api::<VolumeAttachment>::all(ctx.client.clone())
869 .list(&ListParams::default())
870 .await
871 .map_err(|e| Error::kube_error("list", "VolumeAttachment", "", "*", e))?;
872 Ok(!attachments.items.iter().any(|attachment| {
873 attachment
874 .spec
875 .source
876 .persistent_volume_name
877 .as_ref()
878 .is_some_and(|pv_name| pv_names.contains(pv_name))
879 }))
880}
881
882async fn primary_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
883 let ns = target.namespace().unwrap();
884 let name = target.statefulset_name(&primary_group(target)?.name);
885 let sts = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns)
886 .get(&name)
887 .await
888 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
889 Ok(sts
890 .status
891 .as_ref()
892 .and_then(|s| s.ready_replicas)
893 .unwrap_or(0)
894 >= 1)
895}
896
897async fn all_desired_ready(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
898 let ns = target.namespace().unwrap();
899 let api = Api::<StatefulSet>::namespaced(ctx.client.clone(), &ns);
900 for rg in &target.spec.replica_groups {
901 let name = target.statefulset_name(&rg.name);
902 let sts = api
903 .get(&name)
904 .await
905 .map_err(|e| Error::kube_error("get", "StatefulSet", &ns, &name, e))?;
906 if sts
907 .status
908 .as_ref()
909 .and_then(|s| s.ready_replicas)
910 .unwrap_or(0)
911 != rg.replicas
912 {
913 return Ok(false);
914 }
915 }
916 Ok(true)
917}
918
919fn primary_pvc_name(target: &Kanidm) -> Result<String> {
920 let rg = primary_group(target)?;
921 Ok(format!(
922 "{DATA_VOLUME}-{}-0",
923 target.statefulset_name(&rg.name)
924 ))
925}
926
927async fn delete_secondary_pvcs(target: &Kanidm, ctx: &RestoreContext) -> Result<bool> {
928 let ns = target.namespace().unwrap();
929 let api = Api::<PersistentVolumeClaim>::namespaced(ctx.client.clone(), &ns);
930 let mut all_absent = true;
931 for rg in &target.spec.replica_groups {
932 let sts = target.statefulset_name(&rg.name);
933 for ordinal in 0..rg.replicas {
934 if rg.primary_node && ordinal == 0 {
935 continue;
936 }
937 let name = format!("{DATA_VOLUME}-{sts}-{ordinal}");
938 match api
939 .get_opt(&name)
940 .await
941 .map_err(|e| Error::kube_error("get", "PersistentVolumeClaim", &ns, &name, e))?
942 {
943 None => {}
944 Some(pvc) => {
945 all_absent = false;
946 if pvc.metadata.deletion_timestamp.is_none() {
947 match api.delete(&name, &DeleteParams::default()).await {
948 Ok(_) => debug!(pvc = %name, "deleting stale secondary PVC"),
949 Err(kube::Error::Api(status)) if status.code == 404 => {}
950 Err(error) => {
951 return Err(Error::kube_error(
952 "delete",
953 "PersistentVolumeClaim",
954 &ns,
955 &name,
956 error,
957 ));
958 }
959 }
960 }
961 }
962 }
963 }
964 }
965 Ok(all_absent)
966}
967
968fn config_map_name(restore: &KanidmRestore) -> String {
969 format!("{}-config", restore.name_any())
970}
971fn restore_job_name(restore: &KanidmRestore) -> String {
972 format!("{}-restore", restore.name_any())
973}
974fn verify_job_name(restore: &KanidmRestore) -> String {
975 format!("{}-verify", restore.name_any())
976}
977
978async fn ensure_restore_config(
979 restore: &KanidmRestore,
980 target: &Kanidm,
981 ctx: &RestoreContext,
982) -> Result<()> {
983 let ns = restore.namespace().unwrap();
984 let api = Api::<ConfigMap>::namespaced(ctx.client.clone(), &ns);
985 let name = config_map_name(restore);
986 if api
987 .get_opt(&name)
988 .await
989 .map_err(|e| Error::kube_error("get", "ConfigMap", &ns, &name, e))?
990 .is_some()
991 {
992 return Ok(());
993 }
994 let config = ConfigMap {
995 metadata: ObjectMeta {
996 name: Some(name.clone()),
997 namespace: Some(ns.clone()),
998 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
999 ..Default::default()
1000 },
1001 data: Some(std::collections::BTreeMap::from([(
1002 "server.toml".to_string(),
1003 "version = \"2\"\n".to_string(),
1004 )])),
1005 ..Default::default()
1006 };
1007 api.create(&PostParams::default(), &config)
1008 .await
1009 .map(|_| ())
1010 .map_err(|e| Error::kube_error("create", "ConfigMap", &ns, target.name_any(), e))
1011}
1012
1013async fn ensure_database_job(
1014 restore: &KanidmRestore,
1015 target: &Kanidm,
1016 ctx: &RestoreContext,
1017 name: &str,
1018 verify: bool,
1019) -> Result<()> {
1020 let ns = restore.namespace().unwrap();
1021 let api = Api::<Job>::namespaced(ctx.client.clone(), &ns);
1022 if api
1023 .get_opt(name)
1024 .await
1025 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?
1026 .is_some()
1027 {
1028 return Ok(());
1029 }
1030 let mut command = vec![
1031 "kanidmd".to_string(),
1032 "database".to_string(),
1033 if verify { "verify" } else { "restore" }.to_string(),
1034 "-c".to_string(),
1035 format!("{CONFIG_PATH}/server.toml"),
1036 ];
1037 if !verify {
1038 command.push(format!(
1039 "{BACKUP_PATH}/{}",
1040 restore.spec.source.local.file_name
1041 ));
1042 }
1043 let job = Job {
1044 metadata: ObjectMeta {
1045 name: Some(name.to_string()),
1046 namespace: Some(ns.clone()),
1047 owner_references: restore.controller_owner_ref(&()).map(|r| vec![r]),
1048 ..Default::default()
1049 },
1050 spec: Some(JobSpec {
1051 backoff_limit: Some(0),
1052 template: PodTemplateSpec {
1053 metadata: Some(ObjectMeta::default()),
1054 spec: Some(PodSpec {
1055 automount_service_account_token: Some(false),
1056 restart_policy: Some("Never".to_string()),
1057 containers: vec![Container {
1058 name: if verify { "verify" } else { "restore" }.to_string(),
1059 image: Some(restore.spec.restore_image.clone()),
1060 command: Some(command),
1061 env: Some(vec![
1062 EnvVar {
1063 name: "KANIDM_DB_PATH".to_string(),
1064 value: Some("/data/kanidm.db".to_string()),
1065 ..Default::default()
1066 },
1067 EnvVar {
1068 name: "KANIDM_DOMAIN".to_string(),
1069 value: Some(target.spec.domain.clone()),
1070 ..Default::default()
1071 },
1072 ]),
1073 volume_mounts: Some(vec![
1074 VolumeMount {
1075 name: DATA_VOLUME.to_string(),
1076 mount_path: DATA_PATH.to_string(),
1077 ..Default::default()
1078 },
1079 VolumeMount {
1080 name: CONFIG_VOLUME.to_string(),
1081 mount_path: CONFIG_PATH.to_string(),
1082 read_only: Some(true),
1083 ..Default::default()
1084 },
1085 ]),
1086 ..Default::default()
1087 }],
1088 volumes: Some(vec![
1089 Volume {
1090 name: DATA_VOLUME.to_string(),
1091 persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource {
1092 claim_name: primary_pvc_name(target)?,
1093 read_only: Some(false),
1094 }),
1095 ..Default::default()
1096 },
1097 Volume {
1098 name: CONFIG_VOLUME.to_string(),
1099 config_map: Some(ConfigMapVolumeSource {
1100 name: config_map_name(restore),
1101 ..Default::default()
1102 }),
1103 ..Default::default()
1104 },
1105 ]),
1106 ..Default::default()
1107 }),
1108 },
1109 ..Default::default()
1110 }),
1111 ..Default::default()
1112 };
1113 api.create(&PostParams::default(), &job)
1114 .await
1115 .map(|_| ())
1116 .map_err(|e| Error::kube_error("create", "Job", &ns, name, e))
1117}
1118
1119enum JobState {
1120 Running,
1121 Complete,
1122 Failed,
1123}
1124
1125async fn job_state(restore: &KanidmRestore, ctx: &RestoreContext, name: &str) -> Result<JobState> {
1126 let ns = restore.namespace().unwrap();
1127 let job = Api::<Job>::namespaced(ctx.client.clone(), &ns)
1128 .get(name)
1129 .await
1130 .map_err(|e| Error::kube_error("get", "Job", &ns, name, e))?;
1131 if job.status.as_ref().and_then(|s| s.failed).unwrap_or(0) > 0 {
1132 return Ok(JobState::Failed);
1133 }
1134 if job.status.as_ref().and_then(|s| s.succeeded).unwrap_or(0) > 0 {
1135 return Ok(JobState::Complete);
1136 }
1137 Ok(JobState::Running)
1138}
1139
1140#[cfg(test)]
1141mod tests {
1142 use super::{
1143 CONDITION_FAILED, CONDITION_READY, CONDITION_TRUE, KanidmRestorePhase, KanidmRestoreStatus,
1144 mutable_image, safe_basename, update_restore_conditions,
1145 };
1146
1147 #[test]
1148 fn mutation_boundary_defaults_to_fail_open_before_restore_starts() {
1149 assert!(!KanidmRestoreStatus::default().database_mutation_started);
1150 }
1151
1152 #[test]
1153 fn completed_restore_sets_ready_condition() {
1154 let mut status = KanidmRestoreStatus {
1155 phase: KanidmRestorePhase::Completed,
1156 ..Default::default()
1157 };
1158 update_restore_conditions(&mut status, Some(7));
1159 assert!(status.conditions.iter().any(|condition| {
1160 condition.type_ == CONDITION_READY && condition.status == CONDITION_TRUE
1161 }));
1162 assert!(!status.conditions.iter().any(|condition| {
1163 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
1164 }));
1165 }
1166
1167 #[test]
1168 fn failed_restore_sets_failed_condition() {
1169 let mut status = KanidmRestoreStatus {
1170 phase: KanidmRestorePhase::Failed,
1171 message: Some("verification failed".to_string()),
1172 ..Default::default()
1173 };
1174 update_restore_conditions(&mut status, Some(8));
1175 assert!(status.conditions.iter().any(|condition| {
1176 condition.type_ == CONDITION_FAILED && condition.status == CONDITION_TRUE
1177 }));
1178 }
1179
1180 #[test]
1181 fn restore_filename_is_confined_to_backup_directory() {
1182 assert!(safe_basename("backup.json.gz"));
1183 assert!(!safe_basename("../backup.json.gz"));
1184 assert!(!safe_basename("a/b"));
1185 assert!(!safe_basename(""));
1186 }
1187
1188 #[test]
1189 fn restore_image_rejects_latest_and_unpinned_names() {
1190 assert!(mutable_image("kanidm/server:latest"));
1191 assert!(mutable_image("kanidm/server"));
1192 assert!(!mutable_image("kanidm/server:1.10.0"));
1193 assert!(!mutable_image("kanidm/server@sha256:abc"));
1194 }
1195}