1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22 StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23 preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80 BTreeMap::from([
81 (NAME_LABEL.to_string(), "kanidm".to_string()),
82 (
83 MANAGED_BY_LABEL.to_string(),
84 format!("kaniop-{CONTROLLER_ID}"),
85 ),
86 ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90 ctx: Arc<Context>,
91 sts_api: &Api<StatefulSet>,
92 namespace: &str,
93 statefulset_names: &[String],
94 sequential: bool,
95) -> Result<()> {
96 let restart_futures = statefulset_names
99 .iter()
100 .map(|sts_name| sts_api.restart(sts_name));
101
102 if sequential {
103 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105 return Err(Error::kube_error(
106 "restart",
107 "StatefulSet",
108 namespace,
109 "replica certificate secret update statefulsets",
110 first_err,
111 ));
112 }
113 } else {
114 let results: Vec<_> = join_all(restart_futures).await;
115 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116 return Err(Error::kube_error(
117 "restart",
118 "StatefulSet",
119 namespace,
120 "replica certificate secret update statefulsets",
121 first_err,
122 ));
123 }
124 }
125
126 let rollout_futures = statefulset_names
127 .iter()
128 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129 try_join_all(rollout_futures).await?;
130
131 Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135 kanidm: Arc<Kanidm>,
136 ctx: Arc<Context>,
137 status: &KanidmStatus,
138) -> Result<bool> {
139 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141 kanidm.patch(&ctx, admins_secret).await?;
142 return Ok(true);
143 }
144 Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148 kanidm: Arc<Kanidm>,
149 ctx: Arc<Context>,
150 status: &KanidmStatus,
151) -> Result<bool> {
152 let mut changed = false;
153 let expected_secret_names = if kanidm.is_replication_enabled() {
154 status
155 .replica_statuses
156 .iter()
157 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158 .collect::<Vec<_>>()
159 } else {
160 Vec::new()
161 };
162 let replica_cert_label =
163 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
164 let deprecated_secrets = ctx
165 .stores
166 .secret_store
167 .state()
168 .into_iter()
169 .filter(|secret| {
170 secret.namespace() == kanidm.namespace()
171 && kanidm.admins_secret_name() != secret.name_any()
172 && !expected_secret_names.contains(&secret.name_any())
173 && secret.metadata.labels.as_ref().is_some_and(|l| {
174 l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
175 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
176 })
177 })
178 .collect::<Vec<_>>();
179 if !deprecated_secrets.is_empty() {
180 changed = true;
181 }
182 let secret_delete_future = deprecated_secrets
183 .iter()
184 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
185 .collect::<TryJoinAll<_>>();
186 try_join!(secret_delete_future)?;
187
188 if kanidm.is_replication_enabled() {
189 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
190 let statefulset_names = status
191 .replica_statuses
192 .iter()
193 .map(|rs| rs.statefulset_name.clone())
194 .collect::<BTreeSet<_>>()
195 .into_iter()
196 .collect::<Vec<_>>();
197
198 let has_pending = status
199 .replica_statuses
200 .iter()
201 .any(|rs| rs.state == KanidmReplicaState::Pending);
202
203 let sts_api =
204 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
205
206 if has_pending {
207 changed = true;
208 stream::iter(
209 status
210 .replica_statuses
211 .iter()
212 .filter(|rs| rs.state == KanidmReplicaState::Pending),
213 )
214 .then(|rs| async {
215 let secret = kanidm
216 .generate_replica_secret(ctx.clone(), &rs.pod_name)
217 .await?;
218 kanidm.patch(&ctx, secret).await
219 })
220 .collect::<Vec<_>>()
221 .await
222 .into_iter()
223 .collect::<Result<Vec<_>>>()?;
224
225 let restart_futures = status
226 .replica_statuses
227 .iter()
228 .map(|rs| sts_api.restart(&rs.statefulset_name));
229
230 if has_single_replica {
231 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
232 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
233 return Err(Error::kube_error(
234 "restart",
235 "StatefulSet",
236 kanidm.get_namespace(),
237 "pending replica statefulsets",
238 first_err,
239 ));
240 }
241 } else {
242 let results: Vec<_> = join_all(restart_futures).await;
243 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
244 return Err(Error::kube_error(
245 "restart",
246 "StatefulSet",
247 kanidm.get_namespace(),
248 "pending replica statefulsets",
249 first_err,
250 ));
251 }
252 }
253 }
254
255 let has_certificate_host_invalid = status
256 .replica_statuses
257 .iter()
258 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
259
260 let has_certificate_expiring = status
261 .replica_statuses
262 .iter()
263 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
264
265 let has_non_ready_replicas = status
266 .replica_statuses
267 .iter()
268 .any(|rs| rs.state != KanidmReplicaState::Ready);
269
270 let host_invalid_replicas: Vec<_> = status
271 .replica_statuses
272 .iter()
273 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
274 .collect();
275
276 let cert_expiring_replicas: Vec<_> = status
277 .replica_statuses
278 .iter()
279 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
280 .collect();
281
282 if !host_invalid_replicas.is_empty() {
283 changed = true;
284 let restart_futures = host_invalid_replicas
285 .iter()
286 .map(|rs| sts_api.restart(&rs.statefulset_name));
287
288 if has_single_replica {
289 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
290 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291 return Err(Error::kube_error(
292 "restart",
293 "StatefulSet",
294 kanidm.get_namespace(),
295 "certificate host invalid statefulsets",
296 first_err,
297 ));
298 }
299 } else {
300 let results: Vec<_> = join_all(restart_futures).await;
301 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
302 return Err(Error::kube_error(
303 "restart",
304 "StatefulSet",
305 kanidm.get_namespace(),
306 "certificate host invalid statefulsets",
307 first_err,
308 ));
309 }
310 }
311
312 let restarted_sts_names: Vec<_> = host_invalid_replicas
313 .iter()
314 .map(|rs| rs.statefulset_name.clone())
315 .collect::<BTreeSet<_>>()
316 .into_iter()
317 .collect();
318
319 let namespace = kanidm.get_namespace();
320 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
321 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
322 });
323 try_join_all(rollout_futures).await?;
324
325 let renew_futures = host_invalid_replicas
326 .iter()
327 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
328 let results: Vec<_> = join_all(renew_futures).await;
329 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
330 return Err(first_err);
331 }
332
333 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
334
335 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
336 let cert = show_replica_cert_with_retries(
337 &kanidm,
338 ctx.clone(),
339 &rs.pod_name,
340 CERT_SHOW_RETRY_ATTEMPTS,
341 )
342 .await?;
343 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
344 kanidm.patch(&ctx, secret.clone()).await
345 });
346 try_join_all(cert_futures).await?;
347
348 restart_statefulsets_after_replica_secret_update(
349 ctx.clone(),
350 &sts_api,
351 &kanidm.get_namespace(),
352 &statefulset_names,
353 has_single_replica,
354 )
355 .await?;
356
357 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
358 }
359
360 if !cert_expiring_replicas.is_empty() {
361 changed = true;
362 let renew_futures = cert_expiring_replicas
363 .iter()
364 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
365 let results: Vec<_> = join_all(renew_futures).await;
366 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
367 return Err(first_err);
368 }
369
370 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
371
372 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
373 let cert = show_replica_cert_with_retries(
374 &kanidm,
375 ctx.clone(),
376 &rs.pod_name,
377 CERT_SHOW_RETRY_ATTEMPTS,
378 )
379 .await?;
380 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
381 kanidm.patch(&ctx, secret.clone()).await
382 });
383 try_join_all(cert_futures).await?;
384
385 restart_statefulsets_after_replica_secret_update(
386 ctx.clone(),
387 &sts_api,
388 &kanidm.get_namespace(),
389 &statefulset_names,
390 has_single_replica,
391 )
392 .await?;
393
394 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
395 }
396
397 if has_non_ready_replicas
398 && !has_pending
399 && !has_certificate_host_invalid
400 && !has_certificate_expiring
401 {
402 changed = true;
403 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
404
405 let restart_futures = status
406 .replica_statuses
407 .iter()
408 .map(|rs| sts_api.restart(&rs.statefulset_name));
409
410 if has_single_replica {
411 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
412 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
413 return Err(Error::kube_error(
414 "restart",
415 "StatefulSet",
416 kanidm.get_namespace(),
417 "replica statefulsets",
418 first_err,
419 ));
420 }
421 } else {
422 let results: Vec<_> = join_all(restart_futures).await;
423 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
424 return Err(Error::kube_error(
425 "restart",
426 "StatefulSet",
427 kanidm.get_namespace(),
428 "replica statefulsets",
429 first_err,
430 ));
431 }
432 }
433 }
434 }
435
436 Ok(changed)
437}
438
439#[instrument(skip(ctx, kanidm))]
440pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
441 let trace_id = telemetry::get_trace_id();
442 Span::current().record("trace_id", field::display(&trace_id));
443 let _timer = ctx
444 .kaniop_ctx
445 .metrics
446 .reconcile_count_and_measure(&trace_id);
447 info!("reconciling Kanidm");
448
449 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
450 debug!(%e, "failed to reconcile status");
451 ctx.kaniop_ctx.metrics.status_update_errors_inc();
452 e
453 })?;
454 let kanidm_api: Api<Kanidm> =
455 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
456 let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
457 let outcome_clone = outcome.clone();
458 let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
459 let outcome = outcome_clone.clone();
460 let ctx = ctx.clone();
461 let status = status.clone();
462 async move {
463 match event {
464 Finalizer::Apply(kanidm) => {
465 let (action, changed) = reconcile(kanidm, ctx, status).await?;
466 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
467 Ok(action)
468 }
469 Finalizer::Cleanup(kanidm) => {
470 let (action, changed) = cleanup(kanidm, ctx).await?;
471 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
472 Ok(action)
473 }
474 }
475 }
476 })
477 .await
478 .or_else(|e| match e {
479 FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
480 debug!("resource already removed during finalizer cleanup");
481 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
482 }
483 _ => Err(Error::FinalizerError(
484 "failed on kanidm account finalizer".to_string(),
485 Box::new(e),
486 )),
487 })?;
488 let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
489 Ok((action, changed))
490}
491
492async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
496 let secret_name = kanidm.effective_tls_secret_name();
497 let secret_api =
498 Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
499 let secret = secret_api
500 .get_opt(&secret_name)
501 .await
502 .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
503 Ok(secret.as_ref().map(hash_secret_data))
504}
505
506fn previous_tls_secret_hash(
511 kanidm: &Kanidm,
512 stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
513) -> Option<String> {
514 stateful_sets
515 .into_iter()
516 .filter(|sts| {
517 sts.namespace() == kanidm.namespace()
518 && sts
519 .metadata
520 .labels
521 .as_ref()
522 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
523 })
524 .find_map(|sts| {
525 sts.spec
526 .as_ref()?
527 .template
528 .metadata
529 .as_ref()?
530 .annotations
531 .as_ref()?
532 .get(TLS_SECRET_HASH_ANNOTATION)
533 .cloned()
534 })
535}
536
537fn is_headless_service(service: &Service) -> bool {
538 service
539 .spec
540 .as_ref()
541 .and_then(|spec| spec.cluster_ip.as_deref())
542 == Some("None")
543}
544
545fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
546 if is_headless_service(desired) != is_headless_service(current) {
547 return;
548 }
549 let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
550 else {
551 return;
552 };
553 if desired_spec.cluster_ip.is_none() {
554 desired_spec.cluster_ip = current_spec.cluster_ip.clone();
555 }
556 if desired_spec.cluster_ips.is_none() {
557 desired_spec.cluster_ips = current_spec.cluster_ips.clone();
558 }
559}
560
561async fn wait_for_statefulset_deletion(
562 client: kube::Client,
563 namespace: &str,
564 name: &str,
565) -> Result<()> {
566 let api = Api::<StatefulSet>::namespaced(client, namespace);
567 let start = std::time::Instant::now();
568 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
569 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
570
571 loop {
572 match api.get_opt(name).await {
573 Ok(None) => return Ok(()),
574 Ok(Some(_)) => {}
575 Err(e) => {
576 return Err(Error::kube_error(
577 "wait for deletion of",
578 "StatefulSet",
579 namespace,
580 name,
581 e,
582 ));
583 }
584 }
585
586 if start.elapsed() >= timeout {
587 return Err(Error::ReceiveOutput(format!(
588 "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
589 )));
590 }
591 sleep(poll_interval).await;
592 }
593}
594
595async fn reconcile_service(
596 kanidm: &Kanidm,
597 ctx: &Context,
598 mut desired: Service,
599) -> Result<Service> {
600 let namespace = kanidm.get_namespace();
601 let name = desired.name_any();
602 let desired_headless = is_headless_service(&desired);
603
604 if let Some(cached) = ctx
605 .stores
606 .service_store
607 .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
608 {
609 if is_headless_service(cached.as_ref()) == desired_headless {
610 preserve_defaulted_service_fields(&mut desired, cached.as_ref());
611 } else {
612 let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
613 let current = service_api
614 .get_opt(&name)
615 .await
616 .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
617
618 if let Some(current) = current {
619 let current_headless = is_headless_service(¤t);
620 if current_headless != desired_headless {
621 info!(
622 resource.name = &name,
623 resource.namespace = &namespace,
624 current_headless,
625 desired_headless,
626 "recreating Service to change headless mode"
627 );
628 kanidm.delete(ctx, ¤t).await?;
629 ctx.kaniop_ctx
630 .metrics
631 .reconcile_deploy_delete_create_inc("Service", "headless_transition");
632 wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
633 .await?;
634 } else {
635 preserve_defaulted_service_fields(&mut desired, ¤t);
636 }
637 }
638 }
639 }
640
641 kanidm.patch(ctx, desired).await
642}
643
644async fn wait_for_service_deletion(
645 client: kube::Client,
646 namespace: &str,
647 name: &str,
648) -> Result<()> {
649 let api = Api::<Service>::namespaced(client, namespace);
650 let start = std::time::Instant::now();
651 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
652 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
653
654 loop {
655 match api.get_opt(name).await {
656 Ok(None) => return Ok(()),
657 Ok(Some(_)) => {}
658 Err(e) => {
659 return Err(Error::kube_error(
660 "wait for deletion of",
661 "Service",
662 namespace,
663 name,
664 e,
665 ));
666 }
667 }
668
669 if start.elapsed() >= timeout {
670 return Err(Error::ReceiveOutput(format!(
671 "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
672 )));
673 }
674 sleep(poll_interval).await;
675 }
676}
677
678async fn validate_statefulset_replacement(
679 api: &Api<StatefulSet>,
680 desired: &StatefulSet,
681 namespace: &str,
682 name: &str,
683) -> Result<()> {
684 let mut candidate = desired.clone();
685 candidate.metadata.name = None;
686 candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
687 candidate.metadata.namespace = None;
688 candidate.metadata.resource_version = None;
689 candidate.metadata.uid = None;
690 candidate.metadata.creation_timestamp = None;
691 candidate.metadata.generation = None;
692 candidate.metadata.managed_fields = None;
693 candidate.metadata.deletion_timestamp = None;
694 candidate.metadata.deletion_grace_period_seconds = None;
695 candidate.status = None;
696
697 api.create(
698 &PostParams {
699 dry_run: true,
700 ..Default::default()
701 },
702 &candidate,
703 )
704 .await
705 .map(|_| ())
706 .map_err(|e| {
707 Error::kube_error(
708 "validate replacement for",
709 "StatefulSet",
710 namespace,
711 name,
712 e,
713 )
714 })
715}
716
717async fn reconcile_statefulset(
718 kanidm: Arc<Kanidm>,
719 ctx: Arc<Context>,
720 desired: StatefulSet,
721) -> Result<StatefulSet> {
722 let namespace = kanidm.namespace().unwrap();
723 let name = desired.name_any();
724 let cached_current = ctx.stores.stateful_set_store.find(|current| {
725 current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
726 });
727
728 let cache_miss_422_error = if let Some(cached_current) = cached_current {
733 let mut cached_desired = desired.clone();
734 preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
735 if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
736 == StatefulSetApplyStrategy::Apply
737 {
738 return kanidm.apply(&ctx, cached_desired).await;
739 }
740 None
741 } else {
742 match kanidm.apply(&ctx, desired.clone()).await {
743 Ok(applied) => return Ok(applied),
744 Err(error) => {
745 if is_kube_422(&error) {
749 Some(error)
750 } else {
751 return Err(error);
752 }
753 }
754 }
755 };
756
757 let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
761 let Some(live_current) = api
762 .get_opt(&name)
763 .await
764 .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
765 else {
766 return match cache_miss_422_error {
767 Some(error) => Err(error),
768 None => kanidm.apply(&ctx, desired).await,
769 };
770 };
771
772 let mut live_desired = desired.clone();
775 preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
776 match classify_statefulset_change(&live_current, &live_desired) {
777 StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
778 Some(error) => Err(error),
779 None => kanidm.apply(&ctx, live_desired).await,
780 },
781 StatefulSetApplyStrategy::Recreate { .. } => {
782 validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
786
787 let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
790 Error::kube_error(
791 "re-read before deletion",
792 "StatefulSet",
793 &namespace,
794 &name,
795 e,
796 )
797 })?
798 else {
799 return kanidm.apply(&ctx, live_desired).await;
800 };
801 let mut latest_desired = live_desired.clone();
802 preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
803
804 match classify_statefulset_change(&latest_current, &latest_desired) {
805 StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
806 StatefulSetApplyStrategy::Recreate { immutable_fields } => {
807 validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
812 .await?;
813
814 info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
815 kanidm.delete(&ctx, &latest_current).await?;
816 ctx.kaniop_ctx
817 .metrics
818 .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
819 wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
820 .await?;
821 kanidm.apply(&ctx, latest_desired).await
822 }
823 }
824 }
825 }
826}
827
828fn is_kube_422(error: &Error) -> bool {
830 matches!(
831 error,
832 Error::KubeError(_, cause)
833 if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
834 )
835}
836
837#[cfg(test)]
838mod statefulset_recreate_recovery_tests {
839 use super::{Error, is_kube_422};
840
841 fn kube_api_error(code: u16) -> Error {
842 Error::KubeError(
843 "test error".to_string(),
844 Box::new(kube::Error::Api(Box::new(kube::error::Status {
845 code,
846 message: "test".to_string(),
847 reason: "test".to_string(),
848 ..Default::default()
849 }))),
850 )
851 }
852
853 #[test]
854 fn recreate_recovery_accepts_only_422() {
855 assert!(is_kube_422(&kube_api_error(422)));
856 assert!(!is_kube_422(&kube_api_error(403)));
857 assert!(!is_kube_422(&kube_api_error(429)));
858 assert!(!is_kube_422(&kube_api_error(500)));
859 }
860
861 #[test]
862 fn recreate_recovery_rejects_429_too_many_requests() {
863 assert!(!is_kube_422(&kube_api_error(429)));
864 }
865
866 #[test]
867 fn recreate_recovery_rejects_5xx_server_errors() {
868 assert!(!is_kube_422(&kube_api_error(500)));
869 assert!(!is_kube_422(&kube_api_error(502)));
870 assert!(!is_kube_422(&kube_api_error(503)));
871 assert!(!is_kube_422(&kube_api_error(504)));
872 }
873
874 #[test]
875 fn recreate_recovery_rejects_400_and_403() {
876 assert!(!is_kube_422(&kube_api_error(400)));
877 assert!(!is_kube_422(&kube_api_error(403)));
878 }
879}
880
881async fn reconcile(
882 kanidm: Arc<Kanidm>,
883 ctx: Arc<Context>,
884 status: KanidmStatus,
885) -> Result<(Action, bool)> {
886 if kanidm
887 .annotations()
888 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
889 {
890 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
891 return Ok((Action::requeue(Duration::from_secs(5)), false));
892 }
893
894 let mut changed = false;
895 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
896 let replication_secret_futures =
897 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
898
899 let sts_to_delete = {
900 let expected_sts_names = kanidm
901 .spec
902 .replica_groups
903 .iter()
904 .map(|rg| kanidm.statefulset_name(&rg.name))
905 .collect::<Vec<_>>();
906 ctx.stores
907 .stateful_set_store
908 .state()
909 .into_iter()
910 .filter(|sts| {
911 sts.namespace() == kanidm.namespace()
912 && sts
913 .metadata
914 .labels
915 .as_ref()
916 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
917 && !expected_sts_names.contains(&sts.name_any())
918 })
919 .collect::<Vec<_>>()
920 };
921 if !sts_to_delete.is_empty() {
922 changed = true;
923 }
924 let sts_delete_futures = sts_to_delete
925 .iter()
926 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
927 .collect::<TryJoinAll<_>>();
928
929 let sts_futures = match kanidm.is_updatable(&status) {
930 true => {
931 let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
932 previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
933 });
934
935 let namespace = kanidm.get_namespace();
936 let schedule_api: Api<KanidmBackupSchedule> =
937 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
938 let schedules = schedule_api
939 .list(&Default::default())
940 .await
941 .map_err(|e| {
942 Error::KubeError(
943 "failed to list KanidmBackupSchedules for backup configuration resolution"
944 .to_string(),
945 Box::new(e),
946 )
947 })?
948 .items;
949
950 let repo_api: Api<KanidmBackupRepository> =
951 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
952 let repositories = repo_api
953 .list(&Default::default())
954 .await
955 .map_err(|e| {
956 Error::KubeError(
957 "failed to list KanidmBackupRepositories for backup configuration resolution"
958 .to_string(),
959 Box::new(e),
960 )
961 })?
962 .items;
963 let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
964
965 kanidm
966 .spec
967 .replica_groups
968 .iter()
969 .map(|rg| {
970 let sts = kanidm.create_statefulset(
971 rg,
972 tls_secret_hash.as_deref(),
973 backup_config.as_ref(),
974 )?;
975 Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
976 })
977 .collect::<Result<TryJoinAll<_>, _>>()?
978 }
979 false => {
980 let note = match status.version.as_ref() {
981 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
982 format!(
983 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
984 v.image_tag,
985 crate::version::KANIDM_CLIENT_VERSION
986 )
987 }
988 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
989 };
990 let _ignore_error = ctx
991 .kaniop_ctx
992 .recorder
993 .publish(
994 &Event {
995 type_: EventType::Warning,
996 reason: "UpgradeBlocked".to_string(),
997 note: Some(note),
998 action: "ReconcileStatefulSet".to_string(),
999 secondary: None,
1000 },
1001 &kanidm.object_ref(&()),
1002 )
1003 .await
1004 .map_err(|e| {
1005 warn!(%e, "failed to publish KanidmError event");
1006 Error::KubeError("failed to publish event".to_string(), Box::new(e))
1007 });
1008 try_join_all([])
1009 }
1010 };
1011 let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1012
1013 let deprecated_rg_svcs = {
1014 let expected_rg_svcs_names = kanidm
1015 .spec
1016 .replica_groups
1017 .iter()
1018 .filter(|rg| rg.services.is_some())
1019 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1020 .collect::<Vec<_>>();
1021 ctx.stores
1022 .service_store
1023 .state()
1024 .into_iter()
1025 .filter(|svc| {
1026 let labels = svc.metadata.labels.as_ref();
1027 svc.namespace() == kanidm.namespace()
1028 && !expected_rg_svcs_names.contains(&svc.name_any())
1029 && labels.is_some_and(|l| {
1030 l.contains_key(REPLICA_LABEL)
1031 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1032 })
1033 })
1034 .collect::<Vec<_>>()
1035 };
1036 if !deprecated_rg_svcs.is_empty() {
1037 changed = true;
1038 }
1039 let rg_svcs_delete_futures = deprecated_rg_svcs
1040 .iter()
1041 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1042 .collect::<TryJoinAll<_>>();
1043
1044 let rg_services_futures = kanidm
1045 .spec
1046 .replica_groups
1047 .iter()
1048 .filter(|rg| rg.services.is_some())
1049 .flat_map(|rg| {
1050 (0..rg.replicas).map(|i| {
1051 let mut svc = kanidm.create_replica_group_service(rg, i);
1052 if let Some(current) = ctx.stores.service_store.find(|current| {
1053 current.namespace() == kanidm.namespace()
1054 && current.name_any() == svc.name_any()
1055 }) {
1056 preserve_defaulted_service_fields(&mut svc, current.as_ref());
1057 }
1058 kanidm.patch(&ctx, svc)
1059 })
1060 })
1061 .collect::<TryJoinAll<_>>();
1062
1063 let deprecated_ingresses = {
1064 let names = [
1065 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1066 kanidm.generate_region_ingress_name(),
1067 ]
1068 .into_iter()
1069 .flatten()
1070 .collect::<Vec<_>>();
1071 ctx.stores
1072 .ingress_store
1073 .state()
1074 .into_iter()
1075 .filter(|ing| {
1076 ing.namespace() == kanidm.namespace()
1077 && !names.contains(&ing.name_any())
1078 && ing.metadata.labels == Some(kanidm.generate_labels())
1079 })
1080 .collect::<Vec<_>>()
1081 };
1082 if !deprecated_ingresses.is_empty() {
1083 changed = true;
1084 }
1085 let ingress_delete_futures = deprecated_ingresses
1086 .iter()
1087 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1088 .collect::<TryJoinAll<_>>();
1089
1090 let ingress_futures = kanidm
1091 .create_ingress()
1092 .into_iter()
1093 .chain(kanidm.create_region_ingress())
1094 .map(|ingress| kanidm.patch(&ctx, ingress))
1095 .collect::<TryJoinAll<_>>();
1096
1097 let deprecated_http_routes = match &ctx.stores.http_route_store {
1098 Some(store) => {
1099 let expected_names = kanidm
1100 .spec
1101 .gateway
1102 .as_ref()
1103 .map(|_| vec![kanidm.name_any()])
1104 .unwrap_or_default();
1105 let routes: Vec<_> = store
1106 .state()
1107 .into_iter()
1108 .filter(|route| {
1109 route.namespace() == kanidm.namespace()
1110 && !expected_names.contains(&route.name_any())
1111 && route.metadata.labels == Some(kanidm.generate_labels())
1112 })
1113 .collect();
1114 if !routes.is_empty() {
1115 changed = true;
1116 }
1117 routes
1118 }
1119 None => Vec::new(),
1120 };
1121 let http_route_delete_futures = deprecated_http_routes
1122 .iter()
1123 .map(|route| kanidm.delete(&ctx, route.as_ref()))
1124 .collect::<TryJoinAll<_>>();
1125
1126 let http_route_futures = if ctx.stores.http_route_store.is_some() {
1127 kanidm
1128 .create_http_route()
1129 .into_iter()
1130 .map(|route| kanidm.patch(&ctx, route))
1131 .collect::<TryJoinAll<_>>()
1132 } else {
1133 try_join_all(Vec::new())
1134 };
1135
1136 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1137 Some(store) => {
1138 let expected_names = kanidm
1139 .spec
1140 .gateway
1141 .as_ref()
1142 .and_then(|g| g.backend_tls_policy.as_ref())
1143 .map(|_| vec![kanidm.name_any()])
1144 .unwrap_or_default();
1145 let policies: Vec<_> = store
1146 .state()
1147 .into_iter()
1148 .filter(|policy| {
1149 policy.namespace() == kanidm.namespace()
1150 && !expected_names.contains(&policy.name_any())
1151 && policy.metadata.labels == Some(kanidm.generate_labels())
1152 })
1153 .collect();
1154 if !policies.is_empty() {
1155 changed = true;
1156 }
1157 policies
1158 }
1159 None => Vec::new(),
1160 };
1161 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1162 .iter()
1163 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1164 .collect::<TryJoinAll<_>>();
1165
1166 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1167 kanidm
1168 .create_backend_tls_policy()
1169 .into_iter()
1170 .map(|policy| kanidm.patch(&ctx, policy))
1171 .collect::<TryJoinAll<_>>()
1172 } else {
1173 try_join_all(Vec::new())
1174 };
1175
1176 let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1177 sts_delete_futures,
1178 admin_secret_future,
1179 replication_secret_futures,
1180 sts_futures,
1181 service_future,
1182 rg_svcs_delete_futures,
1183 rg_services_futures,
1184 ingress_delete_futures,
1185 ingress_futures,
1186 http_route_delete_futures,
1187 http_route_futures,
1188 backend_tls_policy_delete_futures,
1189 backend_tls_policy_futures
1190 )?;
1191
1192 if admin_secret_changed || replication_secrets_changed {
1193 changed = true;
1194 }
1195
1196 if is_kanidm_available(status.clone()) {
1197 let namespace = kanidm.namespace().unwrap();
1198 let name = kanidm.name_any();
1199 match crate::controller::kanidm::KanidmClients::create_client(
1200 &namespace,
1201 &name,
1202 crate::controller::kanidm::KanidmUser::Admin,
1203 ctx.kaniop_ctx.client.clone(),
1204 )
1205 .await
1206 {
1207 Ok(system_client) => {
1208 if let Err(e) =
1209 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1210 {
1211 warn!(%e, "failed to reconcile domain appearance");
1212 }
1213 }
1214 Err(e) => {
1215 warn!(%e, "failed to create admin client for domain appearance reconciliation");
1216 }
1217 }
1218
1219 match crate::controller::kanidm::KanidmClients::create_client(
1220 &namespace,
1221 &name,
1222 crate::controller::kanidm::KanidmUser::IdmAdmin,
1223 ctx.kaniop_ctx.client.clone(),
1224 )
1225 .await
1226 {
1227 Ok(kanidm_client) => {
1228 let (mail_sender_status, mail_sender_mutated) =
1229 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1230 .await
1231 .unwrap_or_else(|e| {
1232 warn!(%e, "failed to reconcile mail sender");
1233 (None, false)
1234 });
1235 if mail_sender_mutated {
1236 changed = true;
1237 }
1238 let current_mail_sender_status =
1239 kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1240 if mail_sender_status != current_mail_sender_status {
1241 let kanidm_api: Api<Kanidm> =
1242 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1243 let status_patch = serde_json::json!({
1244 "status": {
1245 "mailSender": mail_sender_status
1246 }
1247 });
1248 if let Err(e) = kanidm_api
1249 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1250 .await
1251 {
1252 warn!(%e, "failed to patch Kanidm/status for mail sender");
1253 }
1254 }
1255 }
1256 Err(e) => {
1257 warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1258 }
1259 }
1260 }
1261
1262 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1263}
1264
1265async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1266 debug!("cleanup");
1267
1268 let mut changed = false;
1269 let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1270 if mail_resources_cleaned {
1271 changed = true;
1272 }
1273
1274 let namespace = kanidm.namespace().unwrap();
1275 let name = kanidm.name_any();
1276
1277 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1278 &namespace,
1279 &name,
1280 crate::controller::kanidm::KanidmUser::IdmAdmin,
1281 ctx.kaniop_ctx.client.clone(),
1282 )
1283 .await
1284 {
1285 let kanidm_cleanup_done =
1286 cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1287 if kanidm_cleanup_done {
1288 changed = true;
1289 }
1290 }
1291
1292 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1293 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1294}
1295
1296async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1297 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1298 let start = std::time::Instant::now();
1299 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1300 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1301
1302 loop {
1303 let sts = sts_api.get(sts_name).await.map_err(|e| {
1304 Error::KubeError(
1305 format!("failed to get StatefulSet {namespace}/{sts_name}"),
1306 Box::new(e),
1307 )
1308 })?;
1309
1310 if sts.metadata.deletion_timestamp.is_some() {
1311 return Err(Error::ReceiveOutput(format!(
1312 "StatefulSet {namespace}/{sts_name} is being deleted"
1313 )));
1314 }
1315
1316 let spec = sts.spec.as_ref().ok_or_else(|| {
1317 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1318 })?;
1319
1320 let status = match sts.status {
1321 Some(ref s) => s,
1322 None => {
1323 if start.elapsed() >= timeout {
1324 return Err(Error::ReceiveOutput(format!(
1325 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1326 )));
1327 }
1328 sleep(poll_interval).await;
1329 continue;
1330 }
1331 };
1332
1333 let replicas = spec.replicas.unwrap_or(0);
1334
1335 if replicas == 0 {
1336 return Ok(());
1337 }
1338
1339 let updated_replicas = status.updated_replicas.unwrap_or(0);
1340 let current_revision = status.current_revision.as_deref().unwrap_or("");
1341 let update_revision = status.update_revision.as_deref().unwrap_or("");
1342 let ready_replicas = status.ready_replicas.unwrap_or(0);
1343
1344 if updated_replicas == replicas
1345 && current_revision == update_revision
1346 && ready_replicas == replicas
1347 {
1348 return Ok(());
1349 }
1350
1351 if start.elapsed() >= timeout {
1352 return Err(Error::ReceiveOutput(format!(
1353 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1354 )));
1355 }
1356
1357 sleep(poll_interval).await;
1358 }
1359}
1360
1361async fn show_replica_cert_with_retries(
1362 kanidm: &Kanidm,
1363 ctx: Arc<Context>,
1364 pod_name: &str,
1365 max_attempts: u32,
1366) -> std::result::Result<String, Error> {
1367 let mut last_error = None;
1368 for attempt in 0..max_attempts {
1369 if attempt > 0 {
1370 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1371 }
1372 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1373 Ok(cert) => return Ok(cert),
1374 Err(e) => {
1375 info!(
1376 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1377 attempt + 1,
1378 max_attempts
1379 );
1380 last_error = Some(e);
1381 }
1382 }
1383 }
1384 Err(last_error.unwrap_or_else(|| {
1385 Error::ReceiveOutput(format!(
1386 "failed to get certificate for {pod_name} after {max_attempts} attempts"
1387 ))
1388 }))
1389}
1390
1391impl Kanidm {
1392 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1394 where
1395 K: Resource<Scope = NamespaceResourceScope>
1396 + Serialize
1397 + Clone
1398 + std::fmt::Debug
1399 + for<'de> Deserialize<'de>,
1400 <K as kube::Resource>::DynamicType: Default,
1401 <K as Resource>::Scope: std::marker::Sized,
1402 {
1403 self.kube_delete(
1404 ctx.kaniop_ctx.client.clone(),
1405 &ctx.kaniop_ctx.metrics,
1406 resource,
1407 )
1408 .await
1409 }
1410
1411 pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1414 where
1415 K: Resource<Scope = NamespaceResourceScope>
1416 + Serialize
1417 + Clone
1418 + std::fmt::Debug
1419 + for<'de> Deserialize<'de>,
1420 <K as kube::Resource>::DynamicType: Default,
1421 <K as Resource>::Scope: std::marker::Sized,
1422 {
1423 self.kube_apply(
1424 ctx.kaniop_ctx.client.clone(),
1425 resource,
1426 KANIDM_OPERATOR_NAME,
1427 )
1428 .await
1429 }
1430
1431 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1432 where
1433 K: Resource<Scope = NamespaceResourceScope>
1434 + Serialize
1435 + Clone
1436 + std::fmt::Debug
1437 + for<'de> Deserialize<'de>,
1438 <K as kube::Resource>::DynamicType: Default,
1439 <K as Resource>::Scope: std::marker::Sized,
1440 {
1441 self.kube_patch(
1442 ctx.kaniop_ctx.client.clone(),
1443 &ctx.kaniop_ctx.metrics,
1444 resource,
1445 KANIDM_OPERATOR_NAME,
1446 )
1447 .await
1448 }
1449
1450 #[inline]
1451 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1452 LABELS
1453 .clone()
1454 .into_iter()
1455 .chain([
1456 (INSTANCE_LABEL.to_string(), self.name_any()),
1457 (CLUSTER_LABEL.to_string(), self.name_any()),
1458 ])
1459 .collect()
1460 }
1461
1462 #[inline]
1463 fn generate_labels(&self) -> BTreeMap<String, String> {
1464 self.generate_resource_labels()
1465 }
1466
1467 #[inline]
1468 fn get_tls_secret_name(&self) -> String {
1469 format!("{}-tls", self.name_any())
1470 }
1471
1472 pub(crate) fn effective_tls_secret_name(&self) -> String {
1475 self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1476 self.spec
1477 .ingress
1478 .as_ref()
1479 .and_then(|i| i.tls_secret_name.clone())
1480 .unwrap_or_else(|| self.get_tls_secret_name())
1481 })
1482 }
1483
1484 #[inline]
1485 fn get_namespace(&self) -> String {
1486 self.namespace().unwrap()
1488 }
1489
1490 #[inline]
1491 fn is_replication_enabled(&self) -> bool {
1492 self.spec.replica_groups.len() > 1
1493 || self
1494 .spec
1495 .replica_groups
1496 .first()
1497 .is_some_and(|rg| rg.replicas > 1)
1498 || !self.spec.external_replication_nodes.is_empty()
1499 }
1500
1501 #[inline]
1502 fn is_updatable(&self, status: &KanidmStatus) -> bool {
1503 status
1504 .version
1505 .as_ref()
1506 .map(|v| {
1507 if v.compatibility_result == VersionCompatibilityResult::Incompatible {
1508 return false;
1509 }
1510 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1511 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
1512 match v.upgrade_check_result {
1513 KanidmUpgradeCheckResult::Passed => {
1514 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1515 minor <= v_minor + 1
1516 } else {
1517 true
1518 }
1519 }
1520 KanidmUpgradeCheckResult::Failed => {
1521 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1522 versions
1523 {
1524 major == v_major && minor == v_minor && patch >= v_patch
1525 } else {
1526 v.image_tag == current_tag
1527 }
1528 }
1529 }
1530 })
1531 .unwrap_or(true)
1532 }
1533
1534 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1535 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1536 let pod = pod_api.get(pod_name).await.map_err(|e| {
1537 Error::KubeError(
1538 format!("failed to get pod {namespace}/{pod_name}"),
1539 Box::new(e),
1540 )
1541 })?;
1542 Ok(pod
1543 .status
1544 .as_ref()
1545 .and_then(|s| s.conditions.as_ref())
1546 .is_some_and(|conditions| {
1547 conditions
1548 .iter()
1549 .any(|c| c.type_ == "Ready" && c.status == "True")
1550 }))
1551 }
1552
1553 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1554 let start = std::time::Instant::now();
1555 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1556 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1557
1558 loop {
1559 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1560 Ok(true) => return Ok(()),
1561 Ok(false) => {}
1562 Err(e) => {
1563 if !e.is_retryable() {
1564 return Err(e);
1565 }
1566 trace!(%e, "pod readiness check failed, retrying");
1567 }
1568 }
1569
1570 if start.elapsed() >= timeout {
1571 return Err(Error::ReceiveOutput(format!(
1572 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1573 )));
1574 }
1575
1576 sleep(poll_interval).await;
1577 }
1578 }
1579
1580 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1581 where
1582 I: IntoIterator<Item = T> + Debug,
1583 T: Into<String>,
1584 {
1585 let namespace = &self.get_namespace();
1586 trace!(
1587 resource.name = &pod_name,
1588 resource.namespace = &namespace,
1589 ?command,
1590 "pod exec"
1591 );
1592 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1593 let attached = pod
1594 .exec(
1595 pod_name,
1596 command,
1597 &AttachParams::default().container("kanidm"),
1598 )
1599 .await
1600 .map_err(|e| {
1601 Error::KubeError(
1602 format!("failed to exec pod {namespace}/{pod_name}"),
1603 Box::new(e),
1604 )
1605 })?;
1606 get_output(attached).await
1607 }
1608
1609 async fn exec_with_wait<I, T>(
1610 &self,
1611 ctx: Arc<Context>,
1612 pod_name: &str,
1613 command: I,
1614 ) -> Result<String>
1615 where
1616 I: IntoIterator<Item = T> + Debug,
1617 T: Into<String>,
1618 {
1619 let namespace = &self.get_namespace();
1620 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1621 self.exec(ctx, pod_name, command).await
1622 }
1623
1624 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1625 where
1626 I: IntoIterator<Item = T> + Debug,
1627 T: Into<String>,
1628 {
1629 let first_replica_group = self
1631 .spec
1632 .replica_groups
1633 .first()
1634 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1635 let sts_name = self.statefulset_name(&first_replica_group.name);
1636 let pod_name = format!("{sts_name}-0");
1637 self.exec(ctx, &pod_name, command).await
1638 }
1639
1640 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1641 where
1642 I: IntoIterator<Item = T> + Debug,
1643 T: Into<String>,
1644 {
1645 let first_replica_group = self
1646 .spec
1647 .replica_groups
1648 .first()
1649 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1650 let sts_name = self.statefulset_name(&first_replica_group.name);
1651 let pod_name = format!("{sts_name}-0");
1652 self.exec_with_wait(ctx, &pod_name, command).await
1653 }
1654}
1655
1656#[cfg(test)]
1657mod test {
1658 use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1659 use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1660
1661 use crate::controller::State;
1662 use crate::kanidm::controller::context::{Context, Stores};
1663 use crate::kanidm::crd::KanidmStatus;
1664
1665 use kaniop_k8s_util::error::Result;
1666 use kaniop_k8s_util::resources::hash_secret_data;
1667
1668 use std::collections::BTreeMap;
1669 use std::sync::Arc;
1670
1671 use http::{Request, Response};
1672 use k8s_openapi::ByteString;
1673 use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1674 use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1675 use k8s_openapi::api::networking::v1::Ingress;
1676 use kube::api::ObjectMeta;
1677 use kube::runtime::reflector::store::Writer;
1678 use kube::{Client, Resource, ResourceExt, client::Body};
1679 use opentelemetry::metrics::MeterProvider;
1680 use serde_json::json;
1681
1682 impl Kanidm {
1683 pub fn test() -> Self {
1685 let mut e = Kanidm::new(
1686 "test",
1687 serde_json::from_value(json!({
1688 "domain": "idm.example.com",
1689 "replicaGroups": [
1690 {
1691 "name": "default",
1692 "replicas": 1
1693 }
1694 ]
1695 }))
1696 .unwrap(),
1697 );
1698 e.meta_mut().namespace = Some("default".into());
1699 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1702 e
1703 }
1704
1705 pub fn with_ingress(mut self) -> Self {
1706 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1707 self
1708 }
1709
1710 pub fn with_replicas(mut self, replicas: i32) -> Self {
1712 self.spec.replica_groups[0].replicas = replicas;
1713 self
1714 }
1715
1716 pub fn needs_delete(mut self) -> Self {
1718 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1719 use k8s_openapi::jiff::Timestamp;
1720 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1722 self
1723 }
1724
1725 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1727 self.status = Some(status);
1728 self
1729 }
1730 }
1731
1732 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1734 pub struct ApiServerVerifier(ApiServerHandle);
1735
1736 pub enum Scenario {
1738 Create(Kanidm),
1739 CreateWithTwoReplicas(Kanidm),
1740 CreateWithIngress(Kanidm),
1741 CreateWithIngressWithTwoReplicas(Kanidm),
1742 CreateWithTlsSecret(Kanidm, Box<Secret>),
1743 }
1744
1745 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1746 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1747 .await
1748 .expect("timeout on mock apiserver")
1749 .expect("scenario succeeded")
1750 }
1751
1752 impl ApiServerVerifier {
1753 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1764 tokio::spawn(async move {
1765 match scenario {
1767 Scenario::Create(kanidm) => {
1768 self.handle_kanidm_get(kanidm.clone())
1769 .await
1770 .unwrap()
1771 .handle_kanidm_status_patch(kanidm.clone())
1772 .await
1773 .unwrap()
1774 .handle_tls_secret_get_not_found(kanidm.clone())
1775 .await
1776 .unwrap()
1777 .handle_backup_schedule_list()
1778 .await
1779 .unwrap()
1780 .handle_backup_repository_list()
1781 .await
1782 .unwrap()
1783 .handle_statefulset_patch(kanidm.clone(), None)
1784 .await
1785 .unwrap()
1786 .handle_service_patch(kanidm.clone())
1787 .await
1788 }
1789 Scenario::CreateWithTwoReplicas(kanidm) => {
1790 self.handle_kanidm_get(kanidm.clone())
1791 .await
1792 .unwrap()
1793 .handle_kanidm_status_patch(kanidm.clone())
1794 .await
1795 .unwrap()
1796 .handle_tls_secret_get_not_found(kanidm.clone())
1797 .await
1798 .unwrap()
1799 .handle_backup_schedule_list()
1800 .await
1801 .unwrap()
1802 .handle_backup_repository_list()
1803 .await
1804 .unwrap()
1805 .handle_statefulset_patch(kanidm.clone(), None)
1806 .await
1807 .unwrap()
1808 .handle_service_patch(kanidm.clone())
1809 .await
1810 }
1811 Scenario::CreateWithIngress(kanidm) => {
1812 self.handle_kanidm_get(kanidm.clone())
1813 .await
1814 .unwrap()
1815 .handle_kanidm_status_patch(kanidm.clone())
1816 .await
1817 .unwrap()
1818 .handle_tls_secret_get_not_found(kanidm.clone())
1819 .await
1820 .unwrap()
1821 .handle_backup_schedule_list()
1822 .await
1823 .unwrap()
1824 .handle_backup_repository_list()
1825 .await
1826 .unwrap()
1827 .handle_statefulset_patch(kanidm.clone(), None)
1828 .await
1829 .unwrap()
1830 .handle_service_patch(kanidm.clone())
1831 .await
1832 .unwrap()
1833 .handle_ingress_patch(kanidm.clone())
1834 .await
1835 }
1836 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1837 self.handle_kanidm_get(kanidm.clone())
1838 .await
1839 .unwrap()
1840 .handle_kanidm_status_patch(kanidm.clone())
1841 .await
1842 .unwrap()
1843 .handle_tls_secret_get_not_found(kanidm.clone())
1844 .await
1845 .unwrap()
1846 .handle_backup_schedule_list()
1847 .await
1848 .unwrap()
1849 .handle_backup_repository_list()
1850 .await
1851 .unwrap()
1852 .handle_statefulset_patch(kanidm.clone(), None)
1853 .await
1854 .unwrap()
1855 .handle_service_patch(kanidm.clone())
1856 .await
1857 .unwrap()
1858 .handle_ingress_patch(kanidm.clone())
1859 .await
1860 }
1861 Scenario::CreateWithTlsSecret(kanidm, secret) => {
1862 let expected_hash = hash_secret_data(&secret);
1863 self.handle_kanidm_get(kanidm.clone())
1864 .await
1865 .unwrap()
1866 .handle_kanidm_status_patch(kanidm.clone())
1867 .await
1868 .unwrap()
1869 .handle_tls_secret_get(kanidm.clone(), *secret)
1870 .await
1871 .unwrap()
1872 .handle_backup_schedule_list()
1873 .await
1874 .unwrap()
1875 .handle_backup_repository_list()
1876 .await
1877 .unwrap()
1878 .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1879 .await
1880 .unwrap()
1881 .handle_service_patch(kanidm.clone())
1882 .await
1883 }
1884 }
1885 .expect("scenario completed without errors");
1886 })
1887 }
1888
1889 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1890 let (request, send) = self.0.next_request().await.expect("service not called");
1891 assert_eq!(request.method(), http::Method::GET);
1892 let expected_uri_prefix = format!(
1893 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1894 kanidm.name_any()
1895 );
1896 let uri = request.uri().to_string();
1897 assert!(
1898 uri.starts_with(&expected_uri_prefix),
1899 "expected uri to start with {}, got {}",
1900 expected_uri_prefix,
1901 uri
1902 );
1903 let response = serde_json::to_vec(&kanidm).unwrap();
1904 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1905 Ok(self)
1906 }
1907
1908 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1909 let (request, send) = self.0.next_request().await.expect("service not called");
1910 assert_eq!(request.method(), http::Method::PATCH);
1911 assert_eq!(
1912 request.uri().to_string(),
1913 format!(
1914 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1915 kanidm.name_any()
1916 )
1917 );
1918
1919 let req_body = request.into_body().collect_bytes().await.unwrap();
1920 let json: serde_json::Value =
1921 serde_json::from_slice(&req_body).expect("patch object is json");
1922 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1923 .expect("valid kanidm status");
1924 let mut kanidm = Kanidm::test();
1925 kanidm.status = Some(status.clone());
1926 let response = serde_json::to_vec(&kanidm).unwrap();
1927 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1929 Ok(self)
1930 }
1931
1932 async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1933 let (request, send) = self.0.next_request().await.expect("service not called");
1934 assert_eq!(request.method(), http::Method::GET);
1935 let expected_uri_prefix = format!(
1936 "/api/v1/namespaces/default/secrets/{}",
1937 kanidm.effective_tls_secret_name()
1938 );
1939 let uri = request.uri().to_string();
1940 assert!(
1941 uri.starts_with(&expected_uri_prefix),
1942 "expected uri to start with {expected_uri_prefix}, got {uri}"
1943 );
1944 let status = serde_json::json!({
1945 "kind": "Status",
1946 "apiVersion": "v1",
1947 "metadata": {},
1948 "status": "Failure",
1949 "message": "secret not found",
1950 "reason": "NotFound",
1951 "code": 404
1952 });
1953 send.send_response(
1954 Response::builder()
1955 .status(404)
1956 .body(Body::from(serde_json::to_vec(&status).unwrap()))
1957 .unwrap(),
1958 );
1959 Ok(self)
1960 }
1961
1962 async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1963 let (request, send) = self.0.next_request().await.expect("service not called");
1964 assert_eq!(request.method(), http::Method::GET);
1965 let expected_uri_prefix = format!(
1966 "/api/v1/namespaces/default/secrets/{}",
1967 kanidm.effective_tls_secret_name()
1968 );
1969 let uri = request.uri().to_string();
1970 assert!(
1971 uri.starts_with(&expected_uri_prefix),
1972 "expected uri to start with {expected_uri_prefix}, got {uri}"
1973 );
1974 let response = serde_json::to_vec(&secret).unwrap();
1975 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1976 Ok(self)
1977 }
1978
1979 async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1980 let (request, send) = self.0.next_request().await.expect("service not called");
1981 assert_eq!(request.method(), http::Method::GET);
1982 assert!(
1983 request
1984 .uri()
1985 .to_string()
1986 .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1987 "expected backup schedule list request, got {}",
1988 request.uri()
1989 );
1990 let list = json!({
1991 "apiVersion": "kaniop.rs/v1alpha1",
1992 "kind": "KanidmBackupScheduleList",
1993 "metadata": {},
1994 "items": []
1995 });
1996 let response = serde_json::to_vec(&list).unwrap();
1997 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1998 Ok(self)
1999 }
2000
2001 async fn handle_backup_repository_list(mut self) -> Result<Self> {
2002 let (request, send) = self.0.next_request().await.expect("service not called");
2003 assert_eq!(request.method(), http::Method::GET);
2004 assert!(
2005 request.uri().to_string().contains(
2006 "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2007 ),
2008 "expected backup repository list request, got {}",
2009 request.uri()
2010 );
2011 let list = json!({
2012 "apiVersion": "kaniop.rs/v1alpha1",
2013 "kind": "KanidmBackupRepositoryList",
2014 "metadata": {},
2015 "items": []
2016 });
2017 let response = serde_json::to_vec(&list).unwrap();
2018 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2019 Ok(self)
2020 }
2021
2022 async fn handle_statefulset_patch(
2023 mut self,
2024 kanidm: Kanidm,
2025 expected_tls_hash: Option<String>,
2026 ) -> Result<Self> {
2027 for rg in kanidm.spec.replica_groups.iter() {
2028 let (request, send) = self.0.next_request().await.expect("service not called");
2029 assert_eq!(request.method(), http::Method::PATCH);
2030 assert_eq!(
2031 request.uri().to_string(),
2032 format!(
2033 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2034 kanidm.statefulset_name(&rg.name)
2035 )
2036 );
2037 let req_body = request.into_body().collect_bytes().await.unwrap();
2038 let json: serde_json::Value =
2039 serde_json::from_slice(&req_body).expect("patch object is json");
2040 let statefulset: StatefulSet =
2041 serde_json::from_value(json).expect("valid statefulset");
2042 assert_eq!(
2043 statefulset.clone().spec.unwrap().replicas.unwrap(),
2044 rg.replicas
2045 );
2046 let tls_hash_annotation = statefulset
2047 .spec
2048 .as_ref()
2049 .unwrap()
2050 .template
2051 .metadata
2052 .as_ref()
2053 .unwrap()
2054 .annotations
2055 .as_ref()
2056 .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2057 .cloned();
2058 assert_eq!(tls_hash_annotation, expected_tls_hash);
2059 let response = serde_json::to_vec(&statefulset).unwrap();
2060 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2062 }
2063
2064 Ok(self)
2065 }
2066
2067 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2068 let (request, send) = self.0.next_request().await.expect("service not called");
2069 assert_eq!(request.method(), http::Method::PATCH);
2070 assert_eq!(
2071 request.uri().to_string(),
2072 format!(
2073 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2074 kanidm.name_any()
2075 )
2076 );
2077
2078 let req_body = request.into_body().collect_bytes().await.unwrap();
2079 let json: serde_json::Value =
2080 serde_json::from_slice(&req_body).expect("patch object is json");
2081 let service: Service = serde_json::from_value(json).expect("valid service");
2082 let response = serde_json::to_vec(&service).unwrap();
2083 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2085 Ok(self)
2086 }
2087
2088 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2089 let (request, send) = self.0.next_request().await.expect("service not called");
2090 assert_eq!(request.method(), http::Method::PATCH);
2091 assert_eq!(
2092 request.uri().to_string(),
2093 format!(
2094 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2095 kanidm.name_any()
2096 )
2097 );
2098
2099 let req_body = request.into_body().collect_bytes().await.unwrap();
2100 let json: serde_json::Value =
2101 serde_json::from_slice(&req_body).expect("patch object is json");
2102 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2103 let response = serde_json::to_vec(&ingress).unwrap();
2104 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2106 Ok(self)
2107 }
2108 }
2109
2110 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2111 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2112 let mock_client = Client::new(mock_service, "default");
2113 let stores = Stores {
2114 stateful_set_store: Writer::default().as_reader(),
2115 service_store: Writer::default().as_reader(),
2116 ingress_store: Writer::default().as_reader(),
2117 secret_store: Writer::default().as_reader(),
2118 http_route_store: Some(Writer::default().as_reader()),
2119 backend_tls_policy_store: Some(Writer::default().as_reader()),
2120 deployment_store: Writer::default().as_reader(),
2121 config_map_store: Writer::default().as_reader(),
2122 };
2123 let controller_id = "test";
2124
2125 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2127 let meter = provider.meter("test");
2128 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2129
2130 let state = State::new(
2131 metrics,
2132 Writer::default().as_reader(),
2133 Writer::default().as_reader(),
2134 None,
2135 );
2136 let ctx = Arc::new(Context::new(
2137 state.to_context(mock_client, controller_id),
2138 stores,
2139 ));
2140 (ctx, ApiServerVerifier(handle))
2141 }
2142
2143 #[tokio::test]
2144 async fn kanidm_create() {
2145 let (testctx, fakeserver) = get_test_context();
2146 let kanidm = Kanidm::test();
2147 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2148 reconcile_kanidm(Arc::new(kanidm), testctx)
2149 .await
2150 .expect("reconciler");
2151 timeout_after_1s(mocksrv).await;
2152 }
2153
2154 #[tokio::test]
2155 async fn kanidm_create_with_two_replicas() {
2156 let (testctx, fakeserver) = get_test_context();
2157 let kanidm = Kanidm::test().with_replicas(2);
2158 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2159 reconcile_kanidm(Arc::new(kanidm), testctx)
2160 .await
2161 .expect("reconciler");
2162 timeout_after_1s(mocksrv).await;
2163 }
2164
2165 #[tokio::test]
2166 async fn kanidm_create_with_ingress() {
2167 let (testctx, fakeserver) = get_test_context();
2168 let kanidm = Kanidm::test().with_ingress();
2169 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2170 reconcile_kanidm(Arc::new(kanidm), testctx)
2171 .await
2172 .expect("reconciler");
2173 timeout_after_1s(mocksrv).await;
2174 }
2175
2176 #[tokio::test]
2177 async fn kanidm_create_with_ingress_with_two_replicas() {
2178 let (testctx, fakeserver) = get_test_context();
2179 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2180 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2181 reconcile_kanidm(Arc::new(kanidm), testctx)
2182 .await
2183 .expect("reconciler");
2184 timeout_after_1s(mocksrv).await;
2185 }
2186
2187 fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2188 Arc::new(StatefulSet {
2189 metadata: ObjectMeta {
2190 name: Some(format!("{cluster}-default")),
2191 namespace: Some(namespace.to_string()),
2192 labels: Some(BTreeMap::from([(
2193 CLUSTER_LABEL.to_string(),
2194 cluster.to_string(),
2195 )])),
2196 ..Default::default()
2197 },
2198 spec: Some(StatefulSetSpec {
2199 template: PodTemplateSpec {
2200 metadata: Some(ObjectMeta {
2201 annotations: tls_hash.map(|h| {
2202 BTreeMap::from([(
2203 TLS_SECRET_HASH_ANNOTATION.to_string(),
2204 h.to_string(),
2205 )])
2206 }),
2207 ..Default::default()
2208 }),
2209 ..Default::default()
2210 },
2211 ..Default::default()
2212 }),
2213 ..Default::default()
2214 })
2215 }
2216
2217 #[test]
2218 fn test_previous_tls_secret_hash() {
2219 let kanidm = Kanidm::test();
2220
2221 let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2223 assert_eq!(
2224 previous_tls_secret_hash(&kanidm, stss),
2225 Some("abc123".to_string())
2226 );
2227
2228 let stss = vec![statefulset_of("test", "default", None)];
2230 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2231
2232 let stss = vec![
2234 statefulset_of("other", "default", Some("abc123")),
2235 statefulset_of("test", "other-ns", Some("abc123")),
2236 ];
2237 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2238 }
2239
2240 #[tokio::test]
2241 async fn kanidm_create_with_tls_secret() {
2242 let (testctx, fakeserver) = get_test_context();
2243 let kanidm = Kanidm::test();
2244 let secret = Secret {
2245 metadata: ObjectMeta {
2246 name: Some(kanidm.effective_tls_secret_name()),
2247 namespace: Some("default".to_string()),
2248 ..Default::default()
2249 },
2250 data: Some(BTreeMap::from([
2251 ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2252 ("tls.key".to_string(), ByteString(b"key".to_vec())),
2253 ])),
2254 ..Default::default()
2255 };
2256 let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2257 kanidm.clone(),
2258 Box::new(secret),
2259 ));
2260 reconcile_kanidm(Arc::new(kanidm), testctx)
2261 .await
2262 .expect("reconciler");
2263 timeout_after_1s(mocksrv).await;
2264 }
2265
2266 #[tokio::test]
2267 async fn replication_secret_gc_skips_non_replica_secrets() {
2268 use super::reconcile_replication_secrets;
2269 use super::secret::{SECRET_TYPE_LABEL, SecretType};
2270
2271 let kanidm = Kanidm::test();
2272 let status = kanidm.status.clone().unwrap_or_default();
2273
2274 let replica_cert_label =
2275 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2276
2277 let replica_cert_secret = Secret {
2278 metadata: ObjectMeta {
2279 name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2280 namespace: Some("default".into()),
2281 labels: Some(BTreeMap::from([
2282 (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2283 (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2284 ])),
2285 ..Default::default()
2286 },
2287 ..Default::default()
2288 };
2289
2290 let mail_sender_secret = Secret {
2291 metadata: ObjectMeta {
2292 name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2293 namespace: Some("default".into()),
2294 labels: Some(BTreeMap::from([(
2295 CLUSTER_LABEL.to_string(),
2296 kanidm.name_any(),
2297 )])),
2298 ..Default::default()
2299 },
2300 ..Default::default()
2301 };
2302
2303 let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2304 let mock_client = Client::new(mock_service, "default");
2305
2306 let mut secret_writer = Writer::default();
2307 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2308 replica_cert_secret.clone(),
2309 ));
2310 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2311 mail_sender_secret.clone(),
2312 ));
2313
2314 let stores = Stores {
2315 stateful_set_store: Writer::default().as_reader(),
2316 service_store: Writer::default().as_reader(),
2317 ingress_store: Writer::default().as_reader(),
2318 secret_store: secret_writer.as_reader(),
2319 http_route_store: Some(Writer::default().as_reader()),
2320 backend_tls_policy_store: Some(Writer::default().as_reader()),
2321 deployment_store: Writer::default().as_reader(),
2322 config_map_store: Writer::default().as_reader(),
2323 };
2324
2325 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2326 let meter = provider.meter("test");
2327 let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2328 let state = State::new(
2329 metrics,
2330 Writer::default().as_reader(),
2331 Writer::default().as_reader(),
2332 None,
2333 );
2334 let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2335
2336 let mock_srv = tokio::spawn(async move {
2337 let (request, send) = handle.next_request().await.expect("expected a request");
2338 assert_eq!(request.method(), http::Method::DELETE);
2339 let uri = request.uri().to_string();
2340 assert!(
2341 uri.contains("-default-0-cert"),
2342 "expected delete for replica cert secret, got {uri}"
2343 );
2344 let not_found = serde_json::json!({
2345 "kind": "Status",
2346 "apiVersion": "v1",
2347 "metadata": {},
2348 "status": "Failure",
2349 "message": "not found",
2350 "reason": "NotFound",
2351 "code": 404
2352 });
2353 send.send_response(
2354 Response::builder()
2355 .status(404)
2356 .body(Body::from(serde_json::to_vec(¬_found).unwrap()))
2357 .unwrap(),
2358 );
2359
2360 if let Ok(Some((req, _))) =
2361 tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2362 .await
2363 {
2364 panic!("unexpected second request: {} {}", req.method(), req.uri());
2365 }
2366 });
2367
2368 let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2369 .await
2370 .expect("reconcile succeeds despite 404 on delete");
2371 assert!(
2372 changed,
2373 "expected changed=true since a deprecated secret was found"
2374 );
2375
2376 tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2377 .await
2378 .expect("timeout waiting for mock server")
2379 .expect("mock server task succeeded");
2380 }
2381}