1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod mail_sender;
5pub mod secret;
6mod service;
7pub mod statefulset;
8mod status;
9pub mod transport;
10
11use self::domain_appearance::reconcile_domain_appearance;
12use self::mail_sender::{
13 cleanup_mail_sender_in_kanidm, cleanup_mail_sender_resources, reconcile_mail_sender,
14};
15use super::controller::{CONTROLLER_ID, context::Context};
16
17use self::gateway::GatewayExt;
18use self::ingress::IngressExt;
19use self::secret::{SECRET_TYPE_LABEL, SecretExt, SecretType};
20use self::service::ServiceExt;
21use self::statefulset::{
22 StatefulSetApplyStrategy, StatefulSetExt, classify_statefulset_change,
23 preserve_defaulted_statefulset_fields,
24};
25use self::status::StatusExt;
26use self::status::{is_kanidm_available, is_kanidm_initialized};
27use self::transport::resolve_backup_config;
28
29use crate::controller::context::KubeOperations;
30use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
31use crate::kanidm::crd::{
32 Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
33};
34use crate::kanidm::reconcile::statefulset::{REPLICA_LABEL, TLS_SECRET_HASH_ANNOTATION};
35use crate::telemetry;
36
37use kaniop_backup_core::crd::{KanidmBackupRepository, KanidmBackupSchedule};
38use kaniop_k8s_util::client::get_output;
39use kaniop_k8s_util::error::{Error, Result};
40use kaniop_k8s_util::parse::parse_semver;
41use kaniop_k8s_util::resources::{get_image_tag, hash_secret_data};
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::fmt::Debug;
45use std::sync::{Arc, LazyLock};
46
47use futures::future::{TryJoinAll, join_all, try_join_all};
48use futures::stream::{self, StreamExt};
49use futures::try_join;
50use k8s_openapi::NamespaceResourceScope;
51use k8s_openapi::api::apps::v1::StatefulSet;
52use k8s_openapi::api::core::v1::{Pod, Secret, Service};
53use kube::Resource;
54use kube::ResourceExt;
55use kube::api::{Api, AttachParams, Patch, PatchParams, PostParams};
56use kube::runtime::controller::Action;
57use kube::runtime::events::{Event, EventType};
58use kube::runtime::finalizer::{Error as FinalizerError, Event as Finalizer, finalizer};
59use serde::{Deserialize, Serialize};
60use tokio::time::{Duration, sleep};
61use tracing::{Span, debug, field, info, instrument, trace, warn};
62
63const POD_READY_WAIT_TIMEOUT_SECONDS: u64 = 30;
64const POD_READY_POLL_INTERVAL_SECONDS: u64 = 2;
65const STS_ROLLOUT_WAIT_TIMEOUT_SECONDS: u64 = 180;
66const STS_ROLLOUT_POLL_INTERVAL_SECONDS: u64 = 2;
67const STS_DELETE_WAIT_TIMEOUT_SECONDS: u64 = 30;
68const STS_DELETE_POLL_INTERVAL_MILLIS: u64 = 500;
69const CERT_SHOW_RETRY_ATTEMPTS: u32 = 6;
70const CERT_SHOW_INITIAL_DELAY_SECONDS: u64 = 15;
71const CERT_SHOW_RETRY_DELAY_SECONDS: u64 = 15;
72
73pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
74const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
75pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
76
77const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
78
79static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
80 BTreeMap::from([
81 (NAME_LABEL.to_string(), "kanidm".to_string()),
82 (
83 MANAGED_BY_LABEL.to_string(),
84 format!("kaniop-{CONTROLLER_ID}"),
85 ),
86 ])
87});
88
89async fn restart_statefulsets_after_replica_secret_update(
90 ctx: Arc<Context>,
91 sts_api: &Api<StatefulSet>,
92 namespace: &str,
93 statefulset_names: &[String],
94 sequential: bool,
95) -> Result<()> {
96 let restart_futures = statefulset_names
99 .iter()
100 .map(|sts_name| sts_api.restart(sts_name));
101
102 if sequential {
103 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
104 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
105 return Err(Error::kube_error(
106 "restart",
107 "StatefulSet",
108 namespace,
109 "replica certificate secret update statefulsets",
110 first_err,
111 ));
112 }
113 } else {
114 let results: Vec<_> = join_all(restart_futures).await;
115 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
116 return Err(Error::kube_error(
117 "restart",
118 "StatefulSet",
119 namespace,
120 "replica certificate secret update statefulsets",
121 first_err,
122 ));
123 }
124 }
125
126 let rollout_futures = statefulset_names
127 .iter()
128 .map(|sts_name| wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), namespace, sts_name));
129 try_join_all(rollout_futures).await?;
130
131 Ok(())
132}
133
134pub async fn reconcile_admins_secret(
135 kanidm: Arc<Kanidm>,
136 ctx: Arc<Context>,
137 status: &KanidmStatus,
138) -> Result<bool> {
139 if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
140 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
141 kanidm.patch(&ctx, admins_secret).await?;
142 return Ok(true);
143 }
144 Ok(false)
145}
146
147pub async fn reconcile_replication_secrets(
148 kanidm: Arc<Kanidm>,
149 ctx: Arc<Context>,
150 status: &KanidmStatus,
151) -> Result<bool> {
152 let mut changed = false;
153 let expected_secret_names = if kanidm.is_replication_enabled() {
154 status
155 .replica_statuses
156 .iter()
157 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
158 .collect::<Vec<_>>()
159 } else {
160 Vec::new()
161 };
162 let replica_cert_label =
163 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
164 let deprecated_secrets = ctx
165 .stores
166 .secret_store
167 .state()
168 .into_iter()
169 .filter(|secret| {
170 secret.namespace() == kanidm.namespace()
171 && kanidm.admins_secret_name() != secret.name_any()
172 && !expected_secret_names.contains(&secret.name_any())
173 && secret.metadata.labels.as_ref().is_some_and(|l| {
174 l.get(SECRET_TYPE_LABEL) == Some(&replica_cert_label)
175 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
176 })
177 })
178 .collect::<Vec<_>>();
179 if !deprecated_secrets.is_empty() {
180 changed = true;
181 }
182 let secret_delete_future = deprecated_secrets
183 .iter()
184 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
185 .collect::<TryJoinAll<_>>();
186 try_join!(secret_delete_future)?;
187
188 if kanidm.is_replication_enabled() {
189 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
190 let statefulset_names = status
191 .replica_statuses
192 .iter()
193 .map(|rs| rs.statefulset_name.clone())
194 .collect::<BTreeSet<_>>()
195 .into_iter()
196 .collect::<Vec<_>>();
197
198 let has_pending = status
199 .replica_statuses
200 .iter()
201 .any(|rs| rs.state == KanidmReplicaState::Pending);
202
203 let sts_api =
204 Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
205
206 if has_pending {
207 changed = true;
208 stream::iter(
209 status
210 .replica_statuses
211 .iter()
212 .filter(|rs| rs.state == KanidmReplicaState::Pending),
213 )
214 .then(|rs| async {
215 let secret = kanidm
216 .generate_replica_secret(ctx.clone(), &rs.pod_name)
217 .await?;
218 kanidm.patch(&ctx, secret).await
219 })
220 .collect::<Vec<_>>()
221 .await
222 .into_iter()
223 .collect::<Result<Vec<_>>>()?;
224
225 let restart_futures = status
226 .replica_statuses
227 .iter()
228 .map(|rs| sts_api.restart(&rs.statefulset_name));
229
230 if has_single_replica {
231 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
232 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
233 return Err(Error::kube_error(
234 "restart",
235 "StatefulSet",
236 kanidm.get_namespace(),
237 "pending replica statefulsets",
238 first_err,
239 ));
240 }
241 } else {
242 let results: Vec<_> = join_all(restart_futures).await;
243 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
244 return Err(Error::kube_error(
245 "restart",
246 "StatefulSet",
247 kanidm.get_namespace(),
248 "pending replica statefulsets",
249 first_err,
250 ));
251 }
252 }
253 }
254
255 let has_certificate_host_invalid = status
256 .replica_statuses
257 .iter()
258 .any(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid);
259
260 let has_certificate_expiring = status
261 .replica_statuses
262 .iter()
263 .any(|rs| rs.state == KanidmReplicaState::CertificateExpiring);
264
265 let has_non_ready_replicas = status
266 .replica_statuses
267 .iter()
268 .any(|rs| rs.state != KanidmReplicaState::Ready);
269
270 let host_invalid_replicas: Vec<_> = status
271 .replica_statuses
272 .iter()
273 .filter(|rs| rs.state == KanidmReplicaState::CertificateHostInvalid)
274 .collect();
275
276 let cert_expiring_replicas: Vec<_> = status
277 .replica_statuses
278 .iter()
279 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
280 .collect();
281
282 if !host_invalid_replicas.is_empty() {
283 changed = true;
284 let restart_futures = host_invalid_replicas
285 .iter()
286 .map(|rs| sts_api.restart(&rs.statefulset_name));
287
288 if has_single_replica {
289 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
290 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
291 return Err(Error::kube_error(
292 "restart",
293 "StatefulSet",
294 kanidm.get_namespace(),
295 "certificate host invalid statefulsets",
296 first_err,
297 ));
298 }
299 } else {
300 let results: Vec<_> = join_all(restart_futures).await;
301 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
302 return Err(Error::kube_error(
303 "restart",
304 "StatefulSet",
305 kanidm.get_namespace(),
306 "certificate host invalid statefulsets",
307 first_err,
308 ));
309 }
310 }
311
312 let restarted_sts_names: Vec<_> = host_invalid_replicas
313 .iter()
314 .map(|rs| rs.statefulset_name.clone())
315 .collect::<BTreeSet<_>>()
316 .into_iter()
317 .collect();
318
319 let namespace = kanidm.get_namespace();
320 let rollout_futures = restarted_sts_names.iter().map(|sts_name| {
321 wait_for_sts_rollout(ctx.kaniop_ctx.client.clone(), &namespace, sts_name)
322 });
323 try_join_all(rollout_futures).await?;
324
325 let renew_futures = host_invalid_replicas
326 .iter()
327 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
328 let results: Vec<_> = join_all(renew_futures).await;
329 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
330 return Err(first_err);
331 }
332
333 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
334
335 let cert_futures = host_invalid_replicas.iter().map(|rs| async {
336 let cert = show_replica_cert_with_retries(
337 &kanidm,
338 ctx.clone(),
339 &rs.pod_name,
340 CERT_SHOW_RETRY_ATTEMPTS,
341 )
342 .await?;
343 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
344 kanidm.patch(&ctx, secret.clone()).await
345 });
346 try_join_all(cert_futures).await?;
347
348 restart_statefulsets_after_replica_secret_update(
349 ctx.clone(),
350 &sts_api,
351 &kanidm.get_namespace(),
352 &statefulset_names,
353 has_single_replica,
354 )
355 .await?;
356
357 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
358 }
359
360 if !cert_expiring_replicas.is_empty() {
361 changed = true;
362 let renew_futures = cert_expiring_replicas
363 .iter()
364 .map(|rs| kanidm.renew_replica_cert(ctx.clone(), &rs.pod_name));
365 let results: Vec<_> = join_all(renew_futures).await;
366 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
367 return Err(first_err);
368 }
369
370 sleep(Duration::from_secs(CERT_SHOW_INITIAL_DELAY_SECONDS)).await;
371
372 let cert_futures = cert_expiring_replicas.iter().map(|rs| async {
373 let cert = show_replica_cert_with_retries(
374 &kanidm,
375 ctx.clone(),
376 &rs.pod_name,
377 CERT_SHOW_RETRY_ATTEMPTS,
378 )
379 .await?;
380 let secret = kanidm.build_replica_secret(cert, &rs.pod_name);
381 kanidm.patch(&ctx, secret.clone()).await
382 });
383 try_join_all(cert_futures).await?;
384
385 restart_statefulsets_after_replica_secret_update(
386 ctx.clone(),
387 &sts_api,
388 &kanidm.get_namespace(),
389 &statefulset_names,
390 has_single_replica,
391 )
392 .await?;
393
394 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
395 }
396
397 if has_non_ready_replicas
398 && !has_pending
399 && !has_certificate_host_invalid
400 && !has_certificate_expiring
401 {
402 changed = true;
403 let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
404
405 let restart_futures = status
406 .replica_statuses
407 .iter()
408 .map(|rs| sts_api.restart(&rs.statefulset_name));
409
410 if has_single_replica {
411 let results: Vec<_> = stream::iter(restart_futures).then(|f| f).collect().await;
412 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
413 return Err(Error::kube_error(
414 "restart",
415 "StatefulSet",
416 kanidm.get_namespace(),
417 "replica statefulsets",
418 first_err,
419 ));
420 }
421 } else {
422 let results: Vec<_> = join_all(restart_futures).await;
423 if let Some(first_err) = results.into_iter().find_map(|r| r.err()) {
424 return Err(Error::kube_error(
425 "restart",
426 "StatefulSet",
427 kanidm.get_namespace(),
428 "replica statefulsets",
429 first_err,
430 ));
431 }
432 }
433 }
434 }
435
436 Ok(changed)
437}
438
439#[instrument(skip(ctx, kanidm))]
440pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
441 let trace_id = telemetry::get_trace_id();
442 Span::current().record("trace_id", field::display(&trace_id));
443 let _timer = ctx
444 .kaniop_ctx
445 .metrics
446 .reconcile_count_and_measure(&trace_id);
447 info!("reconciling Kanidm");
448
449 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
450 debug!(%e, "failed to reconcile status");
451 ctx.kaniop_ctx.metrics.status_update_errors_inc();
452 e
453 })?;
454 let kanidm_api: Api<Kanidm> =
455 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
456 let outcome = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
457 let outcome_clone = outcome.clone();
458 let action = finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, move |event| {
459 let outcome = outcome_clone.clone();
460 let ctx = ctx.clone();
461 let status = status.clone();
462 async move {
463 match event {
464 Finalizer::Apply(kanidm) => {
465 let (action, changed) = reconcile(kanidm, ctx, status).await?;
466 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
467 Ok(action)
468 }
469 Finalizer::Cleanup(kanidm) => {
470 let (action, changed) = cleanup(kanidm, ctx).await?;
471 outcome.store(changed, std::sync::atomic::Ordering::Relaxed);
472 Ok(action)
473 }
474 }
475 }
476 })
477 .await
478 .or_else(|e| match e {
479 FinalizerError::RemoveFinalizer(kube::Error::Api(ae)) if ae.code == 404 => {
480 debug!("resource already removed during finalizer cleanup");
481 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
482 }
483 _ => Err(Error::FinalizerError(
484 "failed on kanidm account finalizer".to_string(),
485 Box::new(e),
486 )),
487 })?;
488 let changed = outcome.load(std::sync::atomic::Ordering::Relaxed);
489 Ok((action, changed))
490}
491
492async fn get_tls_secret_hash(kanidm: &Kanidm, ctx: &Arc<Context>) -> Result<Option<String>> {
496 let secret_name = kanidm.effective_tls_secret_name();
497 let secret_api =
498 Api::<Secret>::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
499 let secret = secret_api
500 .get_opt(&secret_name)
501 .await
502 .map_err(|e| Error::kube_error("get", "Secret", kanidm.get_namespace(), &secret_name, e))?;
503 Ok(secret.as_ref().map(hash_secret_data))
504}
505
506fn previous_tls_secret_hash(
511 kanidm: &Kanidm,
512 stateful_sets: impl IntoIterator<Item = Arc<StatefulSet>>,
513) -> Option<String> {
514 stateful_sets
515 .into_iter()
516 .filter(|sts| {
517 sts.namespace() == kanidm.namespace()
518 && sts
519 .metadata
520 .labels
521 .as_ref()
522 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
523 })
524 .find_map(|sts| {
525 sts.spec
526 .as_ref()?
527 .template
528 .metadata
529 .as_ref()?
530 .annotations
531 .as_ref()?
532 .get(TLS_SECRET_HASH_ANNOTATION)
533 .cloned()
534 })
535}
536
537fn is_headless_service(service: &Service) -> bool {
538 service
539 .spec
540 .as_ref()
541 .and_then(|spec| spec.cluster_ip.as_deref())
542 == Some("None")
543}
544
545fn preserve_defaulted_service_fields(desired: &mut Service, current: &Service) {
546 if is_headless_service(desired) != is_headless_service(current) {
547 return;
548 }
549 let (Some(desired_spec), Some(current_spec)) = (desired.spec.as_mut(), current.spec.as_ref())
550 else {
551 return;
552 };
553 if desired_spec.cluster_ip.is_none() {
554 desired_spec.cluster_ip = current_spec.cluster_ip.clone();
555 }
556 if desired_spec.cluster_ips.is_none() {
557 desired_spec.cluster_ips = current_spec.cluster_ips.clone();
558 }
559}
560
561async fn wait_for_statefulset_deletion(
562 client: kube::Client,
563 namespace: &str,
564 name: &str,
565) -> Result<()> {
566 let api = Api::<StatefulSet>::namespaced(client, namespace);
567 let start = std::time::Instant::now();
568 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
569 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
570
571 loop {
572 match api.get_opt(name).await {
573 Ok(None) => return Ok(()),
574 Ok(Some(_)) => {}
575 Err(e) => {
576 return Err(Error::kube_error(
577 "wait for deletion of",
578 "StatefulSet",
579 namespace,
580 name,
581 e,
582 ));
583 }
584 }
585
586 if start.elapsed() >= timeout {
587 return Err(Error::ReceiveOutput(format!(
588 "StatefulSet {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
589 )));
590 }
591 sleep(poll_interval).await;
592 }
593}
594
595async fn reconcile_service(
596 kanidm: &Kanidm,
597 ctx: &Context,
598 mut desired: Service,
599) -> Result<Service> {
600 let namespace = kanidm.get_namespace();
601 let name = desired.name_any();
602 let desired_headless = is_headless_service(&desired);
603
604 if let Some(cached) = ctx
605 .stores
606 .service_store
607 .find(|current| current.namespace() == kanidm.namespace() && current.name_any() == name)
608 {
609 if is_headless_service(cached.as_ref()) == desired_headless {
610 preserve_defaulted_service_fields(&mut desired, cached.as_ref());
611 } else {
612 let service_api = Api::<Service>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
613 let current = service_api
614 .get_opt(&name)
615 .await
616 .map_err(|e| Error::kube_error("get", "Service", &namespace, &name, e))?;
617
618 if let Some(current) = current {
619 let current_headless = is_headless_service(¤t);
620 if current_headless != desired_headless {
621 info!(
622 resource.name = &name,
623 resource.namespace = &namespace,
624 current_headless,
625 desired_headless,
626 "recreating Service to change headless mode"
627 );
628 kanidm.delete(ctx, ¤t).await?;
629 ctx.kaniop_ctx
630 .metrics
631 .reconcile_deploy_delete_create_inc("Service", "headless_transition");
632 wait_for_service_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
633 .await?;
634 } else {
635 preserve_defaulted_service_fields(&mut desired, ¤t);
636 }
637 }
638 }
639 }
640
641 kanidm.patch(ctx, desired).await
642}
643
644async fn wait_for_service_deletion(
645 client: kube::Client,
646 namespace: &str,
647 name: &str,
648) -> Result<()> {
649 let api = Api::<Service>::namespaced(client, namespace);
650 let start = std::time::Instant::now();
651 let timeout = Duration::from_secs(STS_DELETE_WAIT_TIMEOUT_SECONDS);
652 let poll_interval = Duration::from_millis(STS_DELETE_POLL_INTERVAL_MILLIS);
653
654 loop {
655 match api.get_opt(name).await {
656 Ok(None) => return Ok(()),
657 Ok(Some(_)) => {}
658 Err(e) => {
659 return Err(Error::kube_error(
660 "wait for deletion of",
661 "Service",
662 namespace,
663 name,
664 e,
665 ));
666 }
667 }
668
669 if start.elapsed() >= timeout {
670 return Err(Error::ReceiveOutput(format!(
671 "Service {namespace}/{name} was not deleted after {STS_DELETE_WAIT_TIMEOUT_SECONDS}s"
672 )));
673 }
674 sleep(poll_interval).await;
675 }
676}
677
678async fn validate_statefulset_replacement(
679 api: &Api<StatefulSet>,
680 desired: &StatefulSet,
681 namespace: &str,
682 name: &str,
683) -> Result<()> {
684 let mut candidate = desired.clone();
685 candidate.metadata.name = None;
686 candidate.metadata.generate_name = Some("kaniop-preflight-".to_string());
687 candidate.metadata.namespace = None;
688 candidate.metadata.resource_version = None;
689 candidate.metadata.uid = None;
690 candidate.metadata.creation_timestamp = None;
691 candidate.metadata.generation = None;
692 candidate.metadata.managed_fields = None;
693 candidate.metadata.deletion_timestamp = None;
694 candidate.metadata.deletion_grace_period_seconds = None;
695 candidate.status = None;
696
697 api.create(
698 &PostParams {
699 dry_run: true,
700 ..Default::default()
701 },
702 &candidate,
703 )
704 .await
705 .map(|_| ())
706 .map_err(|e| {
707 Error::kube_error(
708 "validate replacement for",
709 "StatefulSet",
710 namespace,
711 name,
712 e,
713 )
714 })
715}
716
717async fn reconcile_statefulset(
718 kanidm: Arc<Kanidm>,
719 ctx: Arc<Context>,
720 desired: StatefulSet,
721) -> Result<StatefulSet> {
722 let namespace = kanidm.namespace().unwrap();
723 let name = desired.name_any();
724 let cached_current = ctx.stores.stateful_set_store.find(|current| {
725 current.namespace().as_deref() == Some(namespace.as_str()) && current.name_any() == name
726 });
727
728 let cache_miss_422_error = if let Some(cached_current) = cached_current {
733 let mut cached_desired = desired.clone();
734 preserve_defaulted_statefulset_fields(&mut cached_desired, cached_current.as_ref());
735 if classify_statefulset_change(cached_current.as_ref(), &cached_desired)
736 == StatefulSetApplyStrategy::Apply
737 {
738 return kanidm.apply(&ctx, cached_desired).await;
739 }
740 None
741 } else {
742 match kanidm.apply(&ctx, desired.clone()).await {
743 Ok(applied) => return Ok(applied),
744 Err(error) => {
745 if is_kube_422(&error) {
749 Some(error)
750 } else {
751 return Err(error);
752 }
753 }
754 }
755 };
756
757 let api = Api::<StatefulSet>::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
761 let Some(live_current) = api
762 .get_opt(&name)
763 .await
764 .map_err(|e| Error::kube_error("get", "StatefulSet", &namespace, &name, e))?
765 else {
766 return match cache_miss_422_error {
767 Some(error) => Err(error),
768 None => kanidm.apply(&ctx, desired).await,
769 };
770 };
771
772 let mut live_desired = desired.clone();
775 preserve_defaulted_statefulset_fields(&mut live_desired, &live_current);
776 match classify_statefulset_change(&live_current, &live_desired) {
777 StatefulSetApplyStrategy::Apply => match cache_miss_422_error {
778 Some(error) => Err(error),
779 None => kanidm.apply(&ctx, live_desired).await,
780 },
781 StatefulSetApplyStrategy::Recreate { .. } => {
782 validate_statefulset_replacement(&api, &live_desired, &namespace, &name).await?;
786
787 let Some(latest_current) = api.get_opt(&name).await.map_err(|e| {
790 Error::kube_error(
791 "re-read before deletion",
792 "StatefulSet",
793 &namespace,
794 &name,
795 e,
796 )
797 })?
798 else {
799 return kanidm.apply(&ctx, live_desired).await;
800 };
801 let mut latest_desired = live_desired.clone();
802 preserve_defaulted_statefulset_fields(&mut latest_desired, &latest_current);
803
804 match classify_statefulset_change(&latest_current, &latest_desired) {
805 StatefulSetApplyStrategy::Apply => kanidm.apply(&ctx, latest_desired).await,
806 StatefulSetApplyStrategy::Recreate { immutable_fields } => {
807 validate_statefulset_replacement(&api, &latest_desired, &namespace, &name)
812 .await?;
813
814 info!(resource.name = %name, resource.namespace = %namespace, ?immutable_fields, "recreating StatefulSet because immutable fields changed");
815 kanidm.delete(&ctx, &latest_current).await?;
816 ctx.kaniop_ctx
817 .metrics
818 .reconcile_deploy_delete_create_inc("StatefulSet", "immutable_spec");
819 wait_for_statefulset_deletion(ctx.kaniop_ctx.client.clone(), &namespace, &name)
820 .await?;
821 kanidm.apply(&ctx, latest_desired).await
822 }
823 }
824 }
825 }
826}
827
828fn is_kube_422(error: &Error) -> bool {
830 matches!(
831 error,
832 Error::KubeError(_, cause)
833 if matches!(cause.as_ref(), kube::Error::Api(status) if status.code == 422)
834 )
835}
836
837#[cfg(test)]
838mod statefulset_recreate_recovery_tests {
839 use super::{Error, is_kube_422};
840
841 fn kube_api_error(code: u16) -> Error {
842 Error::KubeError(
843 "test error".to_string(),
844 Box::new(kube::Error::Api(Box::new(kube::error::Status {
845 code,
846 message: "test".to_string(),
847 reason: "test".to_string(),
848 ..Default::default()
849 }))),
850 )
851 }
852
853 #[test]
854 fn recreate_recovery_accepts_only_422() {
855 assert!(is_kube_422(&kube_api_error(422)));
856 assert!(!is_kube_422(&kube_api_error(403)));
857 assert!(!is_kube_422(&kube_api_error(429)));
858 assert!(!is_kube_422(&kube_api_error(500)));
859 }
860
861 #[test]
862 fn recreate_recovery_rejects_429_too_many_requests() {
863 assert!(!is_kube_422(&kube_api_error(429)));
864 }
865
866 #[test]
867 fn recreate_recovery_rejects_5xx_server_errors() {
868 assert!(!is_kube_422(&kube_api_error(500)));
869 assert!(!is_kube_422(&kube_api_error(502)));
870 assert!(!is_kube_422(&kube_api_error(503)));
871 assert!(!is_kube_422(&kube_api_error(504)));
872 }
873
874 #[test]
875 fn recreate_recovery_rejects_400_and_403() {
876 assert!(!is_kube_422(&kube_api_error(400)));
877 assert!(!is_kube_422(&kube_api_error(403)));
878 }
879}
880
881async fn reconcile(
882 kanidm: Arc<Kanidm>,
883 ctx: Arc<Context>,
884 status: KanidmStatus,
885) -> Result<(Action, bool)> {
886 if kanidm
887 .annotations()
888 .contains_key(crate::kanidm::restore::RESTORE_ANNOTATION)
889 {
890 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
891 return Ok((Action::requeue(Duration::from_secs(5)), false));
892 }
893
894 let mut changed = false;
895 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
896 let replication_secret_futures =
897 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
898
899 let sts_to_delete = {
900 let expected_sts_names = kanidm
901 .spec
902 .replica_groups
903 .iter()
904 .map(|rg| kanidm.statefulset_name(&rg.name))
905 .collect::<Vec<_>>();
906 ctx.stores
907 .stateful_set_store
908 .state()
909 .into_iter()
910 .filter(|sts| {
911 sts.namespace() == kanidm.namespace()
912 && sts
913 .metadata
914 .labels
915 .as_ref()
916 .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
917 && !expected_sts_names.contains(&sts.name_any())
918 })
919 .collect::<Vec<_>>()
920 };
921 if !sts_to_delete.is_empty() {
922 changed = true;
923 }
924 let sts_delete_futures = sts_to_delete
925 .iter()
926 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
927 .collect::<TryJoinAll<_>>();
928
929 let sts_futures = match kanidm.is_updatable(&status) {
930 true => {
931 let tls_secret_hash = get_tls_secret_hash(&kanidm, &ctx).await?.or_else(|| {
932 previous_tls_secret_hash(&kanidm, ctx.stores.stateful_set_store.state())
933 });
934
935 let namespace = kanidm.get_namespace();
936 let schedule_api: Api<KanidmBackupSchedule> =
937 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
938 let schedules = schedule_api
939 .list(&Default::default())
940 .await
941 .map_err(|e| {
942 Error::KubeError(
943 "failed to list KanidmBackupSchedules for backup configuration resolution"
944 .to_string(),
945 Box::new(e),
946 )
947 })?
948 .items;
949
950 let repo_api: Api<KanidmBackupRepository> =
951 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
952 let repositories = repo_api
953 .list(&Default::default())
954 .await
955 .map_err(|e| {
956 Error::KubeError(
957 "failed to list KanidmBackupRepositories for backup configuration resolution"
958 .to_string(),
959 Box::new(e),
960 )
961 })?
962 .items;
963 let backup_config = resolve_backup_config(&kanidm, &schedules, &repositories);
964
965 kanidm
966 .spec
967 .replica_groups
968 .iter()
969 .map(|rg| {
970 let sts = kanidm.create_statefulset(
971 rg,
972 tls_secret_hash.as_deref(),
973 backup_config.as_ref(),
974 )?;
975 Ok(reconcile_statefulset(kanidm.clone(), ctx.clone(), sts))
976 })
977 .collect::<Result<TryJoinAll<_>, _>>()?
978 }
979 false => {
980 let note = match status.version.as_ref() {
981 Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
982 format!(
983 "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
984 v.image_tag,
985 crate::version::KANIDM_CLIENT_VERSION
986 )
987 }
988 _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
989 };
990 let _ignore_error = ctx
991 .kaniop_ctx
992 .recorder
993 .publish(
994 &Event {
995 type_: EventType::Warning,
996 reason: "UpgradeBlocked".to_string(),
997 note: Some(note),
998 action: "ReconcileStatefulSet".to_string(),
999 secondary: None,
1000 },
1001 &kanidm.object_ref(&()),
1002 )
1003 .await
1004 .map_err(|e| {
1005 warn!(%e, "failed to publish KanidmError event");
1006 Error::KubeError("failed to publish event".to_string(), Box::new(e))
1007 });
1008 try_join_all([])
1009 }
1010 };
1011 let service_future = reconcile_service(&kanidm, &ctx, kanidm.create_service());
1012
1013 let deprecated_rg_svcs = {
1014 let expected_rg_svcs_names = kanidm
1015 .spec
1016 .replica_groups
1017 .iter()
1018 .filter(|rg| rg.services.is_some())
1019 .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
1020 .collect::<Vec<_>>();
1021 ctx.stores
1022 .service_store
1023 .state()
1024 .into_iter()
1025 .filter(|svc| {
1026 let labels = svc.metadata.labels.as_ref();
1027 svc.namespace() == kanidm.namespace()
1028 && !expected_rg_svcs_names.contains(&svc.name_any())
1029 && labels.is_some_and(|l| {
1030 l.contains_key(REPLICA_LABEL)
1031 && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
1032 })
1033 })
1034 .collect::<Vec<_>>()
1035 };
1036 if !deprecated_rg_svcs.is_empty() {
1037 changed = true;
1038 }
1039 let rg_svcs_delete_futures = deprecated_rg_svcs
1040 .iter()
1041 .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
1042 .collect::<TryJoinAll<_>>();
1043
1044 let rg_services_futures = kanidm
1045 .spec
1046 .replica_groups
1047 .iter()
1048 .filter(|rg| rg.services.is_some())
1049 .flat_map(|rg| {
1050 (0..rg.replicas).map(|i| {
1051 let mut svc = kanidm.create_replica_group_service(rg, i);
1052 if let Some(current) = ctx.stores.service_store.find(|current| {
1053 current.namespace() == kanidm.namespace()
1054 && current.name_any() == svc.name_any()
1055 }) {
1056 preserve_defaulted_service_fields(&mut svc, current.as_ref());
1057 }
1058 kanidm.patch(&ctx, svc)
1059 })
1060 })
1061 .collect::<TryJoinAll<_>>();
1062
1063 let deprecated_ingresses = {
1064 let names = [
1065 kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
1066 kanidm.generate_region_ingress_name(),
1067 ]
1068 .into_iter()
1069 .flatten()
1070 .collect::<Vec<_>>();
1071 ctx.stores
1072 .ingress_store
1073 .state()
1074 .into_iter()
1075 .filter(|ing| {
1076 ing.namespace() == kanidm.namespace()
1077 && !names.contains(&ing.name_any())
1078 && ing.metadata.labels == Some(kanidm.generate_labels())
1079 })
1080 .collect::<Vec<_>>()
1081 };
1082 if !deprecated_ingresses.is_empty() {
1083 changed = true;
1084 }
1085 let ingress_delete_futures = deprecated_ingresses
1086 .iter()
1087 .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
1088 .collect::<TryJoinAll<_>>();
1089
1090 let ingress_futures = kanidm
1091 .create_ingress()
1092 .into_iter()
1093 .chain(kanidm.create_region_ingress())
1094 .map(|ingress| kanidm.patch(&ctx, ingress))
1095 .collect::<TryJoinAll<_>>();
1096
1097 let deprecated_http_routes = match &ctx.stores.http_route_store {
1098 Some(store) => {
1099 let expected_names = kanidm
1100 .spec
1101 .gateway
1102 .as_ref()
1103 .map(|_| vec![kanidm.name_any()])
1104 .unwrap_or_default();
1105 let routes: Vec<_> = store
1106 .state()
1107 .into_iter()
1108 .filter(|route| {
1109 route.namespace() == kanidm.namespace()
1110 && !expected_names.contains(&route.name_any())
1111 && route.metadata.labels == Some(kanidm.generate_labels())
1112 })
1113 .collect();
1114 if !routes.is_empty() {
1115 changed = true;
1116 }
1117 routes
1118 }
1119 None => Vec::new(),
1120 };
1121 let http_route_delete_futures = deprecated_http_routes
1122 .iter()
1123 .map(|route| kanidm.delete(&ctx, route.as_ref()))
1124 .collect::<TryJoinAll<_>>();
1125
1126 let http_route_futures = if ctx.stores.http_route_store.is_some() {
1127 kanidm
1128 .create_http_route()
1129 .into_iter()
1130 .map(|route| kanidm.patch(&ctx, route))
1131 .collect::<TryJoinAll<_>>()
1132 } else {
1133 try_join_all(Vec::new())
1134 };
1135
1136 let deprecated_backend_tls_policies = match &ctx.stores.backend_tls_policy_store {
1137 Some(store) => {
1138 let expected_names = kanidm
1139 .spec
1140 .gateway
1141 .as_ref()
1142 .and_then(|g| g.backend_tls_policy.as_ref())
1143 .map(|_| vec![kanidm.name_any()])
1144 .unwrap_or_default();
1145 let policies: Vec<_> = store
1146 .state()
1147 .into_iter()
1148 .filter(|policy| {
1149 policy.namespace() == kanidm.namespace()
1150 && !expected_names.contains(&policy.name_any())
1151 && policy.metadata.labels == Some(kanidm.generate_labels())
1152 })
1153 .collect();
1154 if !policies.is_empty() {
1155 changed = true;
1156 }
1157 policies
1158 }
1159 None => Vec::new(),
1160 };
1161 let backend_tls_policy_delete_futures = deprecated_backend_tls_policies
1162 .iter()
1163 .map(|policy| kanidm.delete(&ctx, policy.as_ref()))
1164 .collect::<TryJoinAll<_>>();
1165
1166 let backend_tls_policy_futures = if ctx.stores.backend_tls_policy_store.is_some() {
1167 kanidm
1168 .create_backend_tls_policy()
1169 .into_iter()
1170 .map(|policy| kanidm.patch(&ctx, policy))
1171 .collect::<TryJoinAll<_>>()
1172 } else {
1173 try_join_all(Vec::new())
1174 };
1175
1176 let (_, admin_secret_changed, replication_secrets_changed, _, _, _, _, _, _, _, _, _, _) = try_join!(
1177 sts_delete_futures,
1178 admin_secret_future,
1179 replication_secret_futures,
1180 sts_futures,
1181 service_future,
1182 rg_svcs_delete_futures,
1183 rg_services_futures,
1184 ingress_delete_futures,
1185 ingress_futures,
1186 http_route_delete_futures,
1187 http_route_futures,
1188 backend_tls_policy_delete_futures,
1189 backend_tls_policy_futures
1190 )?;
1191
1192 if admin_secret_changed || replication_secrets_changed {
1193 changed = true;
1194 }
1195
1196 if is_kanidm_available(status.clone()) {
1197 let namespace = kanidm.namespace().unwrap();
1198 let name = kanidm.name_any();
1199 match crate::controller::kanidm::KanidmClients::create_client(
1200 &namespace,
1201 &name,
1202 crate::controller::kanidm::KanidmUser::Admin,
1203 ctx.kaniop_ctx.client.clone(),
1204 )
1205 .await
1206 {
1207 Ok(system_client) => {
1208 if let Err(e) =
1209 reconcile_domain_appearance(&kanidm, system_client, &status, ctx.clone()).await
1210 {
1211 warn!(%e, "failed to reconcile domain appearance");
1212 }
1213 }
1214 Err(e) => {
1215 warn!(%e, "failed to create admin client for domain appearance reconciliation");
1216 }
1217 }
1218
1219 match crate::controller::kanidm::KanidmClients::create_client(
1220 &namespace,
1221 &name,
1222 crate::controller::kanidm::KanidmUser::IdmAdmin,
1223 ctx.kaniop_ctx.client.clone(),
1224 )
1225 .await
1226 {
1227 Ok(kanidm_client) => {
1228 let (mail_sender_status, mail_sender_mutated) =
1229 reconcile_mail_sender(&kanidm, kanidm_client.clone(), ctx.clone())
1230 .await
1231 .unwrap_or_else(|e| {
1232 warn!(%e, "failed to reconcile mail sender");
1233 (None, false)
1234 });
1235 if mail_sender_mutated {
1236 changed = true;
1237 }
1238 let current_mail_sender_status =
1239 kanidm.status.as_ref().and_then(|s| s.mail_sender.clone());
1240 if mail_sender_status != current_mail_sender_status {
1241 let kanidm_api: Api<Kanidm> =
1242 Api::namespaced(ctx.kaniop_ctx.client.clone(), &namespace);
1243 let status_patch = serde_json::json!({
1244 "status": {
1245 "mailSender": mail_sender_status
1246 }
1247 });
1248 if let Err(e) = kanidm_api
1249 .patch_status(&name, &PatchParams::default(), &Patch::Merge(&status_patch))
1250 .await
1251 {
1252 warn!(%e, "failed to patch Kanidm/status for mail sender");
1253 }
1254 }
1255 }
1256 Err(e) => {
1257 warn!(%e, "failed to create idm_admin client for mail sender reconciliation");
1258 }
1259 }
1260 }
1261
1262 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1263}
1264
1265async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<(Action, bool)> {
1266 debug!("cleanup");
1267
1268 let mut changed = false;
1269 let mail_resources_cleaned = cleanup_mail_sender_resources(&kanidm, &ctx).await?;
1270 if mail_resources_cleaned {
1271 changed = true;
1272 }
1273
1274 let namespace = kanidm.namespace().unwrap();
1275 let name = kanidm.name_any();
1276
1277 if let Ok(kanidm_client) = crate::controller::kanidm::KanidmClients::create_client(
1278 &namespace,
1279 &name,
1280 crate::controller::kanidm::KanidmUser::IdmAdmin,
1281 ctx.kaniop_ctx.client.clone(),
1282 )
1283 .await
1284 {
1285 let kanidm_cleanup_done =
1286 cleanup_mail_sender_in_kanidm(&kanidm_client, &name, &ctx.kaniop_ctx.metrics).await?;
1287 if kanidm_cleanup_done {
1288 changed = true;
1289 }
1290 }
1291
1292 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
1293 Ok((Action::requeue(DEFAULT_RECONCILE_INTERVAL), changed))
1294}
1295
1296async fn wait_for_sts_rollout(client: kube::Client, namespace: &str, sts_name: &str) -> Result<()> {
1297 let sts_api = Api::<StatefulSet>::namespaced(client, namespace);
1298 let start = std::time::Instant::now();
1299 let timeout = Duration::from_secs(STS_ROLLOUT_WAIT_TIMEOUT_SECONDS);
1300 let poll_interval = Duration::from_secs(STS_ROLLOUT_POLL_INTERVAL_SECONDS);
1301
1302 loop {
1303 let sts = sts_api.get(sts_name).await.map_err(|e| {
1304 Error::KubeError(
1305 format!("failed to get StatefulSet {namespace}/{sts_name}"),
1306 Box::new(e),
1307 )
1308 })?;
1309
1310 if sts.metadata.deletion_timestamp.is_some() {
1311 return Err(Error::ReceiveOutput(format!(
1312 "StatefulSet {namespace}/{sts_name} is being deleted"
1313 )));
1314 }
1315
1316 let spec = sts.spec.as_ref().ok_or_else(|| {
1317 Error::MissingData(format!("StatefulSet {namespace}/{sts_name} has no spec"))
1318 })?;
1319
1320 let status = match sts.status {
1321 Some(ref s) => s,
1322 None => {
1323 if start.elapsed() >= timeout {
1324 return Err(Error::ReceiveOutput(format!(
1325 "StatefulSet {namespace}/{sts_name} has no status after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s"
1326 )));
1327 }
1328 sleep(poll_interval).await;
1329 continue;
1330 }
1331 };
1332
1333 let replicas = spec.replicas.unwrap_or(0);
1334
1335 if replicas == 0 {
1336 return Ok(());
1337 }
1338
1339 let updated_replicas = status.updated_replicas.unwrap_or(0);
1340 let current_revision = status.current_revision.as_deref().unwrap_or("");
1341 let update_revision = status.update_revision.as_deref().unwrap_or("");
1342 let ready_replicas = status.ready_replicas.unwrap_or(0);
1343
1344 if updated_replicas == replicas
1345 && current_revision == update_revision
1346 && ready_replicas == replicas
1347 {
1348 return Ok(());
1349 }
1350
1351 if start.elapsed() >= timeout {
1352 return Err(Error::ReceiveOutput(format!(
1353 "StatefulSet {namespace}/{sts_name} rollout not complete after {STS_ROLLOUT_WAIT_TIMEOUT_SECONDS}s (updated={updated_replicas}/{replicas}, ready={ready_replicas}/{replicas})"
1354 )));
1355 }
1356
1357 sleep(poll_interval).await;
1358 }
1359}
1360
1361async fn show_replica_cert_with_retries(
1362 kanidm: &Kanidm,
1363 ctx: Arc<Context>,
1364 pod_name: &str,
1365 max_attempts: u32,
1366) -> std::result::Result<String, Error> {
1367 let mut last_error = None;
1368 for attempt in 0..max_attempts {
1369 if attempt > 0 {
1370 sleep(Duration::from_secs(CERT_SHOW_RETRY_DELAY_SECONDS)).await;
1371 }
1372 match kanidm.show_replica_cert(ctx.clone(), pod_name).await {
1373 Ok(cert) => return Ok(cert),
1374 Err(e) => {
1375 info!(
1376 "show-replication-certificate attempt {}/{} failed for {pod_name}: {e}",
1377 attempt + 1,
1378 max_attempts
1379 );
1380 last_error = Some(e);
1381 }
1382 }
1383 }
1384 Err(last_error.unwrap_or_else(|| {
1385 Error::ReceiveOutput(format!(
1386 "failed to get certificate for {pod_name} after {max_attempts} attempts"
1387 ))
1388 }))
1389}
1390
1391impl Kanidm {
1392 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
1394 where
1395 K: Resource<Scope = NamespaceResourceScope>
1396 + Serialize
1397 + Clone
1398 + std::fmt::Debug
1399 + for<'de> Deserialize<'de>,
1400 <K as kube::Resource>::DynamicType: Default,
1401 <K as Resource>::Scope: std::marker::Sized,
1402 {
1403 self.kube_delete(
1404 ctx.kaniop_ctx.client.clone(),
1405 &ctx.kaniop_ctx.metrics,
1406 resource,
1407 )
1408 .await
1409 }
1410
1411 pub async fn apply<K>(&self, ctx: &Context, resource: K) -> Result<K>
1414 where
1415 K: Resource<Scope = NamespaceResourceScope>
1416 + Serialize
1417 + Clone
1418 + std::fmt::Debug
1419 + for<'de> Deserialize<'de>,
1420 <K as kube::Resource>::DynamicType: Default,
1421 <K as Resource>::Scope: std::marker::Sized,
1422 {
1423 self.kube_apply(
1424 ctx.kaniop_ctx.client.clone(),
1425 resource,
1426 KANIDM_OPERATOR_NAME,
1427 )
1428 .await
1429 }
1430
1431 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
1432 where
1433 K: Resource<Scope = NamespaceResourceScope>
1434 + Serialize
1435 + Clone
1436 + std::fmt::Debug
1437 + for<'de> Deserialize<'de>,
1438 <K as kube::Resource>::DynamicType: Default,
1439 <K as Resource>::Scope: std::marker::Sized,
1440 {
1441 self.kube_patch(
1442 ctx.kaniop_ctx.client.clone(),
1443 &ctx.kaniop_ctx.metrics,
1444 resource,
1445 KANIDM_OPERATOR_NAME,
1446 )
1447 .await
1448 }
1449
1450 #[inline]
1451 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
1452 LABELS
1453 .clone()
1454 .into_iter()
1455 .chain([
1456 (INSTANCE_LABEL.to_string(), self.name_any()),
1457 (CLUSTER_LABEL.to_string(), self.name_any()),
1458 ])
1459 .collect()
1460 }
1461
1462 #[inline]
1463 fn generate_labels(&self) -> BTreeMap<String, String> {
1464 self.generate_resource_labels()
1465 }
1466
1467 #[inline]
1468 fn get_tls_secret_name(&self) -> String {
1469 format!("{}-tls", self.name_any())
1470 }
1471
1472 pub(crate) fn effective_tls_secret_name(&self) -> String {
1475 self.spec.tls_secret_name.clone().unwrap_or_else(|| {
1476 self.spec
1477 .ingress
1478 .as_ref()
1479 .and_then(|i| i.tls_secret_name.clone())
1480 .unwrap_or_else(|| self.get_tls_secret_name())
1481 })
1482 }
1483
1484 #[inline]
1485 fn get_namespace(&self) -> String {
1486 self.namespace().unwrap()
1488 }
1489
1490 #[inline]
1491 fn is_replication_enabled(&self) -> bool {
1492 self.spec.replica_groups.len() > 1
1493 || self
1494 .spec
1495 .replica_groups
1496 .first()
1497 .is_some_and(|rg| rg.replicas > 1)
1498 || !self.spec.external_replication_nodes.is_empty()
1499 }
1500
1501 #[inline]
1502 fn is_updatable(&self, status: &KanidmStatus) -> bool {
1503 if self.spec.disable_upgrade_checks {
1506 return true;
1507 }
1508 status
1509 .version
1510 .as_ref()
1511 .map(|v| {
1512 if matches!(
1513 v.compatibility_result,
1514 VersionCompatibilityResult::Incompatible
1515 | VersionCompatibilityResult::NotChecked
1516 ) {
1517 return false;
1518 }
1519 let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
1520 let versions = (parse_semver(¤t_tag), parse_semver(&v.image_tag));
1521 match v.upgrade_check_result {
1522 KanidmUpgradeCheckResult::Passed => {
1523 if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
1524 minor <= v_minor + 1
1525 } else {
1526 true
1527 }
1528 }
1529 KanidmUpgradeCheckResult::Failed => {
1530 if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
1531 versions
1532 {
1533 major == v_major && minor == v_minor && patch >= v_patch
1534 } else {
1535 v.image_tag == current_tag
1536 }
1537 }
1538 KanidmUpgradeCheckResult::Skipped => false,
1540 }
1541 })
1542 .unwrap_or(true)
1543 }
1544
1545 async fn is_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<bool> {
1546 let pod_api = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1547 let pod = pod_api.get(pod_name).await.map_err(|e| {
1548 Error::KubeError(
1549 format!("failed to get pod {namespace}/{pod_name}"),
1550 Box::new(e),
1551 )
1552 })?;
1553 Ok(pod
1554 .status
1555 .as_ref()
1556 .and_then(|s| s.conditions.as_ref())
1557 .is_some_and(|conditions| {
1558 conditions
1559 .iter()
1560 .any(|c| c.type_ == "Ready" && c.status == "True")
1561 }))
1562 }
1563
1564 async fn wait_for_pod_ready(ctx: Arc<Context>, namespace: &str, pod_name: &str) -> Result<()> {
1565 let start = std::time::Instant::now();
1566 let timeout = Duration::from_secs(POD_READY_WAIT_TIMEOUT_SECONDS);
1567 let poll_interval = Duration::from_secs(POD_READY_POLL_INTERVAL_SECONDS);
1568
1569 loop {
1570 match Self::is_pod_ready(ctx.clone(), namespace, pod_name).await {
1571 Ok(true) => return Ok(()),
1572 Ok(false) => {}
1573 Err(e) => {
1574 if !e.is_retryable() {
1575 return Err(e);
1576 }
1577 trace!(%e, "pod readiness check failed, retrying");
1578 }
1579 }
1580
1581 if start.elapsed() >= timeout {
1582 return Err(Error::ReceiveOutput(format!(
1583 "pod {namespace}/{pod_name} not ready after {POD_READY_WAIT_TIMEOUT_SECONDS}s"
1584 )));
1585 }
1586
1587 sleep(poll_interval).await;
1588 }
1589 }
1590
1591 async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
1592 where
1593 I: IntoIterator<Item = T> + Debug,
1594 T: Into<String>,
1595 {
1596 let namespace = &self.get_namespace();
1597 trace!(
1598 resource.name = &pod_name,
1599 resource.namespace = &namespace,
1600 ?command,
1601 "pod exec"
1602 );
1603 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
1604 let attached = pod
1605 .exec(
1606 pod_name,
1607 command,
1608 &AttachParams::default().container("kanidm"),
1609 )
1610 .await
1611 .map_err(|e| {
1612 Error::KubeError(
1613 format!("failed to exec pod {namespace}/{pod_name}"),
1614 Box::new(e),
1615 )
1616 })?;
1617 get_output(attached).await
1618 }
1619
1620 async fn exec_with_wait<I, T>(
1621 &self,
1622 ctx: Arc<Context>,
1623 pod_name: &str,
1624 command: I,
1625 ) -> Result<String>
1626 where
1627 I: IntoIterator<Item = T> + Debug,
1628 T: Into<String>,
1629 {
1630 let namespace = &self.get_namespace();
1631 Self::wait_for_pod_ready(ctx.clone(), namespace, pod_name).await?;
1632 self.exec(ctx, pod_name, command).await
1633 }
1634
1635 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1636 where
1637 I: IntoIterator<Item = T> + Debug,
1638 T: Into<String>,
1639 {
1640 let first_replica_group = self
1642 .spec
1643 .replica_groups
1644 .first()
1645 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1646 let sts_name = self.statefulset_name(&first_replica_group.name);
1647 let pod_name = format!("{sts_name}-0");
1648 self.exec(ctx, &pod_name, command).await
1649 }
1650
1651 async fn exec_any_with_wait<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
1652 where
1653 I: IntoIterator<Item = T> + Debug,
1654 T: Into<String>,
1655 {
1656 let first_replica_group = self
1657 .spec
1658 .replica_groups
1659 .first()
1660 .ok_or_else(|| Error::MissingData("no replica groups configured".to_string()))?;
1661 let sts_name = self.statefulset_name(&first_replica_group.name);
1662 let pod_name = format!("{sts_name}-0");
1663 self.exec_with_wait(ctx, &pod_name, command).await
1664 }
1665}
1666
1667#[cfg(test)]
1668mod test {
1669 use super::statefulset::{StatefulSetExt, TLS_SECRET_HASH_ANNOTATION};
1670 use super::{CLUSTER_LABEL, Kanidm, previous_tls_secret_hash, reconcile_kanidm};
1671
1672 use crate::controller::State;
1673 use crate::kanidm::controller::context::{Context, Stores};
1674 use crate::kanidm::crd::KanidmStatus;
1675
1676 use kaniop_k8s_util::error::Result;
1677 use kaniop_k8s_util::resources::hash_secret_data;
1678
1679 use std::collections::BTreeMap;
1680 use std::sync::Arc;
1681
1682 use http::{Request, Response};
1683 use k8s_openapi::ByteString;
1684 use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec};
1685 use k8s_openapi::api::core::v1::{PodTemplateSpec, Secret, Service};
1686 use k8s_openapi::api::networking::v1::Ingress;
1687 use kube::api::ObjectMeta;
1688 use kube::runtime::reflector::store::Writer;
1689 use kube::{Client, Resource, ResourceExt, client::Body};
1690 use opentelemetry::metrics::MeterProvider;
1691 use serde_json::json;
1692
1693 impl Kanidm {
1694 pub fn test() -> Self {
1696 let mut e = Kanidm::new(
1697 "test",
1698 serde_json::from_value(json!({
1699 "domain": "idm.example.com",
1700 "replicaGroups": [
1701 {
1702 "name": "default",
1703 "replicas": 1
1704 }
1705 ]
1706 }))
1707 .unwrap(),
1708 );
1709 e.meta_mut().namespace = Some("default".into());
1710 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
1713 e
1714 }
1715
1716 pub fn with_ingress(mut self) -> Self {
1717 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
1718 self
1719 }
1720
1721 pub fn with_replicas(mut self, replicas: i32) -> Self {
1723 self.spec.replica_groups[0].replicas = replicas;
1724 self
1725 }
1726
1727 pub fn needs_delete(mut self) -> Self {
1729 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
1730 use k8s_openapi::jiff::Timestamp;
1731 let now = Timestamp::from_second(1491138632).unwrap(); self.meta_mut().deletion_timestamp = Some(Time(now));
1733 self
1734 }
1735
1736 pub fn with_status(mut self, status: KanidmStatus) -> Self {
1738 self.status = Some(status);
1739 self
1740 }
1741 }
1742
1743 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
1745 pub struct ApiServerVerifier(ApiServerHandle);
1746
1747 pub enum Scenario {
1749 Create(Kanidm),
1750 CreateWithTwoReplicas(Kanidm),
1751 CreateWithIngress(Kanidm),
1752 CreateWithIngressWithTwoReplicas(Kanidm),
1753 CreateWithTlsSecret(Kanidm, Box<Secret>),
1754 }
1755
1756 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
1757 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
1758 .await
1759 .expect("timeout on mock apiserver")
1760 .expect("scenario succeeded")
1761 }
1762
1763 impl ApiServerVerifier {
1764 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
1775 tokio::spawn(async move {
1776 match scenario {
1778 Scenario::Create(kanidm) => {
1779 self.handle_kanidm_get(kanidm.clone())
1780 .await
1781 .unwrap()
1782 .handle_kanidm_status_patch(kanidm.clone())
1783 .await
1784 .unwrap()
1785 .handle_tls_secret_get_not_found(kanidm.clone())
1786 .await
1787 .unwrap()
1788 .handle_backup_schedule_list()
1789 .await
1790 .unwrap()
1791 .handle_backup_repository_list()
1792 .await
1793 .unwrap()
1794 .handle_statefulset_patch(kanidm.clone(), None)
1795 .await
1796 .unwrap()
1797 .handle_service_patch(kanidm.clone())
1798 .await
1799 }
1800 Scenario::CreateWithTwoReplicas(kanidm) => {
1801 self.handle_kanidm_get(kanidm.clone())
1802 .await
1803 .unwrap()
1804 .handle_kanidm_status_patch(kanidm.clone())
1805 .await
1806 .unwrap()
1807 .handle_tls_secret_get_not_found(kanidm.clone())
1808 .await
1809 .unwrap()
1810 .handle_backup_schedule_list()
1811 .await
1812 .unwrap()
1813 .handle_backup_repository_list()
1814 .await
1815 .unwrap()
1816 .handle_statefulset_patch(kanidm.clone(), None)
1817 .await
1818 .unwrap()
1819 .handle_service_patch(kanidm.clone())
1820 .await
1821 }
1822 Scenario::CreateWithIngress(kanidm) => {
1823 self.handle_kanidm_get(kanidm.clone())
1824 .await
1825 .unwrap()
1826 .handle_kanidm_status_patch(kanidm.clone())
1827 .await
1828 .unwrap()
1829 .handle_tls_secret_get_not_found(kanidm.clone())
1830 .await
1831 .unwrap()
1832 .handle_backup_schedule_list()
1833 .await
1834 .unwrap()
1835 .handle_backup_repository_list()
1836 .await
1837 .unwrap()
1838 .handle_statefulset_patch(kanidm.clone(), None)
1839 .await
1840 .unwrap()
1841 .handle_service_patch(kanidm.clone())
1842 .await
1843 .unwrap()
1844 .handle_ingress_patch(kanidm.clone())
1845 .await
1846 }
1847 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
1848 self.handle_kanidm_get(kanidm.clone())
1849 .await
1850 .unwrap()
1851 .handle_kanidm_status_patch(kanidm.clone())
1852 .await
1853 .unwrap()
1854 .handle_tls_secret_get_not_found(kanidm.clone())
1855 .await
1856 .unwrap()
1857 .handle_backup_schedule_list()
1858 .await
1859 .unwrap()
1860 .handle_backup_repository_list()
1861 .await
1862 .unwrap()
1863 .handle_statefulset_patch(kanidm.clone(), None)
1864 .await
1865 .unwrap()
1866 .handle_service_patch(kanidm.clone())
1867 .await
1868 .unwrap()
1869 .handle_ingress_patch(kanidm.clone())
1870 .await
1871 }
1872 Scenario::CreateWithTlsSecret(kanidm, secret) => {
1873 let expected_hash = hash_secret_data(&secret);
1874 self.handle_kanidm_get(kanidm.clone())
1875 .await
1876 .unwrap()
1877 .handle_kanidm_status_patch(kanidm.clone())
1878 .await
1879 .unwrap()
1880 .handle_tls_secret_get(kanidm.clone(), *secret)
1881 .await
1882 .unwrap()
1883 .handle_backup_schedule_list()
1884 .await
1885 .unwrap()
1886 .handle_backup_repository_list()
1887 .await
1888 .unwrap()
1889 .handle_statefulset_patch(kanidm.clone(), Some(expected_hash))
1890 .await
1891 .unwrap()
1892 .handle_service_patch(kanidm.clone())
1893 .await
1894 }
1895 }
1896 .expect("scenario completed without errors");
1897 })
1898 }
1899
1900 async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
1901 let (request, send) = self.0.next_request().await.expect("service not called");
1902 assert_eq!(request.method(), http::Method::GET);
1903 let expected_uri_prefix = format!(
1904 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
1905 kanidm.name_any()
1906 );
1907 let uri = request.uri().to_string();
1908 assert!(
1909 uri.starts_with(&expected_uri_prefix),
1910 "expected uri to start with {}, got {}",
1911 expected_uri_prefix,
1912 uri
1913 );
1914 let response = serde_json::to_vec(&kanidm).unwrap();
1915 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1916 Ok(self)
1917 }
1918
1919 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
1920 let (request, send) = self.0.next_request().await.expect("service not called");
1921 assert_eq!(request.method(), http::Method::PATCH);
1922 assert_eq!(
1923 request.uri().to_string(),
1924 format!(
1925 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
1926 kanidm.name_any()
1927 )
1928 );
1929
1930 let req_body = request.into_body().collect_bytes().await.unwrap();
1931 let json: serde_json::Value =
1932 serde_json::from_slice(&req_body).expect("patch object is json");
1933 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
1934 .expect("valid kanidm status");
1935 let mut kanidm = Kanidm::test();
1936 kanidm.status = Some(status.clone());
1937 let response = serde_json::to_vec(&kanidm).unwrap();
1938 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1940 Ok(self)
1941 }
1942
1943 async fn handle_tls_secret_get_not_found(mut self, kanidm: Kanidm) -> Result<Self> {
1944 let (request, send) = self.0.next_request().await.expect("service not called");
1945 assert_eq!(request.method(), http::Method::GET);
1946 let expected_uri_prefix = format!(
1947 "/api/v1/namespaces/default/secrets/{}",
1948 kanidm.effective_tls_secret_name()
1949 );
1950 let uri = request.uri().to_string();
1951 assert!(
1952 uri.starts_with(&expected_uri_prefix),
1953 "expected uri to start with {expected_uri_prefix}, got {uri}"
1954 );
1955 let status = serde_json::json!({
1956 "kind": "Status",
1957 "apiVersion": "v1",
1958 "metadata": {},
1959 "status": "Failure",
1960 "message": "secret not found",
1961 "reason": "NotFound",
1962 "code": 404
1963 });
1964 send.send_response(
1965 Response::builder()
1966 .status(404)
1967 .body(Body::from(serde_json::to_vec(&status).unwrap()))
1968 .unwrap(),
1969 );
1970 Ok(self)
1971 }
1972
1973 async fn handle_tls_secret_get(mut self, kanidm: Kanidm, secret: Secret) -> Result<Self> {
1974 let (request, send) = self.0.next_request().await.expect("service not called");
1975 assert_eq!(request.method(), http::Method::GET);
1976 let expected_uri_prefix = format!(
1977 "/api/v1/namespaces/default/secrets/{}",
1978 kanidm.effective_tls_secret_name()
1979 );
1980 let uri = request.uri().to_string();
1981 assert!(
1982 uri.starts_with(&expected_uri_prefix),
1983 "expected uri to start with {expected_uri_prefix}, got {uri}"
1984 );
1985 let response = serde_json::to_vec(&secret).unwrap();
1986 send.send_response(Response::builder().body(Body::from(response)).unwrap());
1987 Ok(self)
1988 }
1989
1990 async fn handle_backup_schedule_list(mut self) -> Result<Self> {
1991 let (request, send) = self.0.next_request().await.expect("service not called");
1992 assert_eq!(request.method(), http::Method::GET);
1993 assert!(
1994 request
1995 .uri()
1996 .to_string()
1997 .contains("/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackupschedules"),
1998 "expected backup schedule list request, got {}",
1999 request.uri()
2000 );
2001 let list = json!({
2002 "apiVersion": "kaniop.rs/v1alpha1",
2003 "kind": "KanidmBackupScheduleList",
2004 "metadata": {},
2005 "items": []
2006 });
2007 let response = serde_json::to_vec(&list).unwrap();
2008 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2009 Ok(self)
2010 }
2011
2012 async fn handle_backup_repository_list(mut self) -> Result<Self> {
2013 let (request, send) = self.0.next_request().await.expect("service not called");
2014 assert_eq!(request.method(), http::Method::GET);
2015 assert!(
2016 request.uri().to_string().contains(
2017 "/apis/kaniop.rs/v1alpha1/namespaces/default/kanidmbackuprepositories"
2018 ),
2019 "expected backup repository list request, got {}",
2020 request.uri()
2021 );
2022 let list = json!({
2023 "apiVersion": "kaniop.rs/v1alpha1",
2024 "kind": "KanidmBackupRepositoryList",
2025 "metadata": {},
2026 "items": []
2027 });
2028 let response = serde_json::to_vec(&list).unwrap();
2029 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2030 Ok(self)
2031 }
2032
2033 async fn handle_statefulset_patch(
2034 mut self,
2035 kanidm: Kanidm,
2036 expected_tls_hash: Option<String>,
2037 ) -> Result<Self> {
2038 for rg in kanidm.spec.replica_groups.iter() {
2039 let (request, send) = self.0.next_request().await.expect("service not called");
2040 assert_eq!(request.method(), http::Method::PATCH);
2041 assert_eq!(
2042 request.uri().to_string(),
2043 format!(
2044 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2045 kanidm.statefulset_name(&rg.name)
2046 )
2047 );
2048 let req_body = request.into_body().collect_bytes().await.unwrap();
2049 let json: serde_json::Value =
2050 serde_json::from_slice(&req_body).expect("patch object is json");
2051 let statefulset: StatefulSet =
2052 serde_json::from_value(json).expect("valid statefulset");
2053 assert_eq!(
2054 statefulset.clone().spec.unwrap().replicas.unwrap(),
2055 rg.replicas
2056 );
2057 let tls_hash_annotation = statefulset
2058 .spec
2059 .as_ref()
2060 .unwrap()
2061 .template
2062 .metadata
2063 .as_ref()
2064 .unwrap()
2065 .annotations
2066 .as_ref()
2067 .and_then(|a| a.get(TLS_SECRET_HASH_ANNOTATION))
2068 .cloned();
2069 assert_eq!(tls_hash_annotation, expected_tls_hash);
2070 let response = serde_json::to_vec(&statefulset).unwrap();
2071 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2073 }
2074
2075 Ok(self)
2076 }
2077
2078 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2079 let (request, send) = self.0.next_request().await.expect("service not called");
2080 assert_eq!(request.method(), http::Method::PATCH);
2081 assert_eq!(
2082 request.uri().to_string(),
2083 format!(
2084 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2085 kanidm.name_any()
2086 )
2087 );
2088
2089 let req_body = request.into_body().collect_bytes().await.unwrap();
2090 let json: serde_json::Value =
2091 serde_json::from_slice(&req_body).expect("patch object is json");
2092 let service: Service = serde_json::from_value(json).expect("valid service");
2093 let response = serde_json::to_vec(&service).unwrap();
2094 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2096 Ok(self)
2097 }
2098
2099 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
2100 let (request, send) = self.0.next_request().await.expect("service not called");
2101 assert_eq!(request.method(), http::Method::PATCH);
2102 assert_eq!(
2103 request.uri().to_string(),
2104 format!(
2105 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
2106 kanidm.name_any()
2107 )
2108 );
2109
2110 let req_body = request.into_body().collect_bytes().await.unwrap();
2111 let json: serde_json::Value =
2112 serde_json::from_slice(&req_body).expect("patch object is json");
2113 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
2114 let response = serde_json::to_vec(&ingress).unwrap();
2115 send.send_response(Response::builder().body(Body::from(response)).unwrap());
2117 Ok(self)
2118 }
2119 }
2120
2121 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
2122 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2123 let mock_client = Client::new(mock_service, "default");
2124 let stores = Stores {
2125 stateful_set_store: Writer::default().as_reader(),
2126 service_store: Writer::default().as_reader(),
2127 ingress_store: Writer::default().as_reader(),
2128 secret_store: Writer::default().as_reader(),
2129 http_route_store: Some(Writer::default().as_reader()),
2130 backend_tls_policy_store: Some(Writer::default().as_reader()),
2131 deployment_store: Writer::default().as_reader(),
2132 config_map_store: Writer::default().as_reader(),
2133 };
2134 let controller_id = "test";
2135
2136 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2138 let meter = provider.meter("test");
2139 let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
2140
2141 let state = State::new(
2142 metrics,
2143 Writer::default().as_reader(),
2144 Writer::default().as_reader(),
2145 None,
2146 );
2147 let ctx = Arc::new(Context::new(
2148 state.to_context(mock_client, controller_id),
2149 stores,
2150 ));
2151 (ctx, ApiServerVerifier(handle))
2152 }
2153
2154 #[tokio::test]
2155 async fn kanidm_create() {
2156 let (testctx, fakeserver) = get_test_context();
2157 let kanidm = Kanidm::test();
2158 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
2159 reconcile_kanidm(Arc::new(kanidm), testctx)
2160 .await
2161 .expect("reconciler");
2162 timeout_after_1s(mocksrv).await;
2163 }
2164
2165 #[tokio::test]
2166 async fn kanidm_create_with_two_replicas() {
2167 let (testctx, fakeserver) = get_test_context();
2168 let kanidm = Kanidm::test().with_replicas(2);
2169 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
2170 reconcile_kanidm(Arc::new(kanidm), testctx)
2171 .await
2172 .expect("reconciler");
2173 timeout_after_1s(mocksrv).await;
2174 }
2175
2176 #[tokio::test]
2177 async fn kanidm_create_with_ingress() {
2178 let (testctx, fakeserver) = get_test_context();
2179 let kanidm = Kanidm::test().with_ingress();
2180 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
2181 reconcile_kanidm(Arc::new(kanidm), testctx)
2182 .await
2183 .expect("reconciler");
2184 timeout_after_1s(mocksrv).await;
2185 }
2186
2187 #[tokio::test]
2188 async fn kanidm_create_with_ingress_with_two_replicas() {
2189 let (testctx, fakeserver) = get_test_context();
2190 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
2191 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
2192 reconcile_kanidm(Arc::new(kanidm), testctx)
2193 .await
2194 .expect("reconciler");
2195 timeout_after_1s(mocksrv).await;
2196 }
2197
2198 fn statefulset_of(cluster: &str, namespace: &str, tls_hash: Option<&str>) -> Arc<StatefulSet> {
2199 Arc::new(StatefulSet {
2200 metadata: ObjectMeta {
2201 name: Some(format!("{cluster}-default")),
2202 namespace: Some(namespace.to_string()),
2203 labels: Some(BTreeMap::from([(
2204 CLUSTER_LABEL.to_string(),
2205 cluster.to_string(),
2206 )])),
2207 ..Default::default()
2208 },
2209 spec: Some(StatefulSetSpec {
2210 template: PodTemplateSpec {
2211 metadata: Some(ObjectMeta {
2212 annotations: tls_hash.map(|h| {
2213 BTreeMap::from([(
2214 TLS_SECRET_HASH_ANNOTATION.to_string(),
2215 h.to_string(),
2216 )])
2217 }),
2218 ..Default::default()
2219 }),
2220 ..Default::default()
2221 },
2222 ..Default::default()
2223 }),
2224 ..Default::default()
2225 })
2226 }
2227
2228 #[test]
2229 fn test_previous_tls_secret_hash() {
2230 let kanidm = Kanidm::test();
2231
2232 let stss = vec![statefulset_of("test", "default", Some("abc123"))];
2234 assert_eq!(
2235 previous_tls_secret_hash(&kanidm, stss),
2236 Some("abc123".to_string())
2237 );
2238
2239 let stss = vec![statefulset_of("test", "default", None)];
2241 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2242
2243 let stss = vec![
2245 statefulset_of("other", "default", Some("abc123")),
2246 statefulset_of("test", "other-ns", Some("abc123")),
2247 ];
2248 assert_eq!(previous_tls_secret_hash(&kanidm, stss), None);
2249 }
2250
2251 #[tokio::test]
2252 async fn kanidm_create_with_tls_secret() {
2253 let (testctx, fakeserver) = get_test_context();
2254 let kanidm = Kanidm::test();
2255 let secret = Secret {
2256 metadata: ObjectMeta {
2257 name: Some(kanidm.effective_tls_secret_name()),
2258 namespace: Some("default".to_string()),
2259 ..Default::default()
2260 },
2261 data: Some(BTreeMap::from([
2262 ("tls.crt".to_string(), ByteString(b"cert".to_vec())),
2263 ("tls.key".to_string(), ByteString(b"key".to_vec())),
2264 ])),
2265 ..Default::default()
2266 };
2267 let mocksrv = fakeserver.run(Scenario::CreateWithTlsSecret(
2268 kanidm.clone(),
2269 Box::new(secret),
2270 ));
2271 reconcile_kanidm(Arc::new(kanidm), testctx)
2272 .await
2273 .expect("reconciler");
2274 timeout_after_1s(mocksrv).await;
2275 }
2276
2277 #[tokio::test]
2278 async fn replication_secret_gc_skips_non_replica_secrets() {
2279 use super::reconcile_replication_secrets;
2280 use super::secret::{SECRET_TYPE_LABEL, SecretType};
2281
2282 let kanidm = Kanidm::test();
2283 let status = kanidm.status.clone().unwrap_or_default();
2284
2285 let replica_cert_label =
2286 serde_plain::to_string(&SecretType::ReplicaCert).expect("SecretType serializes");
2287
2288 let replica_cert_secret = Secret {
2289 metadata: ObjectMeta {
2290 name: Some(format!("{}-default-0-cert", kanidm.name_any())),
2291 namespace: Some("default".into()),
2292 labels: Some(BTreeMap::from([
2293 (CLUSTER_LABEL.to_string(), kanidm.name_any()),
2294 (SECRET_TYPE_LABEL.to_string(), replica_cert_label.clone()),
2295 ])),
2296 ..Default::default()
2297 },
2298 ..Default::default()
2299 };
2300
2301 let mail_sender_secret = Secret {
2302 metadata: ObjectMeta {
2303 name: Some(format!("{}-mail-sender-config", kanidm.name_any())),
2304 namespace: Some("default".into()),
2305 labels: Some(BTreeMap::from([(
2306 CLUSTER_LABEL.to_string(),
2307 kanidm.name_any(),
2308 )])),
2309 ..Default::default()
2310 },
2311 ..Default::default()
2312 };
2313
2314 let (mock_service, mut handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
2315 let mock_client = Client::new(mock_service, "default");
2316
2317 let mut secret_writer = Writer::default();
2318 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2319 replica_cert_secret.clone(),
2320 ));
2321 secret_writer.apply_watcher_event(&kube::runtime::watcher::Event::Apply(
2322 mail_sender_secret.clone(),
2323 ));
2324
2325 let stores = Stores {
2326 stateful_set_store: Writer::default().as_reader(),
2327 service_store: Writer::default().as_reader(),
2328 ingress_store: Writer::default().as_reader(),
2329 secret_store: secret_writer.as_reader(),
2330 http_route_store: Some(Writer::default().as_reader()),
2331 backend_tls_policy_store: Some(Writer::default().as_reader()),
2332 deployment_store: Writer::default().as_reader(),
2333 config_map_store: Writer::default().as_reader(),
2334 };
2335
2336 let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
2337 let meter = provider.meter("test");
2338 let metrics = crate::metrics::Metrics::new(&meter, &["test"]);
2339 let state = State::new(
2340 metrics,
2341 Writer::default().as_reader(),
2342 Writer::default().as_reader(),
2343 None,
2344 );
2345 let ctx = Arc::new(Context::new(state.to_context(mock_client, "test"), stores));
2346
2347 let mock_srv = tokio::spawn(async move {
2348 let (request, send) = handle.next_request().await.expect("expected a request");
2349 assert_eq!(request.method(), http::Method::DELETE);
2350 let uri = request.uri().to_string();
2351 assert!(
2352 uri.contains("-default-0-cert"),
2353 "expected delete for replica cert secret, got {uri}"
2354 );
2355 let not_found = serde_json::json!({
2356 "kind": "Status",
2357 "apiVersion": "v1",
2358 "metadata": {},
2359 "status": "Failure",
2360 "message": "not found",
2361 "reason": "NotFound",
2362 "code": 404
2363 });
2364 send.send_response(
2365 Response::builder()
2366 .status(404)
2367 .body(Body::from(serde_json::to_vec(¬_found).unwrap()))
2368 .unwrap(),
2369 );
2370
2371 if let Ok(Some((req, _))) =
2372 tokio::time::timeout(std::time::Duration::from_millis(200), handle.next_request())
2373 .await
2374 {
2375 panic!("unexpected second request: {} {}", req.method(), req.uri());
2376 }
2377 });
2378
2379 let changed = reconcile_replication_secrets(Arc::new(kanidm), ctx, &status)
2380 .await
2381 .expect("reconcile succeeds despite 404 on delete");
2382 assert!(
2383 changed,
2384 "expected changed=true since a deprecated secret was found"
2385 );
2386
2387 tokio::time::timeout(std::time::Duration::from_secs(1), mock_srv)
2388 .await
2389 .expect("timeout waiting for mock server")
2390 .expect("mock server task succeeded");
2391 }
2392
2393 #[test]
2394 fn disable_upgrade_checks_allows_unchecked_version_status() {
2395 let mut kanidm = Kanidm::test();
2396 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2397 kanidm.spec.disable_upgrade_checks = true;
2398 let status = KanidmStatus {
2399 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2400 image_tag: "1.11.2".to_string(),
2401 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2402 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2403 }),
2404 ..Default::default()
2405 };
2406 assert!(kanidm.is_updatable(&status));
2407 }
2408
2409 #[test]
2410 fn disable_upgrade_checks_allows_previous_incompatible_status() {
2411 let mut kanidm = Kanidm::test();
2412 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2413 kanidm.spec.disable_upgrade_checks = true;
2414 let status = KanidmStatus {
2415 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2416 image_tag: "1.11.2".to_string(),
2417 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2418 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2419 }),
2420 ..Default::default()
2421 };
2422 assert!(kanidm.is_updatable(&status));
2423 }
2424
2425 #[test]
2426 fn upgrade_checks_still_block_incompatible_versions_when_enabled() {
2427 let mut kanidm = Kanidm::test();
2428 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2429 kanidm.spec.disable_upgrade_checks = false;
2430 let status = KanidmStatus {
2431 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2432 image_tag: "1.11.2".to_string(),
2433 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Passed,
2434 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::Incompatible,
2435 }),
2436 ..Default::default()
2437 };
2438 assert!(!kanidm.is_updatable(&status));
2439 }
2440
2441 #[test]
2442 fn upgrade_checks_do_not_treat_skipped_checks_as_passed() {
2443 let mut kanidm = Kanidm::test();
2444 kanidm.spec.image = "kanidm/server:1.12.0".to_string();
2445 kanidm.spec.disable_upgrade_checks = false;
2446 let status = KanidmStatus {
2447 version: Some(crate::kanidm::crd::KanidmVersionStatus {
2448 image_tag: "1.11.2".to_string(),
2449 upgrade_check_result: crate::kanidm::crd::KanidmUpgradeCheckResult::Skipped,
2450 compatibility_result: crate::kanidm::crd::VersionCompatibilityResult::NotChecked,
2451 }),
2452 ..Default::default()
2453 };
2454 assert!(!kanidm.is_updatable(&status));
2455 }
2456}