1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::stream::{self, StreamExt};
29use futures::try_join;
30use k8s_openapi::NamespaceResourceScope;
31use k8s_openapi::api::apps::v1::StatefulSet;
32use k8s_openapi::api::core::v1::Pod;
33use kube::Resource;
34use kube::ResourceExt;
35use kube::api::{Api, AttachParams};
36use kube::runtime::finalizer::{Event as Finalizer, finalizer};
37use serde::{Deserialize, Serialize};
38
39use kube::runtime::controller::Action;
40use status::{is_kanidm_available, is_kanidm_initialized};
41use tracing::{Span, debug, field, info, instrument, trace};
42
43pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
44const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
45pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
46
47static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
48 BTreeMap::from([
49 (NAME_LABEL.to_string(), "kanidm".to_string()),
50 (
51 MANAGED_BY_LABEL.to_string(),
52 format!("kaniop-{CONTROLLER_ID}"),
53 ),
54 ])
55});
56
57pub async fn reconcile_admins_secret(
58 kanidm: Arc<Kanidm>,
59 ctx: Arc<Context>,
60 status: &Result<KanidmStatus>,
61) -> Result<()> {
62 if let Ok(s) = status {
63 if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
64 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
65 kanidm.patch(&ctx, admins_secret).await?;
66 }
67 }
68 Ok(())
69}
70
71pub async fn reconcile_replication_secrets(
72 kanidm: Arc<Kanidm>,
73 ctx: Arc<Context>,
74 status: &Result<KanidmStatus>,
75) -> Result<()> {
76 if let Ok(s) = status {
77 let secret_names = s
78 .replica_statuses
79 .iter()
80 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
81 .collect::<Vec<_>>();
82 let deprecated_secrets = ctx
83 .stores
84 .secret_store
85 .state()
86 .into_iter()
87 .filter(|secret| {
88 secret
89 .metadata
90 .labels
91 .iter()
92 .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
93 && kanidm.admins_secret_name() != secret.name_any()
94 && secret_names.iter().all(|sn| sn != &secret.name_any())
95 })
96 .collect::<Vec<_>>();
97 let secret_delete_future = deprecated_secrets
98 .iter()
99 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
100 .collect::<TryJoinAll<_>>();
101 try_join!(secret_delete_future)?;
102
103 if kanidm.is_replication_enabled() {
104 let pending_secrets = s
105 .replica_statuses
106 .iter()
107 .filter(|rs| rs.state == KanidmReplicaState::Pending)
108 .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
109 .collect::<TryJoinAll<_>>();
110
111 let expiring_secrets = s
112 .replica_statuses
113 .iter()
114 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
115 .map(|rs| kanidm.update_replica_secret(ctx.clone(), &rs.pod_name))
116 .collect::<TryJoinAll<_>>();
117
118 let (pending_results, expiring_results) =
119 futures::try_join!(pending_secrets, expiring_secrets)?;
120
121 let secrets = pending_results
122 .into_iter()
123 .chain(expiring_results)
124 .collect::<Vec<_>>();
125 let secret_futures = secrets
126 .into_iter()
127 .map(|secret| kanidm.patch(&ctx, secret))
128 .collect::<Vec<_>>();
129 try_join_all(secret_futures).await?;
130
131 let has_pending_or_expiring = s.replica_statuses.iter().any(|rs| {
134 rs.state == KanidmReplicaState::Pending
135 || rs.state == KanidmReplicaState::CertificateExpiring
136 });
137
138 if has_pending_or_expiring {
139 let sts_api = Api::<StatefulSet>::namespaced(
140 ctx.kaniop_ctx.client.clone(),
141 &kanidm.get_namespace(),
142 );
143
144 let has_single_replica =
146 kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
147
148 let restart_futures = s
149 .replica_statuses
150 .iter()
151 .map(|rs| sts_api.restart(&rs.statefulset_name));
152
153 if has_single_replica {
154 let _ignore_errors = stream::iter(restart_futures)
156 .then(|f| f)
157 .collect::<Vec<_>>()
158 .await;
159 } else {
160 let _ignore_errors = join_all(restart_futures).await;
162 }
163 }
164 }
165 }
166 Ok(())
167}
168
169#[instrument(skip(ctx, kanidm))]
170pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
171 let trace_id = telemetry::get_trace_id();
172 Span::current().record("trace_id", field::display(&trace_id));
173 let _timer = ctx
174 .kaniop_ctx
175 .metrics
176 .reconcile_count_and_measure(&trace_id);
177 info!(msg = "reconciling Kanidm");
178
179 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
180 debug!(msg = "failed to reconcile status", %e);
181 ctx.kaniop_ctx.metrics.status_update_errors_inc();
182 e
183 });
184 let kanidm_api: Api<Kanidm> =
185 Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
186 finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
187 match event {
188 Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
189 Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
190 }
191 })
192 .await
193 .map_err(|e| {
194 Error::FinalizerError(
195 "failed on kanidm account finalizer".to_string(),
196 Box::new(e),
197 )
198 })
199}
200
201async fn reconcile(
202 kanidm: Arc<Kanidm>,
203 ctx: Arc<Context>,
204 status: Result<KanidmStatus>,
206) -> Result<Action> {
207 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
208 let replication_secret_future =
209 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
210
211 let sts_to_delete = ctx
212 .stores
213 .stateful_set_store
214 .state()
215 .into_iter()
216 .filter(|sts| {
217 sts.metadata.labels.iter().any(|l| {
218 l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
219 && match l.get(REPLICA_GROUP_LABEL) {
220 Some(sts_rg_name) => kanidm
221 .spec
222 .replica_groups
223 .iter()
224 .all(|rg| &rg.name != sts_rg_name),
225 None => false,
226 }
227 })
228 })
229 .collect::<Vec<_>>();
230 let sts_delete_future = sts_to_delete
231 .iter()
232 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
233 .collect::<TryJoinAll<_>>();
234
235 let sts_futures = kanidm
236 .spec
237 .replica_groups
238 .iter()
239 .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
240 .collect::<TryJoinAll<_>>();
241 let service_future = kanidm.patch(&ctx, kanidm.create_service());
242 let ingress_future = kanidm
243 .create_ingress()
244 .into_iter()
245 .map(|ingress| kanidm.patch(&ctx, ingress))
246 .collect::<TryJoinAll<_>>();
247
248 try_join!(
249 sts_delete_future,
250 admin_secret_future,
251 replication_secret_future,
252 sts_futures,
253 service_future,
254 ingress_future
255 )?;
256 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
257}
258
259async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
260 debug!(msg = "cleanup");
261 ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
262 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
263}
264
265impl Kanidm {
266 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
268 where
269 K: Resource<Scope = NamespaceResourceScope>
270 + Serialize
271 + Clone
272 + std::fmt::Debug
273 + for<'de> Deserialize<'de>,
274 <K as kube::Resource>::DynamicType: Default,
275 <K as Resource>::Scope: std::marker::Sized,
276 {
277 self.kube_delete(
278 ctx.kaniop_ctx.client.clone(),
279 &ctx.kaniop_ctx.metrics,
280 resource,
281 )
282 .await
283 }
284
285 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
286 where
287 K: Resource<Scope = NamespaceResourceScope>
288 + Serialize
289 + Clone
290 + std::fmt::Debug
291 + for<'de> Deserialize<'de>,
292 <K as kube::Resource>::DynamicType: Default,
293 <K as Resource>::Scope: std::marker::Sized,
294 {
295 self.kube_patch(
296 ctx.kaniop_ctx.client.clone(),
297 &ctx.kaniop_ctx.metrics,
298 resource,
299 KANIDM_OPERATOR_NAME,
300 )
301 .await
302 }
303
304 #[inline]
305 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
306 LABELS
307 .clone()
308 .into_iter()
309 .chain([
310 (INSTANCE_LABEL.to_string(), self.name_any()),
311 (CLUSTER_LABEL.to_string(), self.name_any()),
312 ])
313 .collect()
314 }
315
316 #[inline]
317 fn get_tls_secret_name(&self) -> String {
318 format!("{}-tls", self.name_any())
319 }
320
321 #[inline]
322 fn get_namespace(&self) -> String {
323 self.namespace().unwrap()
325 }
326
327 #[inline]
328 fn is_replication_enabled(&self) -> bool {
329 self.spec.replica_groups.len() > 1
331 || self.spec.replica_groups.first().unwrap().replicas > 1
332 || !self.spec.external_replication_nodes.is_empty()
333 }
334
335 async fn exec<I, T>(
336 &self,
337 ctx: Arc<Context>,
338 pod_name: &str,
339 command: I,
340 ) -> Result<Option<String>>
341 where
342 I: IntoIterator<Item = T> + Debug,
343 T: Into<String>,
344 {
345 let namespace = &self.get_namespace();
346 trace!(
347 msg = "pod exec",
348 resource.name = &pod_name,
349 resource.namespace = &namespace,
350 ?command
351 );
352 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
353 let attached = pod
354 .exec(pod_name, command, &AttachParams::default().stderr(false))
355 .await
356 .map_err(|e| {
357 Error::KubeError(
358 format!("failed to exec pod {namespace}/{pod_name}"),
359 Box::new(e),
360 )
361 })?;
362 Ok(get_output(attached).await)
363 }
364
365 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
366 where
367 I: IntoIterator<Item = T> + Debug,
368 T: Into<String>,
369 {
370 let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
373 let pod_name = format!("{sts_name}-0");
374 self.exec(ctx, &pod_name, command).await
375 }
376}
377
378#[cfg(test)]
379mod test {
380 use super::statefulset::StatefulSetExt;
381 use super::{Kanidm, reconcile_kanidm};
382
383 use crate::controller::State;
384 use crate::error::Result;
385 use crate::kanidm::controller::context::{Context, Stores};
386 use crate::kanidm::crd::KanidmStatus;
387 use k8s_openapi::api::core::v1::Service;
388 use k8s_openapi::api::networking::v1::Ingress;
389
390 use std::sync::Arc;
391
392 use http::{Request, Response};
393 use k8s_openapi::api::apps::v1::StatefulSet;
394 use kube::runtime::reflector::store::Writer;
395 use kube::{Client, Resource, ResourceExt, client::Body};
396 use serde_json::json;
397
398 impl Kanidm {
399 pub fn test() -> Self {
401 let mut e = Kanidm::new(
402 "test",
403 serde_json::from_value(json!({
404 "domain": "idm.example.com",
405 "replicaGroups": [
406 {
407 "name": "default",
408 "replicas": 1
409 }
410 ]
411 }))
412 .unwrap(),
413 );
414 e.meta_mut().namespace = Some("default".into());
415 e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
418 e
419 }
420
421 pub fn with_ingress(mut self) -> Self {
422 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
423 self
424 }
425
426 pub fn with_replicas(mut self, replicas: i32) -> Self {
428 self.spec.replica_groups[0].replicas = replicas;
430 self
431 }
432
433 pub fn needs_delete(mut self) -> Self {
435 use chrono::prelude::{DateTime, TimeZone, Utc};
436 let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
437 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
438 self.meta_mut().deletion_timestamp = Some(Time(now));
439 self
440 }
441
442 pub fn with_status(mut self, status: KanidmStatus) -> Self {
444 self.status = Some(status);
445 self
446 }
447 }
448
449 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
451 pub struct ApiServerVerifier(ApiServerHandle);
452
453 pub enum Scenario {
455 Create(Kanidm),
456 CreateWithTwoReplicas(Kanidm),
457 CreateWithIngress(Kanidm),
458 CreateWithIngressWithTwoReplicas(Kanidm),
459 }
460
461 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
462 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
463 .await
464 .expect("timeout on mock apiserver")
465 .expect("scenario succeeded")
466 }
467
468 impl ApiServerVerifier {
469 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
480 tokio::spawn(async move {
481 match scenario {
483 Scenario::Create(kanidm) => {
484 self.handle_kanidm_status_patch(kanidm.clone())
485 .await
486 .unwrap()
487 .handle_statefulset_patch(kanidm.clone())
488 .await
489 .unwrap()
490 .handle_service_patch(kanidm.clone())
491 .await
492 }
493 Scenario::CreateWithTwoReplicas(kanidm) => {
494 self.handle_kanidm_status_patch(kanidm.clone())
495 .await
496 .unwrap()
497 .handle_statefulset_patch(kanidm.clone())
498 .await
499 .unwrap()
500 .handle_service_patch(kanidm.clone())
501 .await
502 }
503 Scenario::CreateWithIngress(kanidm) => {
504 self.handle_kanidm_status_patch(kanidm.clone())
505 .await
506 .unwrap()
507 .handle_statefulset_patch(kanidm.clone())
508 .await
509 .unwrap()
510 .handle_service_patch(kanidm.clone())
511 .await
512 .unwrap()
513 .handle_ingress_patch(kanidm.clone())
514 .await
515 }
516 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
517 self.handle_kanidm_status_patch(kanidm.clone())
518 .await
519 .unwrap()
520 .handle_statefulset_patch(kanidm.clone())
521 .await
522 .unwrap()
523 .handle_service_patch(kanidm.clone())
524 .await
525 .unwrap()
526 .handle_ingress_patch(kanidm.clone())
527 .await
528 }
529 }
530 .expect("scenario completed without errors");
531 })
532 }
533
534 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
535 let (request, send) = self.0.next_request().await.expect("service not called");
536 assert_eq!(request.method(), http::Method::PATCH);
537 assert_eq!(
538 request.uri().to_string(),
539 format!(
540 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
541 kanidm.name_any()
542 )
543 );
544
545 let req_body = request.into_body().collect_bytes().await.unwrap();
546 let json: serde_json::Value =
547 serde_json::from_slice(&req_body).expect("patch object is json");
548 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
549 .expect("valid kanidm status");
550 let response = serde_json::to_vec(&status).unwrap();
551 send.send_response(Response::builder().body(Body::from(response)).unwrap());
553 Ok(self)
554 }
555
556 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
557 for rg in kanidm.spec.replica_groups.iter() {
558 let (request, send) = self.0.next_request().await.expect("service not called");
559 assert_eq!(request.method(), http::Method::PATCH);
560 assert_eq!(
561 request.uri().to_string(),
562 format!(
563 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
564 kanidm.statefulset_name(&rg.name)
565 )
566 );
567 let req_body = request.into_body().collect_bytes().await.unwrap();
568 let json: serde_json::Value =
569 serde_json::from_slice(&req_body).expect("patch object is json");
570 let statefulset: StatefulSet =
571 serde_json::from_value(json).expect("valid statefulset");
572 assert_eq!(
573 statefulset.clone().spec.unwrap().replicas.unwrap(),
574 rg.replicas
575 );
576 let response = serde_json::to_vec(&statefulset).unwrap();
577 send.send_response(Response::builder().body(Body::from(response)).unwrap());
579 }
580
581 Ok(self)
582 }
583
584 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
585 let (request, send) = self.0.next_request().await.expect("service not called");
586 assert_eq!(request.method(), http::Method::PATCH);
587 assert_eq!(
588 request.uri().to_string(),
589 format!(
590 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
591 kanidm.name_any()
592 )
593 );
594
595 let req_body = request.into_body().collect_bytes().await.unwrap();
596 let json: serde_json::Value =
597 serde_json::from_slice(&req_body).expect("patch object is json");
598 let service: Service = serde_json::from_value(json).expect("valid service");
599 let response = serde_json::to_vec(&service).unwrap();
600 send.send_response(Response::builder().body(Body::from(response)).unwrap());
602 Ok(self)
603 }
604
605 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
606 let (request, send) = self.0.next_request().await.expect("service not called");
607 assert_eq!(request.method(), http::Method::PATCH);
608 assert_eq!(
609 request.uri().to_string(),
610 format!(
611 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
612 kanidm.name_any()
613 )
614 );
615
616 let req_body = request.into_body().collect_bytes().await.unwrap();
617 let json: serde_json::Value =
618 serde_json::from_slice(&req_body).expect("patch object is json");
619 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
620 let response = serde_json::to_vec(&ingress).unwrap();
621 send.send_response(Response::builder().body(Body::from(response)).unwrap());
623 Ok(self)
624 }
625 }
626
627 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
628 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
629 let mock_client = Client::new(mock_service, "default");
630 let stores = Stores {
631 stateful_set_store: Writer::default().as_reader(),
632 service_store: Writer::default().as_reader(),
633 ingress_store: Writer::default().as_reader(),
634 secret_store: Writer::default().as_reader(),
635 };
636 let controller_id = "test";
637 let state = State::new(
638 Default::default(),
639 &[controller_id],
640 Writer::default().as_reader(),
641 Writer::default().as_reader(),
642 );
643 let ctx = Arc::new(Context::new(
644 state.to_context(mock_client, controller_id),
645 stores,
646 ));
647 (ctx, ApiServerVerifier(handle))
648 }
649
650 #[tokio::test]
651 async fn kanidm_create() {
652 let (testctx, fakeserver) = get_test_context();
653 let kanidm = Kanidm::test();
654 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
655 reconcile_kanidm(Arc::new(kanidm), testctx)
656 .await
657 .expect("reconciler");
658 timeout_after_1s(mocksrv).await;
659 }
660
661 #[tokio::test]
662 async fn kanidm_create_with_two_replicas() {
663 let (testctx, fakeserver) = get_test_context();
664 let kanidm = Kanidm::test().with_replicas(2);
665 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
666 reconcile_kanidm(Arc::new(kanidm), testctx)
667 .await
668 .expect("reconciler");
669 timeout_after_1s(mocksrv).await;
670 }
671
672 #[tokio::test]
673 async fn kanidm_create_with_ingress() {
674 let (testctx, fakeserver) = get_test_context();
675 let kanidm = Kanidm::test().with_ingress();
676 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
677 reconcile_kanidm(Arc::new(kanidm), testctx)
678 .await
679 .expect("reconciler");
680 timeout_after_1s(mocksrv).await;
681 }
682
683 #[tokio::test]
684 async fn kanidm_create_with_ingress_with_two_replicas() {
685 let (testctx, fakeserver) = get_test_context();
686 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
687 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
688 reconcile_kanidm(Arc::new(kanidm), testctx)
689 .await
690 .expect("reconciler");
691 timeout_after_1s(mocksrv).await;
692 }
693}