1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::stream::{self, StreamExt};
29use futures::try_join;
30use k8s_openapi::NamespaceResourceScope;
31use k8s_openapi::api::apps::v1::StatefulSet;
32use k8s_openapi::api::core::v1::Pod;
33use kube::Resource;
34use kube::ResourceExt;
35use kube::api::{Api, AttachParams};
36use serde::{Deserialize, Serialize};
37
38use kube::runtime::controller::Action;
39use status::{is_kanidm_available, is_kanidm_initialized};
40use tracing::{Span, debug, field, info, instrument, trace};
41
42pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
43const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
44
45static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
46 BTreeMap::from([
47 (NAME_LABEL.to_string(), "kanidm".to_string()),
48 (
49 MANAGED_BY_LABEL.to_string(),
50 format!("kaniop-{CONTROLLER_ID}"),
51 ),
52 ])
53});
54
55pub async fn reconcile_admins_secret(
56 kanidm: Arc<Kanidm>,
57 ctx: Arc<Context>,
58 status: &Result<KanidmStatus>,
59) -> Result<()> {
60 if let Ok(s) = status {
61 if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
62 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
63 kanidm.patch(&ctx, admins_secret).await?;
64 }
65 }
66 Ok(())
67}
68
69pub async fn reconcile_replication_secrets(
70 kanidm: Arc<Kanidm>,
71 ctx: Arc<Context>,
72 status: &Result<KanidmStatus>,
73) -> Result<()> {
74 if let Ok(s) = status {
75 let secret_names = s
76 .replica_statuses
77 .iter()
78 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
79 .collect::<Vec<_>>();
80 let deprecated_secrets = ctx
81 .stores
82 .secret_store
83 .state()
84 .into_iter()
85 .filter(|secret| {
86 secret
87 .metadata
88 .labels
89 .iter()
90 .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
91 && kanidm.admins_secret_name() != secret.name_any()
92 && secret_names.iter().all(|sn| sn != &secret.name_any())
93 })
94 .collect::<Vec<_>>();
95 let secret_delete_future = deprecated_secrets
96 .iter()
97 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
98 .collect::<TryJoinAll<_>>();
99 try_join!(secret_delete_future)?;
100
101 if kanidm.is_replication_enabled() {
102 let pending_secrets = s
103 .replica_statuses
104 .iter()
105 .filter(|rs| rs.state == KanidmReplicaState::Pending)
106 .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
107 .collect::<TryJoinAll<_>>();
108
109 let expiring_secrets = s
110 .replica_statuses
111 .iter()
112 .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
113 .map(|rs| kanidm.update_replica_secret(ctx.clone(), &rs.pod_name))
114 .collect::<TryJoinAll<_>>();
115
116 let (pending_results, expiring_results) =
117 futures::try_join!(pending_secrets, expiring_secrets)?;
118
119 let secrets = pending_results
120 .into_iter()
121 .chain(expiring_results)
122 .collect::<Vec<_>>();
123 let secret_futures = secrets
124 .into_iter()
125 .map(|secret| kanidm.patch(&ctx, secret))
126 .collect::<Vec<_>>();
127 try_join_all(secret_futures).await?;
128
129 let has_pending_or_expiring = s.replica_statuses.iter().any(|rs| {
132 rs.state == KanidmReplicaState::Pending
133 || rs.state == KanidmReplicaState::CertificateExpiring
134 });
135
136 if has_pending_or_expiring {
137 let sts_api = Api::<StatefulSet>::namespaced(
138 ctx.kaniop_ctx.client.clone(),
139 &kanidm.get_namespace(),
140 );
141
142 let has_single_replica =
144 kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
145
146 let restart_futures = s
147 .replica_statuses
148 .iter()
149 .map(|rs| sts_api.restart(&rs.statefulset_name));
150
151 if has_single_replica {
152 let _ignore_errors = stream::iter(restart_futures)
154 .then(|f| f)
155 .collect::<Vec<_>>()
156 .await;
157 } else {
158 let _ignore_errors = join_all(restart_futures).await;
160 }
161 }
162 }
163 }
164 Ok(())
165}
166
167#[instrument(skip(ctx, kanidm))]
168pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
169 let trace_id = telemetry::get_trace_id();
170 Span::current().record("trace_id", field::display(&trace_id));
171 let _timer = ctx
172 .kaniop_ctx
173 .metrics
174 .reconcile_count_and_measure(&trace_id);
175 info!(msg = "reconciling Kanidm");
176
177 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
178 debug!(msg = "failed to reconcile status", %e);
179 ctx.kaniop_ctx.metrics.status_update_errors_inc();
180 e
181 });
182
183 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
184 let replication_secret_future =
185 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
186
187 let sts_to_delete = ctx
188 .stores
189 .stateful_set_store
190 .state()
191 .into_iter()
192 .filter(|sts| {
193 sts.metadata.labels.iter().any(|l| {
194 l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
195 && match l.get(REPLICA_GROUP_LABEL) {
196 Some(sts_rg_name) => kanidm
197 .spec
198 .replica_groups
199 .iter()
200 .all(|rg| &rg.name != sts_rg_name),
201 None => false,
202 }
203 })
204 })
205 .collect::<Vec<_>>();
206 let sts_delete_future = sts_to_delete
207 .iter()
208 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
209 .collect::<TryJoinAll<_>>();
210
211 let sts_futures = kanidm
212 .spec
213 .replica_groups
214 .iter()
215 .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
216 .collect::<TryJoinAll<_>>();
217 let service_future = kanidm.patch(&ctx, kanidm.create_service());
218 let ingress_future = kanidm
219 .create_ingress()
220 .into_iter()
221 .map(|ingress| kanidm.patch(&ctx, ingress))
222 .collect::<TryJoinAll<_>>();
223
224 try_join!(
225 sts_delete_future,
226 admin_secret_future,
227 replication_secret_future,
228 sts_futures,
229 service_future,
230 ingress_future
231 )?;
232 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
233}
234
235impl Kanidm {
236 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
238 where
239 K: Resource<Scope = NamespaceResourceScope>
240 + Serialize
241 + Clone
242 + std::fmt::Debug
243 + for<'de> Deserialize<'de>,
244 <K as kube::Resource>::DynamicType: Default,
245 <K as Resource>::Scope: std::marker::Sized,
246 {
247 self.kube_delete(
248 ctx.kaniop_ctx.client.clone(),
249 &ctx.kaniop_ctx.metrics,
250 resource,
251 )
252 .await
253 }
254
255 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
256 where
257 K: Resource<Scope = NamespaceResourceScope>
258 + Serialize
259 + Clone
260 + std::fmt::Debug
261 + for<'de> Deserialize<'de>,
262 <K as kube::Resource>::DynamicType: Default,
263 <K as Resource>::Scope: std::marker::Sized,
264 {
265 self.kube_patch(
266 ctx.kaniop_ctx.client.clone(),
267 &ctx.kaniop_ctx.metrics,
268 resource,
269 KANIDM_OPERATOR_NAME,
270 )
271 .await
272 }
273
274 #[inline]
275 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
276 LABELS
277 .clone()
278 .into_iter()
279 .chain([
280 (INSTANCE_LABEL.to_string(), self.name_any()),
281 (CLUSTER_LABEL.to_string(), self.name_any()),
282 ])
283 .collect()
284 }
285
286 #[inline]
287 fn get_tls_secret_name(&self) -> String {
288 format!("{}-tls", self.name_any())
289 }
290
291 #[inline]
292 fn get_namespace(&self) -> String {
293 self.namespace().unwrap()
295 }
296
297 #[inline]
298 fn is_replication_enabled(&self) -> bool {
299 self.spec.replica_groups.len() > 1
301 || self.spec.replica_groups.first().unwrap().replicas > 1
302 || !self.spec.external_replication_nodes.is_empty()
303 }
304
305 async fn exec<I, T>(
306 &self,
307 ctx: Arc<Context>,
308 pod_name: &str,
309 command: I,
310 ) -> Result<Option<String>>
311 where
312 I: IntoIterator<Item = T> + Debug,
313 T: Into<String>,
314 {
315 let namespace = &self.get_namespace();
316 trace!(
317 msg = "pod exec",
318 resource.name = &pod_name,
319 resource.namespace = &namespace,
320 ?command
321 );
322 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
323 let attached = pod
324 .exec(pod_name, command, &AttachParams::default().stderr(false))
325 .await
326 .map_err(|e| {
327 Error::KubeError(
328 format!("failed to exec pod {namespace}/{pod_name}"),
329 Box::new(e),
330 )
331 })?;
332 Ok(get_output(attached).await)
333 }
334
335 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
336 where
337 I: IntoIterator<Item = T> + Debug,
338 T: Into<String>,
339 {
340 let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
343 let pod_name = format!("{sts_name}-0");
344 self.exec(ctx, &pod_name, command).await
345 }
346}
347
348#[cfg(test)]
349mod test {
350 use super::statefulset::StatefulSetExt;
351 use super::{Kanidm, reconcile_kanidm};
352
353 use crate::controller::State;
354 use crate::error::Result;
355 use crate::kanidm::controller::context::{Context, Stores};
356 use crate::kanidm::crd::KanidmStatus;
357 use k8s_openapi::api::core::v1::Service;
358 use k8s_openapi::api::networking::v1::Ingress;
359
360 use std::sync::Arc;
361
362 use http::{Request, Response};
363 use k8s_openapi::api::apps::v1::StatefulSet;
364 use kube::runtime::reflector::store::Writer;
365 use kube::{Client, Resource, ResourceExt, client::Body};
366 use serde_json::json;
367
368 impl Kanidm {
369 pub fn test() -> Self {
371 let mut e = Kanidm::new(
372 "test",
373 serde_json::from_value(json!({
374 "domain": "idm.example.com",
375 "replicaGroups": [
376 {
377 "name": "default",
378 "replicas": 1
379 }
380 ]
381 }))
382 .unwrap(),
383 );
384 e.meta_mut().namespace = Some("default".into());
385 e
386 }
387
388 pub fn with_ingress(mut self) -> Self {
389 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
390 self
391 }
392
393 pub fn with_replicas(mut self, replicas: i32) -> Self {
395 self.spec.replica_groups[0].replicas = replicas;
397 self
398 }
399
400 pub fn needs_delete(mut self) -> Self {
402 use chrono::prelude::{DateTime, TimeZone, Utc};
403 let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
404 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
405 self.meta_mut().deletion_timestamp = Some(Time(now));
406 self
407 }
408
409 pub fn with_status(mut self, status: KanidmStatus) -> Self {
411 self.status = Some(status);
412 self
413 }
414 }
415
416 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
418 pub struct ApiServerVerifier(ApiServerHandle);
419
420 pub enum Scenario {
422 Create(Kanidm),
423 CreateWithTwoReplicas(Kanidm),
424 CreateWithIngress(Kanidm),
425 CreateWithIngressWithTwoReplicas(Kanidm),
426 }
427
428 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
429 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
430 .await
431 .expect("timeout on mock apiserver")
432 .expect("scenario succeeded")
433 }
434
435 impl ApiServerVerifier {
436 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
447 tokio::spawn(async move {
448 match scenario {
450 Scenario::Create(kanidm) => {
451 self.handle_kanidm_status_patch(kanidm.clone())
452 .await
453 .unwrap()
454 .handle_statefulset_patch(kanidm.clone())
455 .await
456 .unwrap()
457 .handle_service_patch(kanidm.clone())
458 .await
459 }
460 Scenario::CreateWithTwoReplicas(kanidm) => {
461 self.handle_kanidm_status_patch(kanidm.clone())
462 .await
463 .unwrap()
464 .handle_statefulset_patch(kanidm.clone())
465 .await
466 .unwrap()
467 .handle_service_patch(kanidm.clone())
468 .await
469 }
470 Scenario::CreateWithIngress(kanidm) => {
471 self.handle_kanidm_status_patch(kanidm.clone())
472 .await
473 .unwrap()
474 .handle_statefulset_patch(kanidm.clone())
475 .await
476 .unwrap()
477 .handle_service_patch(kanidm.clone())
478 .await
479 .unwrap()
480 .handle_ingress_patch(kanidm.clone())
481 .await
482 }
483 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
484 self.handle_kanidm_status_patch(kanidm.clone())
485 .await
486 .unwrap()
487 .handle_statefulset_patch(kanidm.clone())
488 .await
489 .unwrap()
490 .handle_service_patch(kanidm.clone())
491 .await
492 .unwrap()
493 .handle_ingress_patch(kanidm.clone())
494 .await
495 }
496 }
497 .expect("scenario completed without errors");
498 })
499 }
500
501 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
502 let (request, send) = self.0.next_request().await.expect("service not called");
503 assert_eq!(request.method(), http::Method::PATCH);
504 assert_eq!(
505 request.uri().to_string(),
506 format!(
507 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
508 kanidm.name_any()
509 )
510 );
511
512 let req_body = request.into_body().collect_bytes().await.unwrap();
513 let json: serde_json::Value =
514 serde_json::from_slice(&req_body).expect("patch object is json");
515 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
516 .expect("valid kanidm status");
517 let response = serde_json::to_vec(&status).unwrap();
518 send.send_response(Response::builder().body(Body::from(response)).unwrap());
520 Ok(self)
521 }
522
523 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
524 for rg in kanidm.spec.replica_groups.iter() {
525 let (request, send) = self.0.next_request().await.expect("service not called");
526 assert_eq!(request.method(), http::Method::PATCH);
527 assert_eq!(
528 request.uri().to_string(),
529 format!(
530 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
531 kanidm.statefulset_name(&rg.name)
532 )
533 );
534 let req_body = request.into_body().collect_bytes().await.unwrap();
535 let json: serde_json::Value =
536 serde_json::from_slice(&req_body).expect("patch object is json");
537 let statefulset: StatefulSet =
538 serde_json::from_value(json).expect("valid statefulset");
539 assert_eq!(
540 statefulset.clone().spec.unwrap().replicas.unwrap(),
541 rg.replicas
542 );
543 let response = serde_json::to_vec(&statefulset).unwrap();
544 send.send_response(Response::builder().body(Body::from(response)).unwrap());
546 }
547
548 Ok(self)
549 }
550
551 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
552 let (request, send) = self.0.next_request().await.expect("service not called");
553 assert_eq!(request.method(), http::Method::PATCH);
554 assert_eq!(
555 request.uri().to_string(),
556 format!(
557 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
558 kanidm.name_any()
559 )
560 );
561
562 let req_body = request.into_body().collect_bytes().await.unwrap();
563 let json: serde_json::Value =
564 serde_json::from_slice(&req_body).expect("patch object is json");
565 let service: Service = serde_json::from_value(json).expect("valid service");
566 let response = serde_json::to_vec(&service).unwrap();
567 send.send_response(Response::builder().body(Body::from(response)).unwrap());
569 Ok(self)
570 }
571
572 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
573 let (request, send) = self.0.next_request().await.expect("service not called");
574 assert_eq!(request.method(), http::Method::PATCH);
575 assert_eq!(
576 request.uri().to_string(),
577 format!(
578 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
579 kanidm.name_any()
580 )
581 );
582
583 let req_body = request.into_body().collect_bytes().await.unwrap();
584 let json: serde_json::Value =
585 serde_json::from_slice(&req_body).expect("patch object is json");
586 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
587 let response = serde_json::to_vec(&ingress).unwrap();
588 send.send_response(Response::builder().body(Body::from(response)).unwrap());
590 Ok(self)
591 }
592 }
593
594 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
595 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
596 let mock_client = Client::new(mock_service, "default");
597 let stores = Stores {
598 stateful_set_store: Writer::default().as_reader(),
599 service_store: Writer::default().as_reader(),
600 ingress_store: Writer::default().as_reader(),
601 secret_store: Writer::default().as_reader(),
602 };
603 let controller_id = "test";
604 let state = State::new(
605 Default::default(),
606 &[controller_id],
607 Writer::default().as_reader(),
608 Writer::default().as_reader(),
609 );
610 let ctx = Arc::new(Context::new(
611 state.to_context(mock_client, controller_id),
612 stores,
613 ));
614 (ctx, ApiServerVerifier(handle))
615 }
616
617 #[tokio::test]
618 async fn kanidm_create() {
619 let (testctx, fakeserver) = get_test_context();
620 let kanidm = Kanidm::test();
621 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
622 reconcile_kanidm(Arc::new(kanidm), testctx)
623 .await
624 .expect("reconciler");
625 timeout_after_1s(mocksrv).await;
626 }
627
628 #[tokio::test]
629 async fn kanidm_create_with_two_replicas() {
630 let (testctx, fakeserver) = get_test_context();
631 let kanidm = Kanidm::test().with_replicas(2);
632 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
633 reconcile_kanidm(Arc::new(kanidm), testctx)
634 .await
635 .expect("reconciler");
636 timeout_after_1s(mocksrv).await;
637 }
638
639 #[tokio::test]
640 async fn kanidm_create_with_ingress() {
641 let (testctx, fakeserver) = get_test_context();
642 let kanidm = Kanidm::test().with_ingress();
643 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
644 reconcile_kanidm(Arc::new(kanidm), testctx)
645 .await
646 .expect("reconciler");
647 timeout_after_1s(mocksrv).await;
648 }
649
650 #[tokio::test]
651 async fn kanidm_create_with_ingress_with_two_replicas() {
652 let (testctx, fakeserver) = get_test_context();
653 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
654 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
655 reconcile_kanidm(Arc::new(kanidm), testctx)
656 .await
657 .expect("reconciler");
658 timeout_after_1s(mocksrv).await;
659 }
660}