1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::try_join;
29use k8s_openapi::NamespaceResourceScope;
30use k8s_openapi::api::apps::v1::StatefulSet;
31use k8s_openapi::api::core::v1::Pod;
32use kube::Resource;
33use kube::ResourceExt;
34use kube::api::{Api, AttachParams};
35use serde::{Deserialize, Serialize};
36
37use kube::runtime::controller::Action;
38use status::{is_kanidm_available, is_kanidm_initialized};
39use tracing::{Span, debug, field, info, instrument, trace};
40
41pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
42const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
43
44static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
45 BTreeMap::from([
46 (NAME_LABEL.to_string(), "kanidm".to_string()),
47 (
48 MANAGED_BY_LABEL.to_string(),
49 format!("kaniop-{CONTROLLER_ID}"),
50 ),
51 ])
52});
53
54pub async fn reconcile_admins_secret(
55 kanidm: Arc<Kanidm>,
56 ctx: Arc<Context>,
57 status: &Result<KanidmStatus>,
58) -> Result<()> {
59 if let Ok(s) = status {
60 if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
61 let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
62 kanidm.patch(&ctx, admins_secret).await?;
63 }
64 }
65 Ok(())
66}
67
68pub async fn reconcile_replication_secrets(
69 kanidm: Arc<Kanidm>,
70 ctx: Arc<Context>,
71 status: &Result<KanidmStatus>,
72) -> Result<()> {
73 if let Ok(s) = status {
74 let secret_names = s
75 .replica_statuses
76 .iter()
77 .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
78 .collect::<Vec<_>>();
79 let deprecated_secrets = ctx
80 .stores
81 .secret_store
82 .state()
83 .into_iter()
84 .filter(|secret| {
85 secret
86 .metadata
87 .labels
88 .iter()
89 .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
90 && kanidm.admins_secret_name() != secret.name_any()
91 && secret_names.iter().all(|sn| sn != &secret.name_any())
92 })
93 .collect::<Vec<_>>();
94 let secret_delete_future = deprecated_secrets
95 .iter()
96 .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
97 .collect::<TryJoinAll<_>>();
98 try_join!(secret_delete_future)?;
99
100 if kanidm.is_replication_enabled() {
101 let generate_secret_futures = s
102 .replica_statuses
103 .iter()
104 .filter(|rs| rs.state == KanidmReplicaState::Pending)
105 .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
106 .collect::<Vec<_>>();
107 let secrets = try_join_all(generate_secret_futures).await?;
108 let secret_futures = secrets
109 .into_iter()
110 .map(|secret| kanidm.patch(&ctx, secret))
111 .collect::<Vec<_>>();
112 try_join_all(secret_futures).await?;
113 let sts_api = Api::<StatefulSet>::namespaced(
116 ctx.kaniop_ctx.client.clone(),
117 &kanidm.get_namespace(),
118 );
119 let sts_restart_futures = s
120 .replica_statuses
121 .iter()
122 .filter(|rs| rs.state == KanidmReplicaState::Pending)
123 .map(|rs| sts_api.restart(&rs.statefulset_name));
124 let _ignore_errors = join_all(sts_restart_futures).await;
125 }
126 }
127 Ok(())
128}
129
130#[instrument(skip(ctx, kanidm))]
131pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
132 let trace_id = telemetry::get_trace_id();
133 Span::current().record("trace_id", field::display(&trace_id));
134 let _timer = ctx
135 .kaniop_ctx
136 .metrics
137 .reconcile_count_and_measure(&trace_id);
138 info!(msg = "reconciling Kanidm");
139
140 let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
141 debug!(msg = "failed to reconcile status", %e);
142 ctx.kaniop_ctx.metrics.status_update_errors_inc();
143 e
144 });
145
146 let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
147 let replication_secret_future =
148 reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
149
150 let sts_to_delete = ctx
151 .stores
152 .stateful_set_store
153 .state()
154 .into_iter()
155 .filter(|sts| {
156 sts.metadata.labels.iter().any(|l| {
157 l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
158 && match l.get(REPLICA_GROUP_LABEL) {
159 Some(sts_rg_name) => kanidm
160 .spec
161 .replica_groups
162 .iter()
163 .all(|rg| &rg.name != sts_rg_name),
164 None => false,
165 }
166 })
167 })
168 .collect::<Vec<_>>();
169 let sts_delete_future = sts_to_delete
170 .iter()
171 .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
172 .collect::<TryJoinAll<_>>();
173
174 let sts_futures = kanidm
175 .spec
176 .replica_groups
177 .iter()
178 .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
179 .collect::<TryJoinAll<_>>();
180 let service_future = kanidm.patch(&ctx, kanidm.create_service());
181 let ingress_future = kanidm
182 .create_ingress()
183 .into_iter()
184 .map(|ingress| kanidm.patch(&ctx, ingress))
185 .collect::<TryJoinAll<_>>();
186
187 try_join!(
188 sts_delete_future,
189 admin_secret_future,
190 replication_secret_future,
191 sts_futures,
192 service_future,
193 ingress_future
194 )?;
195 Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
196}
197
198impl Kanidm {
199 pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
201 where
202 K: Resource<Scope = NamespaceResourceScope>
203 + Serialize
204 + Clone
205 + std::fmt::Debug
206 + for<'de> Deserialize<'de>,
207 <K as kube::Resource>::DynamicType: Default,
208 <K as Resource>::Scope: std::marker::Sized,
209 {
210 self.kube_delete(
211 ctx.kaniop_ctx.client.clone(),
212 &ctx.kaniop_ctx.metrics,
213 resource,
214 )
215 .await
216 }
217
218 pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
219 where
220 K: Resource<Scope = NamespaceResourceScope>
221 + Serialize
222 + Clone
223 + std::fmt::Debug
224 + for<'de> Deserialize<'de>,
225 <K as kube::Resource>::DynamicType: Default,
226 <K as Resource>::Scope: std::marker::Sized,
227 {
228 self.kube_patch(
229 ctx.kaniop_ctx.client.clone(),
230 &ctx.kaniop_ctx.metrics,
231 resource,
232 KANIDM_OPERATOR_NAME,
233 )
234 .await
235 }
236
237 #[inline]
238 fn generate_resource_labels(&self) -> BTreeMap<String, String> {
239 LABELS
240 .clone()
241 .into_iter()
242 .chain([
243 (INSTANCE_LABEL.to_string(), self.name_any()),
244 (CLUSTER_LABEL.to_string(), self.name_any()),
245 ])
246 .collect()
247 }
248
249 #[inline]
250 fn get_tls_secret_name(&self) -> String {
251 format!("{}-tls", self.name_any())
252 }
253
254 #[inline]
255 fn get_namespace(&self) -> String {
256 self.namespace().unwrap()
258 }
259
260 #[inline]
261 fn is_replication_enabled(&self) -> bool {
262 self.spec.replica_groups.len() > 1
264 || self.spec.replica_groups.first().unwrap().replicas > 1
265 || !self.spec.external_replication_nodes.is_empty()
266 }
267
268 async fn exec<I, T>(
269 &self,
270 ctx: Arc<Context>,
271 pod_name: &str,
272 command: I,
273 ) -> Result<Option<String>>
274 where
275 I: IntoIterator<Item = T> + Debug,
276 T: Into<String>,
277 {
278 let namespace = &self.get_namespace();
279 trace!(
280 msg = "pod exec",
281 resource.name = &pod_name,
282 resource.namespace = &namespace,
283 ?command
284 );
285 let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
286 let attached = pod
287 .exec(pod_name, command, &AttachParams::default().stderr(false))
288 .await
289 .map_err(|e| {
290 Error::KubeError(
291 format!("failed to exec pod {namespace}/{pod_name}"),
292 Box::new(e),
293 )
294 })?;
295 Ok(get_output(attached).await)
296 }
297
298 async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
299 where
300 I: IntoIterator<Item = T> + Debug,
301 T: Into<String>,
302 {
303 let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
306 let pod_name = format!("{sts_name}-0");
307 self.exec(ctx, &pod_name, command).await
308 }
309}
310
311#[cfg(test)]
312mod test {
313 use super::statefulset::StatefulSetExt;
314 use super::{Kanidm, reconcile_kanidm};
315
316 use crate::controller::State;
317 use crate::error::Result;
318 use crate::kanidm::controller::context::{Context, Stores};
319 use crate::kanidm::crd::KanidmStatus;
320 use k8s_openapi::api::core::v1::Service;
321 use k8s_openapi::api::networking::v1::Ingress;
322
323 use std::sync::Arc;
324
325 use http::{Request, Response};
326 use k8s_openapi::api::apps::v1::StatefulSet;
327 use kube::runtime::reflector::store::Writer;
328 use kube::{Client, Resource, ResourceExt, client::Body};
329 use serde_json::json;
330
331 impl Kanidm {
332 pub fn test() -> Self {
334 let mut e = Kanidm::new(
335 "test",
336 serde_json::from_value(json!({
337 "domain": "idm.example.com",
338 "replicaGroups": [
339 {
340 "name": "default",
341 "replicas": 1
342 }
343 ]
344 }))
345 .unwrap(),
346 );
347 e.meta_mut().namespace = Some("default".into());
348 e
349 }
350
351 pub fn with_ingress(mut self) -> Self {
352 self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
353 self
354 }
355
356 pub fn with_replicas(mut self, replicas: i32) -> Self {
358 self.spec.replica_groups[0].replicas = replicas;
360 self
361 }
362
363 pub fn needs_delete(mut self) -> Self {
365 use chrono::prelude::{DateTime, TimeZone, Utc};
366 let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
367 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
368 self.meta_mut().deletion_timestamp = Some(Time(now));
369 self
370 }
371
372 pub fn with_status(mut self, status: KanidmStatus) -> Self {
374 self.status = Some(status);
375 self
376 }
377 }
378
379 type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
381 pub struct ApiServerVerifier(ApiServerHandle);
382
383 pub enum Scenario {
385 Create(Kanidm),
386 CreateWithTwoReplicas(Kanidm),
387 CreateWithIngress(Kanidm),
388 CreateWithIngressWithTwoReplicas(Kanidm),
389 }
390
391 pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
392 tokio::time::timeout(std::time::Duration::from_secs(1), handle)
393 .await
394 .expect("timeout on mock apiserver")
395 .expect("scenario succeeded")
396 }
397
398 impl ApiServerVerifier {
399 pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
410 tokio::spawn(async move {
411 match scenario {
413 Scenario::Create(kanidm) => {
414 self.handle_kanidm_status_patch(kanidm.clone())
415 .await
416 .unwrap()
417 .handle_statefulset_patch(kanidm.clone())
418 .await
419 .unwrap()
420 .handle_service_patch(kanidm.clone())
421 .await
422 }
423 Scenario::CreateWithTwoReplicas(kanidm) => {
424 self.handle_kanidm_status_patch(kanidm.clone())
425 .await
426 .unwrap()
427 .handle_statefulset_patch(kanidm.clone())
428 .await
429 .unwrap()
430 .handle_service_patch(kanidm.clone())
431 .await
432 }
433 Scenario::CreateWithIngress(kanidm) => {
434 self.handle_kanidm_status_patch(kanidm.clone())
435 .await
436 .unwrap()
437 .handle_statefulset_patch(kanidm.clone())
438 .await
439 .unwrap()
440 .handle_service_patch(kanidm.clone())
441 .await
442 .unwrap()
443 .handle_ingress_patch(kanidm.clone())
444 .await
445 }
446 Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
447 self.handle_kanidm_status_patch(kanidm.clone())
448 .await
449 .unwrap()
450 .handle_statefulset_patch(kanidm.clone())
451 .await
452 .unwrap()
453 .handle_service_patch(kanidm.clone())
454 .await
455 .unwrap()
456 .handle_ingress_patch(kanidm.clone())
457 .await
458 }
459 }
460 .expect("scenario completed without errors");
461 })
462 }
463
464 async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
465 let (request, send) = self.0.next_request().await.expect("service not called");
466 assert_eq!(request.method(), http::Method::PATCH);
467 assert_eq!(
468 request.uri().to_string(),
469 format!(
470 "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
471 kanidm.name_any()
472 )
473 );
474
475 let req_body = request.into_body().collect_bytes().await.unwrap();
476 let json: serde_json::Value =
477 serde_json::from_slice(&req_body).expect("patch object is json");
478 let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
479 .expect("valid kanidm status");
480 let response = serde_json::to_vec(&status).unwrap();
481 send.send_response(Response::builder().body(Body::from(response)).unwrap());
483 Ok(self)
484 }
485
486 async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
487 for rg in kanidm.spec.replica_groups.iter() {
488 let (request, send) = self.0.next_request().await.expect("service not called");
489 assert_eq!(request.method(), http::Method::PATCH);
490 assert_eq!(
491 request.uri().to_string(),
492 format!(
493 "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
494 kanidm.statefulset_name(&rg.name)
495 )
496 );
497 let req_body = request.into_body().collect_bytes().await.unwrap();
498 let json: serde_json::Value =
499 serde_json::from_slice(&req_body).expect("patch object is json");
500 let statefulset: StatefulSet =
501 serde_json::from_value(json).expect("valid statefulset");
502 assert_eq!(
503 statefulset.clone().spec.unwrap().replicas.unwrap(),
504 rg.replicas
505 );
506 let response = serde_json::to_vec(&statefulset).unwrap();
507 send.send_response(Response::builder().body(Body::from(response)).unwrap());
509 }
510
511 Ok(self)
512 }
513
514 async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
515 let (request, send) = self.0.next_request().await.expect("service not called");
516 assert_eq!(request.method(), http::Method::PATCH);
517 assert_eq!(
518 request.uri().to_string(),
519 format!(
520 "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
521 kanidm.name_any()
522 )
523 );
524
525 let req_body = request.into_body().collect_bytes().await.unwrap();
526 let json: serde_json::Value =
527 serde_json::from_slice(&req_body).expect("patch object is json");
528 let service: Service = serde_json::from_value(json).expect("valid service");
529 let response = serde_json::to_vec(&service).unwrap();
530 send.send_response(Response::builder().body(Body::from(response)).unwrap());
532 Ok(self)
533 }
534
535 async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
536 let (request, send) = self.0.next_request().await.expect("service not called");
537 assert_eq!(request.method(), http::Method::PATCH);
538 assert_eq!(
539 request.uri().to_string(),
540 format!(
541 "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
542 kanidm.name_any()
543 )
544 );
545
546 let req_body = request.into_body().collect_bytes().await.unwrap();
547 let json: serde_json::Value =
548 serde_json::from_slice(&req_body).expect("patch object is json");
549 let ingress: Ingress = serde_json::from_value(json).expect("valid service");
550 let response = serde_json::to_vec(&ingress).unwrap();
551 send.send_response(Response::builder().body(Body::from(response)).unwrap());
553 Ok(self)
554 }
555 }
556
557 pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
558 let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
559 let mock_client = Client::new(mock_service, "default");
560 let stores = Stores {
561 stateful_set_store: Writer::default().as_reader(),
562 service_store: Writer::default().as_reader(),
563 ingress_store: Writer::default().as_reader(),
564 secret_store: Writer::default().as_reader(),
565 };
566 let controller_id = "test";
567 let state = State::new(
568 Default::default(),
569 &[controller_id],
570 Writer::default().as_reader(),
571 Writer::default().as_reader(),
572 );
573 let ctx = Arc::new(Context::new(
574 state.to_context(mock_client, controller_id),
575 stores,
576 ));
577 (ctx, ApiServerVerifier(handle))
578 }
579
580 #[tokio::test]
581 async fn kanidm_create() {
582 let (testctx, fakeserver) = get_test_context();
583 let kanidm = Kanidm::test();
584 let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
585 reconcile_kanidm(Arc::new(kanidm), testctx)
586 .await
587 .expect("reconciler");
588 timeout_after_1s(mocksrv).await;
589 }
590
591 #[tokio::test]
592 async fn kanidm_create_with_two_replicas() {
593 let (testctx, fakeserver) = get_test_context();
594 let kanidm = Kanidm::test().with_replicas(2);
595 let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
596 reconcile_kanidm(Arc::new(kanidm), testctx)
597 .await
598 .expect("reconciler");
599 timeout_after_1s(mocksrv).await;
600 }
601
602 #[tokio::test]
603 async fn kanidm_create_with_ingress() {
604 let (testctx, fakeserver) = get_test_context();
605 let kanidm = Kanidm::test().with_ingress();
606 let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
607 reconcile_kanidm(Arc::new(kanidm), testctx)
608 .await
609 .expect("reconciler");
610 timeout_after_1s(mocksrv).await;
611 }
612
613 #[tokio::test]
614 async fn kanidm_create_with_ingress_with_two_replicas() {
615 let (testctx, fakeserver) = get_test_context();
616 let kanidm = Kanidm::test().with_ingress().with_replicas(2);
617 let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
618 reconcile_kanidm(Arc::new(kanidm), testctx)
619 .await
620 .expect("reconciler");
621 timeout_after_1s(mocksrv).await;
622 }
623}