kaniop_operator/kanidm/reconcile/
mod.rs

1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::stream::{self, StreamExt};
29use futures::try_join;
30use k8s_openapi::NamespaceResourceScope;
31use k8s_openapi::api::apps::v1::StatefulSet;
32use k8s_openapi::api::core::v1::Pod;
33use kube::Resource;
34use kube::ResourceExt;
35use kube::api::{Api, AttachParams};
36use serde::{Deserialize, Serialize};
37
38use kube::runtime::controller::Action;
39use status::{is_kanidm_available, is_kanidm_initialized};
40use tracing::{Span, debug, field, info, instrument, trace};
41
42pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
43const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
44
45static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
46    BTreeMap::from([
47        (NAME_LABEL.to_string(), "kanidm".to_string()),
48        (
49            MANAGED_BY_LABEL.to_string(),
50            format!("kaniop-{CONTROLLER_ID}"),
51        ),
52    ])
53});
54
55pub async fn reconcile_admins_secret(
56    kanidm: Arc<Kanidm>,
57    ctx: Arc<Context>,
58    status: &Result<KanidmStatus>,
59) -> Result<()> {
60    if let Ok(s) = status {
61        if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
62            let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
63            kanidm.patch(&ctx, admins_secret).await?;
64        }
65    }
66    Ok(())
67}
68
69pub async fn reconcile_replication_secrets(
70    kanidm: Arc<Kanidm>,
71    ctx: Arc<Context>,
72    status: &Result<KanidmStatus>,
73) -> Result<()> {
74    if let Ok(s) = status {
75        let secret_names = s
76            .replica_statuses
77            .iter()
78            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
79            .collect::<Vec<_>>();
80        let deprecated_secrets = ctx
81            .stores
82            .secret_store
83            .state()
84            .into_iter()
85            .filter(|secret| {
86                secret
87                    .metadata
88                    .labels
89                    .iter()
90                    .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
91                    && kanidm.admins_secret_name() != secret.name_any()
92                    && secret_names.iter().all(|sn| sn != &secret.name_any())
93            })
94            .collect::<Vec<_>>();
95        let secret_delete_future = deprecated_secrets
96            .iter()
97            .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
98            .collect::<TryJoinAll<_>>();
99        try_join!(secret_delete_future)?;
100
101        if kanidm.is_replication_enabled() {
102            let pending_secrets = s
103                .replica_statuses
104                .iter()
105                .filter(|rs| rs.state == KanidmReplicaState::Pending)
106                .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
107                .collect::<TryJoinAll<_>>();
108
109            let expiring_secrets = s
110                .replica_statuses
111                .iter()
112                .filter(|rs| rs.state == KanidmReplicaState::CertificateExpiring)
113                .map(|rs| kanidm.update_replica_secret(ctx.clone(), &rs.pod_name))
114                .collect::<TryJoinAll<_>>();
115
116            let (pending_results, expiring_results) =
117                futures::try_join!(pending_secrets, expiring_secrets)?;
118
119            let secrets = pending_results
120                .into_iter()
121                .chain(expiring_results)
122                .collect::<Vec<_>>();
123            let secret_futures = secrets
124                .into_iter()
125                .map(|secret| kanidm.patch(&ctx, secret))
126                .collect::<Vec<_>>();
127            try_join_all(secret_futures).await?;
128
129            // If there are any replicas in pending or cert expiring state, trigger a restart
130            // of all statefulsets to pickup new certs
131            let has_pending_or_expiring = s.replica_statuses.iter().any(|rs| {
132                rs.state == KanidmReplicaState::Pending
133                    || rs.state == KanidmReplicaState::CertificateExpiring
134            });
135
136            if has_pending_or_expiring {
137                let sts_api = Api::<StatefulSet>::namespaced(
138                    ctx.kaniop_ctx.client.clone(),
139                    &kanidm.get_namespace(),
140                );
141
142                // Check if any replica group has only 1 replica
143                let has_single_replica =
144                    kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
145
146                let restart_futures = s
147                    .replica_statuses
148                    .iter()
149                    .map(|rs| sts_api.restart(&rs.statefulset_name));
150
151                if has_single_replica {
152                    // Restart sequentially to avoid downtime
153                    let _ignore_errors = stream::iter(restart_futures)
154                        .then(|f| f)
155                        .collect::<Vec<_>>()
156                        .await;
157                } else {
158                    // Restart concurrently for replica groups with multiple replicas
159                    let _ignore_errors = join_all(restart_futures).await;
160                }
161            }
162        }
163    }
164    Ok(())
165}
166
167#[instrument(skip(ctx, kanidm))]
168pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
169    let trace_id = telemetry::get_trace_id();
170    Span::current().record("trace_id", field::display(&trace_id));
171    let _timer = ctx
172        .kaniop_ctx
173        .metrics
174        .reconcile_count_and_measure(&trace_id);
175    info!(msg = "reconciling Kanidm");
176
177    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
178        debug!(msg = "failed to reconcile status", %e);
179        ctx.kaniop_ctx.metrics.status_update_errors_inc();
180        e
181    });
182
183    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
184    let replication_secret_future =
185        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
186
187    let sts_to_delete = ctx
188        .stores
189        .stateful_set_store
190        .state()
191        .into_iter()
192        .filter(|sts| {
193            sts.metadata.labels.iter().any(|l| {
194                l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
195                    && match l.get(REPLICA_GROUP_LABEL) {
196                        Some(sts_rg_name) => kanidm
197                            .spec
198                            .replica_groups
199                            .iter()
200                            .all(|rg| &rg.name != sts_rg_name),
201                        None => false,
202                    }
203            })
204        })
205        .collect::<Vec<_>>();
206    let sts_delete_future = sts_to_delete
207        .iter()
208        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
209        .collect::<TryJoinAll<_>>();
210
211    let sts_futures = kanidm
212        .spec
213        .replica_groups
214        .iter()
215        .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
216        .collect::<TryJoinAll<_>>();
217    let service_future = kanidm.patch(&ctx, kanidm.create_service());
218    let ingress_future = kanidm
219        .create_ingress()
220        .into_iter()
221        .map(|ingress| kanidm.patch(&ctx, ingress))
222        .collect::<TryJoinAll<_>>();
223
224    try_join!(
225        sts_delete_future,
226        admin_secret_future,
227        replication_secret_future,
228        sts_futures,
229        service_future,
230        ingress_future
231    )?;
232    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
233}
234
235impl Kanidm {
236    // Convenience methods that handle context and operator name
237    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
238    where
239        K: Resource<Scope = NamespaceResourceScope>
240            + Serialize
241            + Clone
242            + std::fmt::Debug
243            + for<'de> Deserialize<'de>,
244        <K as kube::Resource>::DynamicType: Default,
245        <K as Resource>::Scope: std::marker::Sized,
246    {
247        self.kube_delete(
248            ctx.kaniop_ctx.client.clone(),
249            &ctx.kaniop_ctx.metrics,
250            resource,
251        )
252        .await
253    }
254
255    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
256    where
257        K: Resource<Scope = NamespaceResourceScope>
258            + Serialize
259            + Clone
260            + std::fmt::Debug
261            + for<'de> Deserialize<'de>,
262        <K as kube::Resource>::DynamicType: Default,
263        <K as Resource>::Scope: std::marker::Sized,
264    {
265        self.kube_patch(
266            ctx.kaniop_ctx.client.clone(),
267            &ctx.kaniop_ctx.metrics,
268            resource,
269            KANIDM_OPERATOR_NAME,
270        )
271        .await
272    }
273
274    #[inline]
275    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
276        LABELS
277            .clone()
278            .into_iter()
279            .chain([
280                (INSTANCE_LABEL.to_string(), self.name_any()),
281                (CLUSTER_LABEL.to_string(), self.name_any()),
282            ])
283            .collect()
284    }
285
286    #[inline]
287    fn get_tls_secret_name(&self) -> String {
288        format!("{}-tls", self.name_any())
289    }
290
291    #[inline]
292    fn get_namespace(&self) -> String {
293        // safe unwrap: Kanidm is namespaced scoped
294        self.namespace().unwrap()
295    }
296
297    #[inline]
298    fn is_replication_enabled(&self) -> bool {
299        // safe unwrap: at least one replica group is required
300        self.spec.replica_groups.len() > 1
301            || self.spec.replica_groups.first().unwrap().replicas > 1
302            || !self.spec.external_replication_nodes.is_empty()
303    }
304
305    async fn exec<I, T>(
306        &self,
307        ctx: Arc<Context>,
308        pod_name: &str,
309        command: I,
310    ) -> Result<Option<String>>
311    where
312        I: IntoIterator<Item = T> + Debug,
313        T: Into<String>,
314    {
315        let namespace = &self.get_namespace();
316        trace!(
317            msg = "pod exec",
318            resource.name = &pod_name,
319            resource.namespace = &namespace,
320            ?command
321        );
322        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
323        let attached = pod
324            .exec(pod_name, command, &AttachParams::default().stderr(false))
325            .await
326            .map_err(|e| {
327                Error::KubeError(
328                    format!("failed to exec pod {namespace}/{pod_name}"),
329                    Box::new(e),
330                )
331            })?;
332        Ok(get_output(attached).await)
333    }
334
335    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
336    where
337        I: IntoIterator<Item = T> + Debug,
338        T: Into<String>,
339    {
340        // TODO: if replicas > 1 and replicas initialized, exec on pod available
341        // safe unwrap: at least one replica group is required
342        let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
343        let pod_name = format!("{sts_name}-0");
344        self.exec(ctx, &pod_name, command).await
345    }
346}
347
348#[cfg(test)]
349mod test {
350    use super::statefulset::StatefulSetExt;
351    use super::{Kanidm, reconcile_kanidm};
352
353    use crate::controller::State;
354    use crate::error::Result;
355    use crate::kanidm::controller::context::{Context, Stores};
356    use crate::kanidm::crd::KanidmStatus;
357    use k8s_openapi::api::core::v1::Service;
358    use k8s_openapi::api::networking::v1::Ingress;
359
360    use std::sync::Arc;
361
362    use http::{Request, Response};
363    use k8s_openapi::api::apps::v1::StatefulSet;
364    use kube::runtime::reflector::store::Writer;
365    use kube::{Client, Resource, ResourceExt, client::Body};
366    use serde_json::json;
367
368    impl Kanidm {
369        /// A normal test kanidm with a given status
370        pub fn test() -> Self {
371            let mut e = Kanidm::new(
372                "test",
373                serde_json::from_value(json!({
374                    "domain": "idm.example.com",
375                    "replicaGroups": [
376                        {
377                            "name": "default",
378                            "replicas": 1
379                        }
380                    ]
381                }))
382                .unwrap(),
383            );
384            e.meta_mut().namespace = Some("default".into());
385            e
386        }
387
388        pub fn with_ingress(mut self) -> Self {
389            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
390            self
391        }
392
393        /// Modify kanidm replicas
394        pub fn with_replicas(mut self, replicas: i32) -> Self {
395            // TODO: manage replica_groups
396            self.spec.replica_groups[0].replicas = replicas;
397            self
398        }
399
400        /// Modify kanidm to set a deletion timestamp
401        pub fn needs_delete(mut self) -> Self {
402            use chrono::prelude::{DateTime, TimeZone, Utc};
403            let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
404            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
405            self.meta_mut().deletion_timestamp = Some(Time(now));
406            self
407        }
408
409        /// Modify a kanidm to have an expected status
410        pub fn with_status(mut self, status: KanidmStatus) -> Self {
411            self.status = Some(status);
412            self
413        }
414    }
415
416    // We wrap tower_test::mock::Handle
417    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
418    pub struct ApiServerVerifier(ApiServerHandle);
419
420    /// Scenarios we test for in ApiServerVerifier
421    pub enum Scenario {
422        Create(Kanidm),
423        CreateWithTwoReplicas(Kanidm),
424        CreateWithIngress(Kanidm),
425        CreateWithIngressWithTwoReplicas(Kanidm),
426    }
427
428    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
429        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
430            .await
431            .expect("timeout on mock apiserver")
432            .expect("scenario succeeded")
433    }
434
435    impl ApiServerVerifier {
436        /// Tests only get to run specific scenarios that has matching handlers
437        ///
438        /// This setup makes it easy to handle multiple requests by chaining handlers together.
439        ///
440        /// NB: If the controller is making more calls than we are handling in the scenario,
441        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
442        ///
443        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
444        /// scenario runs to completion (i.e. all expected calls were responded to),
445        /// using the timeout to catch missing api calls to Kubernetes.
446        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
447            tokio::spawn(async move {
448                // moving self => one scenario per test
449                match scenario {
450                    Scenario::Create(kanidm) => {
451                        self.handle_kanidm_status_patch(kanidm.clone())
452                            .await
453                            .unwrap()
454                            .handle_statefulset_patch(kanidm.clone())
455                            .await
456                            .unwrap()
457                            .handle_service_patch(kanidm.clone())
458                            .await
459                    }
460                    Scenario::CreateWithTwoReplicas(kanidm) => {
461                        self.handle_kanidm_status_patch(kanidm.clone())
462                            .await
463                            .unwrap()
464                            .handle_statefulset_patch(kanidm.clone())
465                            .await
466                            .unwrap()
467                            .handle_service_patch(kanidm.clone())
468                            .await
469                    }
470                    Scenario::CreateWithIngress(kanidm) => {
471                        self.handle_kanidm_status_patch(kanidm.clone())
472                            .await
473                            .unwrap()
474                            .handle_statefulset_patch(kanidm.clone())
475                            .await
476                            .unwrap()
477                            .handle_service_patch(kanidm.clone())
478                            .await
479                            .unwrap()
480                            .handle_ingress_patch(kanidm.clone())
481                            .await
482                    }
483                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
484                        self.handle_kanidm_status_patch(kanidm.clone())
485                            .await
486                            .unwrap()
487                            .handle_statefulset_patch(kanidm.clone())
488                            .await
489                            .unwrap()
490                            .handle_service_patch(kanidm.clone())
491                            .await
492                            .unwrap()
493                            .handle_ingress_patch(kanidm.clone())
494                            .await
495                    }
496                }
497                .expect("scenario completed without errors");
498            })
499        }
500
501        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
502            let (request, send) = self.0.next_request().await.expect("service not called");
503            assert_eq!(request.method(), http::Method::PATCH);
504            assert_eq!(
505                request.uri().to_string(),
506                format!(
507                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
508                    kanidm.name_any()
509                )
510            );
511
512            let req_body = request.into_body().collect_bytes().await.unwrap();
513            let json: serde_json::Value =
514                serde_json::from_slice(&req_body).expect("patch object is json");
515            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
516                .expect("valid kanidm status");
517            let response = serde_json::to_vec(&status).unwrap();
518            // pass through kanidm "patch accepted"
519            send.send_response(Response::builder().body(Body::from(response)).unwrap());
520            Ok(self)
521        }
522
523        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
524            for rg in kanidm.spec.replica_groups.iter() {
525                let (request, send) = self.0.next_request().await.expect("service not called");
526                assert_eq!(request.method(), http::Method::PATCH);
527                assert_eq!(
528                    request.uri().to_string(),
529                    format!(
530                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
531                        kanidm.statefulset_name(&rg.name)
532                    )
533                );
534                let req_body = request.into_body().collect_bytes().await.unwrap();
535                let json: serde_json::Value =
536                    serde_json::from_slice(&req_body).expect("patch object is json");
537                let statefulset: StatefulSet =
538                    serde_json::from_value(json).expect("valid statefulset");
539                assert_eq!(
540                    statefulset.clone().spec.unwrap().replicas.unwrap(),
541                    rg.replicas
542                );
543                let response = serde_json::to_vec(&statefulset).unwrap();
544                // pass through kanidm "patch accepted"
545                send.send_response(Response::builder().body(Body::from(response)).unwrap());
546            }
547
548            Ok(self)
549        }
550
551        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
552            let (request, send) = self.0.next_request().await.expect("service not called");
553            assert_eq!(request.method(), http::Method::PATCH);
554            assert_eq!(
555                request.uri().to_string(),
556                format!(
557                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
558                    kanidm.name_any()
559                )
560            );
561
562            let req_body = request.into_body().collect_bytes().await.unwrap();
563            let json: serde_json::Value =
564                serde_json::from_slice(&req_body).expect("patch object is json");
565            let service: Service = serde_json::from_value(json).expect("valid service");
566            let response = serde_json::to_vec(&service).unwrap();
567            // pass through kanidm "patch accepted"
568            send.send_response(Response::builder().body(Body::from(response)).unwrap());
569            Ok(self)
570        }
571
572        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
573            let (request, send) = self.0.next_request().await.expect("service not called");
574            assert_eq!(request.method(), http::Method::PATCH);
575            assert_eq!(
576                request.uri().to_string(),
577                format!(
578                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
579                    kanidm.name_any()
580                )
581            );
582
583            let req_body = request.into_body().collect_bytes().await.unwrap();
584            let json: serde_json::Value =
585                serde_json::from_slice(&req_body).expect("patch object is json");
586            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
587            let response = serde_json::to_vec(&ingress).unwrap();
588            // pass through kanidm "patch accepted"
589            send.send_response(Response::builder().body(Body::from(response)).unwrap());
590            Ok(self)
591        }
592    }
593
594    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
595        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
596        let mock_client = Client::new(mock_service, "default");
597        let stores = Stores {
598            stateful_set_store: Writer::default().as_reader(),
599            service_store: Writer::default().as_reader(),
600            ingress_store: Writer::default().as_reader(),
601            secret_store: Writer::default().as_reader(),
602        };
603        let controller_id = "test";
604        let state = State::new(
605            Default::default(),
606            &[controller_id],
607            Writer::default().as_reader(),
608            Writer::default().as_reader(),
609        );
610        let ctx = Arc::new(Context::new(
611            state.to_context(mock_client, controller_id),
612            stores,
613        ));
614        (ctx, ApiServerVerifier(handle))
615    }
616
617    #[tokio::test]
618    async fn kanidm_create() {
619        let (testctx, fakeserver) = get_test_context();
620        let kanidm = Kanidm::test();
621        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
622        reconcile_kanidm(Arc::new(kanidm), testctx)
623            .await
624            .expect("reconciler");
625        timeout_after_1s(mocksrv).await;
626    }
627
628    #[tokio::test]
629    async fn kanidm_create_with_two_replicas() {
630        let (testctx, fakeserver) = get_test_context();
631        let kanidm = Kanidm::test().with_replicas(2);
632        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
633        reconcile_kanidm(Arc::new(kanidm), testctx)
634            .await
635            .expect("reconciler");
636        timeout_after_1s(mocksrv).await;
637    }
638
639    #[tokio::test]
640    async fn kanidm_create_with_ingress() {
641        let (testctx, fakeserver) = get_test_context();
642        let kanidm = Kanidm::test().with_ingress();
643        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
644        reconcile_kanidm(Arc::new(kanidm), testctx)
645            .await
646            .expect("reconciler");
647        timeout_after_1s(mocksrv).await;
648    }
649
650    #[tokio::test]
651    async fn kanidm_create_with_ingress_with_two_replicas() {
652        let (testctx, fakeserver) = get_test_context();
653        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
654        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
655        reconcile_kanidm(Arc::new(kanidm), testctx)
656            .await
657            .expect("reconciler");
658        timeout_after_1s(mocksrv).await;
659    }
660}