kaniop_operator/kanidm/reconcile/
mod.rs

1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::{REPLICA_GROUP_LABEL, StatefulSetExt};
13use self::status::StatusExt;
14
15use crate::controller::context::KubeOperations;
16use crate::controller::{DEFAULT_RECONCILE_INTERVAL, INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
17use crate::error::{Error, Result};
18use crate::kanidm::crd::{Kanidm, KanidmReplicaState, KanidmStatus};
19use crate::telemetry;
20
21use kaniop_k8s_util::client::get_output;
22
23use std::collections::BTreeMap;
24use std::fmt::Debug;
25use std::sync::{Arc, LazyLock};
26
27use futures::future::{TryJoinAll, join_all, try_join_all};
28use futures::try_join;
29use k8s_openapi::NamespaceResourceScope;
30use k8s_openapi::api::apps::v1::StatefulSet;
31use k8s_openapi::api::core::v1::Pod;
32use kube::Resource;
33use kube::ResourceExt;
34use kube::api::{Api, AttachParams};
35use serde::{Deserialize, Serialize};
36
37use kube::runtime::controller::Action;
38use status::{is_kanidm_available, is_kanidm_initialized};
39use tracing::{Span, debug, field, info, instrument, trace};
40
41pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
42const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
43
44static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
45    BTreeMap::from([
46        (NAME_LABEL.to_string(), "kanidm".to_string()),
47        (
48            MANAGED_BY_LABEL.to_string(),
49            format!("kaniop-{CONTROLLER_ID}"),
50        ),
51    ])
52});
53
54pub async fn reconcile_admins_secret(
55    kanidm: Arc<Kanidm>,
56    ctx: Arc<Context>,
57    status: &Result<KanidmStatus>,
58) -> Result<()> {
59    if let Ok(s) = status {
60        if is_kanidm_available(s.clone()) && !is_kanidm_initialized(s.clone()) {
61            let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
62            kanidm.patch(&ctx, admins_secret).await?;
63        }
64    }
65    Ok(())
66}
67
68pub async fn reconcile_replication_secrets(
69    kanidm: Arc<Kanidm>,
70    ctx: Arc<Context>,
71    status: &Result<KanidmStatus>,
72) -> Result<()> {
73    if let Ok(s) = status {
74        let secret_names = s
75            .replica_statuses
76            .iter()
77            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
78            .collect::<Vec<_>>();
79        let deprecated_secrets = ctx
80            .stores
81            .secret_store
82            .state()
83            .into_iter()
84            .filter(|secret| {
85                secret
86                    .metadata
87                    .labels
88                    .iter()
89                    .any(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
90                    && kanidm.admins_secret_name() != secret.name_any()
91                    && secret_names.iter().all(|sn| sn != &secret.name_any())
92            })
93            .collect::<Vec<_>>();
94        let secret_delete_future = deprecated_secrets
95            .iter()
96            .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
97            .collect::<TryJoinAll<_>>();
98        try_join!(secret_delete_future)?;
99
100        if kanidm.is_replication_enabled() {
101            let generate_secret_futures = s
102                .replica_statuses
103                .iter()
104                .filter(|rs| rs.state == KanidmReplicaState::Pending)
105                .map(|rs| kanidm.generate_replica_secret(ctx.clone(), &rs.pod_name))
106                .collect::<Vec<_>>();
107            let secrets = try_join_all(generate_secret_futures).await?;
108            let secret_futures = secrets
109                .into_iter()
110                .map(|secret| kanidm.patch(&ctx, secret))
111                .collect::<Vec<_>>();
112            try_join_all(secret_futures).await?;
113            // TODO: rolling restart all of them one by one if you have write-replicas replica
114            // group with one node
115            let sts_api = Api::<StatefulSet>::namespaced(
116                ctx.kaniop_ctx.client.clone(),
117                &kanidm.get_namespace(),
118            );
119            let sts_restart_futures = s
120                .replica_statuses
121                .iter()
122                .filter(|rs| rs.state == KanidmReplicaState::Pending)
123                .map(|rs| sts_api.restart(&rs.statefulset_name));
124            let _ignore_errors = join_all(sts_restart_futures).await;
125        }
126    }
127    Ok(())
128}
129
130#[instrument(skip(ctx, kanidm))]
131pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
132    let trace_id = telemetry::get_trace_id();
133    Span::current().record("trace_id", field::display(&trace_id));
134    let _timer = ctx
135        .kaniop_ctx
136        .metrics
137        .reconcile_count_and_measure(&trace_id);
138    info!(msg = "reconciling Kanidm");
139
140    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
141        debug!(msg = "failed to reconcile status", %e);
142        ctx.kaniop_ctx.metrics.status_update_errors_inc();
143        e
144    });
145
146    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
147    let replication_secret_future =
148        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
149
150    let sts_to_delete = ctx
151        .stores
152        .stateful_set_store
153        .state()
154        .into_iter()
155        .filter(|sts| {
156            sts.metadata.labels.iter().any(|l| {
157                l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
158                    && match l.get(REPLICA_GROUP_LABEL) {
159                        Some(sts_rg_name) => kanidm
160                            .spec
161                            .replica_groups
162                            .iter()
163                            .all(|rg| &rg.name != sts_rg_name),
164                        None => false,
165                    }
166            })
167        })
168        .collect::<Vec<_>>();
169    let sts_delete_future = sts_to_delete
170        .iter()
171        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
172        .collect::<TryJoinAll<_>>();
173
174    let sts_futures = kanidm
175        .spec
176        .replica_groups
177        .iter()
178        .map(|rg| kanidm.patch(&ctx, kanidm.create_statefulset(rg)))
179        .collect::<TryJoinAll<_>>();
180    let service_future = kanidm.patch(&ctx, kanidm.create_service());
181    let ingress_future = kanidm
182        .create_ingress()
183        .into_iter()
184        .map(|ingress| kanidm.patch(&ctx, ingress))
185        .collect::<TryJoinAll<_>>();
186
187    try_join!(
188        sts_delete_future,
189        admin_secret_future,
190        replication_secret_future,
191        sts_futures,
192        service_future,
193        ingress_future
194    )?;
195    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
196}
197
198impl Kanidm {
199    // Convenience methods that handle context and operator name
200    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
201    where
202        K: Resource<Scope = NamespaceResourceScope>
203            + Serialize
204            + Clone
205            + std::fmt::Debug
206            + for<'de> Deserialize<'de>,
207        <K as kube::Resource>::DynamicType: Default,
208        <K as Resource>::Scope: std::marker::Sized,
209    {
210        self.kube_delete(
211            ctx.kaniop_ctx.client.clone(),
212            &ctx.kaniop_ctx.metrics,
213            resource,
214        )
215        .await
216    }
217
218    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
219    where
220        K: Resource<Scope = NamespaceResourceScope>
221            + Serialize
222            + Clone
223            + std::fmt::Debug
224            + for<'de> Deserialize<'de>,
225        <K as kube::Resource>::DynamicType: Default,
226        <K as Resource>::Scope: std::marker::Sized,
227    {
228        self.kube_patch(
229            ctx.kaniop_ctx.client.clone(),
230            &ctx.kaniop_ctx.metrics,
231            resource,
232            KANIDM_OPERATOR_NAME,
233        )
234        .await
235    }
236
237    #[inline]
238    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
239        LABELS
240            .clone()
241            .into_iter()
242            .chain([
243                (INSTANCE_LABEL.to_string(), self.name_any()),
244                (CLUSTER_LABEL.to_string(), self.name_any()),
245            ])
246            .collect()
247    }
248
249    #[inline]
250    fn get_tls_secret_name(&self) -> String {
251        format!("{}-tls", self.name_any())
252    }
253
254    #[inline]
255    fn get_namespace(&self) -> String {
256        // safe unwrap: Kanidm is namespaced scoped
257        self.namespace().unwrap()
258    }
259
260    #[inline]
261    fn is_replication_enabled(&self) -> bool {
262        // safe unwrap: at least one replica group is required
263        self.spec.replica_groups.len() > 1
264            || self.spec.replica_groups.first().unwrap().replicas > 1
265            || !self.spec.external_replication_nodes.is_empty()
266    }
267
268    async fn exec<I, T>(
269        &self,
270        ctx: Arc<Context>,
271        pod_name: &str,
272        command: I,
273    ) -> Result<Option<String>>
274    where
275        I: IntoIterator<Item = T> + Debug,
276        T: Into<String>,
277    {
278        let namespace = &self.get_namespace();
279        trace!(
280            msg = "pod exec",
281            resource.name = &pod_name,
282            resource.namespace = &namespace,
283            ?command
284        );
285        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
286        let attached = pod
287            .exec(pod_name, command, &AttachParams::default().stderr(false))
288            .await
289            .map_err(|e| {
290                Error::KubeError(
291                    format!("failed to exec pod {namespace}/{pod_name}"),
292                    Box::new(e),
293                )
294            })?;
295        Ok(get_output(attached).await)
296    }
297
298    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<Option<String>>
299    where
300        I: IntoIterator<Item = T> + Debug,
301        T: Into<String>,
302    {
303        // TODO: if replicas > 1 and replicas initialized, exec on pod available
304        // safe unwrap: at least one replica group is required
305        let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
306        let pod_name = format!("{sts_name}-0");
307        self.exec(ctx, &pod_name, command).await
308    }
309}
310
311#[cfg(test)]
312mod test {
313    use super::statefulset::StatefulSetExt;
314    use super::{Kanidm, reconcile_kanidm};
315
316    use crate::controller::State;
317    use crate::error::Result;
318    use crate::kanidm::controller::context::{Context, Stores};
319    use crate::kanidm::crd::KanidmStatus;
320    use k8s_openapi::api::core::v1::Service;
321    use k8s_openapi::api::networking::v1::Ingress;
322
323    use std::sync::Arc;
324
325    use http::{Request, Response};
326    use k8s_openapi::api::apps::v1::StatefulSet;
327    use kube::runtime::reflector::store::Writer;
328    use kube::{Client, Resource, ResourceExt, client::Body};
329    use serde_json::json;
330
331    impl Kanidm {
332        /// A normal test kanidm with a given status
333        pub fn test() -> Self {
334            let mut e = Kanidm::new(
335                "test",
336                serde_json::from_value(json!({
337                    "domain": "idm.example.com",
338                    "replicaGroups": [
339                        {
340                            "name": "default",
341                            "replicas": 1
342                        }
343                    ]
344                }))
345                .unwrap(),
346            );
347            e.meta_mut().namespace = Some("default".into());
348            e
349        }
350
351        pub fn with_ingress(mut self) -> Self {
352            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
353            self
354        }
355
356        /// Modify kanidm replicas
357        pub fn with_replicas(mut self, replicas: i32) -> Self {
358            // TODO: manage replica_groups
359            self.spec.replica_groups[0].replicas = replicas;
360            self
361        }
362
363        /// Modify kanidm to set a deletion timestamp
364        pub fn needs_delete(mut self) -> Self {
365            use chrono::prelude::{DateTime, TimeZone, Utc};
366            let now: DateTime<Utc> = Utc.with_ymd_and_hms(2017, 4, 2, 12, 50, 32).unwrap();
367            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
368            self.meta_mut().deletion_timestamp = Some(Time(now));
369            self
370        }
371
372        /// Modify a kanidm to have an expected status
373        pub fn with_status(mut self, status: KanidmStatus) -> Self {
374            self.status = Some(status);
375            self
376        }
377    }
378
379    // We wrap tower_test::mock::Handle
380    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
381    pub struct ApiServerVerifier(ApiServerHandle);
382
383    /// Scenarios we test for in ApiServerVerifier
384    pub enum Scenario {
385        Create(Kanidm),
386        CreateWithTwoReplicas(Kanidm),
387        CreateWithIngress(Kanidm),
388        CreateWithIngressWithTwoReplicas(Kanidm),
389    }
390
391    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
392        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
393            .await
394            .expect("timeout on mock apiserver")
395            .expect("scenario succeeded")
396    }
397
398    impl ApiServerVerifier {
399        /// Tests only get to run specific scenarios that has matching handlers
400        ///
401        /// This setup makes it easy to handle multiple requests by chaining handlers together.
402        ///
403        /// NB: If the controller is making more calls than we are handling in the scenario,
404        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
405        ///
406        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
407        /// scenario runs to completion (i.e. all expected calls were responded to),
408        /// using the timeout to catch missing api calls to Kubernetes.
409        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
410            tokio::spawn(async move {
411                // moving self => one scenario per test
412                match scenario {
413                    Scenario::Create(kanidm) => {
414                        self.handle_kanidm_status_patch(kanidm.clone())
415                            .await
416                            .unwrap()
417                            .handle_statefulset_patch(kanidm.clone())
418                            .await
419                            .unwrap()
420                            .handle_service_patch(kanidm.clone())
421                            .await
422                    }
423                    Scenario::CreateWithTwoReplicas(kanidm) => {
424                        self.handle_kanidm_status_patch(kanidm.clone())
425                            .await
426                            .unwrap()
427                            .handle_statefulset_patch(kanidm.clone())
428                            .await
429                            .unwrap()
430                            .handle_service_patch(kanidm.clone())
431                            .await
432                    }
433                    Scenario::CreateWithIngress(kanidm) => {
434                        self.handle_kanidm_status_patch(kanidm.clone())
435                            .await
436                            .unwrap()
437                            .handle_statefulset_patch(kanidm.clone())
438                            .await
439                            .unwrap()
440                            .handle_service_patch(kanidm.clone())
441                            .await
442                            .unwrap()
443                            .handle_ingress_patch(kanidm.clone())
444                            .await
445                    }
446                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
447                        self.handle_kanidm_status_patch(kanidm.clone())
448                            .await
449                            .unwrap()
450                            .handle_statefulset_patch(kanidm.clone())
451                            .await
452                            .unwrap()
453                            .handle_service_patch(kanidm.clone())
454                            .await
455                            .unwrap()
456                            .handle_ingress_patch(kanidm.clone())
457                            .await
458                    }
459                }
460                .expect("scenario completed without errors");
461            })
462        }
463
464        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
465            let (request, send) = self.0.next_request().await.expect("service not called");
466            assert_eq!(request.method(), http::Method::PATCH);
467            assert_eq!(
468                request.uri().to_string(),
469                format!(
470                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
471                    kanidm.name_any()
472                )
473            );
474
475            let req_body = request.into_body().collect_bytes().await.unwrap();
476            let json: serde_json::Value =
477                serde_json::from_slice(&req_body).expect("patch object is json");
478            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
479                .expect("valid kanidm status");
480            let response = serde_json::to_vec(&status).unwrap();
481            // pass through kanidm "patch accepted"
482            send.send_response(Response::builder().body(Body::from(response)).unwrap());
483            Ok(self)
484        }
485
486        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
487            for rg in kanidm.spec.replica_groups.iter() {
488                let (request, send) = self.0.next_request().await.expect("service not called");
489                assert_eq!(request.method(), http::Method::PATCH);
490                assert_eq!(
491                    request.uri().to_string(),
492                    format!(
493                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
494                        kanidm.statefulset_name(&rg.name)
495                    )
496                );
497                let req_body = request.into_body().collect_bytes().await.unwrap();
498                let json: serde_json::Value =
499                    serde_json::from_slice(&req_body).expect("patch object is json");
500                let statefulset: StatefulSet =
501                    serde_json::from_value(json).expect("valid statefulset");
502                assert_eq!(
503                    statefulset.clone().spec.unwrap().replicas.unwrap(),
504                    rg.replicas
505                );
506                let response = serde_json::to_vec(&statefulset).unwrap();
507                // pass through kanidm "patch accepted"
508                send.send_response(Response::builder().body(Body::from(response)).unwrap());
509            }
510
511            Ok(self)
512        }
513
514        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
515            let (request, send) = self.0.next_request().await.expect("service not called");
516            assert_eq!(request.method(), http::Method::PATCH);
517            assert_eq!(
518                request.uri().to_string(),
519                format!(
520                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
521                    kanidm.name_any()
522                )
523            );
524
525            let req_body = request.into_body().collect_bytes().await.unwrap();
526            let json: serde_json::Value =
527                serde_json::from_slice(&req_body).expect("patch object is json");
528            let service: Service = serde_json::from_value(json).expect("valid service");
529            let response = serde_json::to_vec(&service).unwrap();
530            // pass through kanidm "patch accepted"
531            send.send_response(Response::builder().body(Body::from(response)).unwrap());
532            Ok(self)
533        }
534
535        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
536            let (request, send) = self.0.next_request().await.expect("service not called");
537            assert_eq!(request.method(), http::Method::PATCH);
538            assert_eq!(
539                request.uri().to_string(),
540                format!(
541                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
542                    kanidm.name_any()
543                )
544            );
545
546            let req_body = request.into_body().collect_bytes().await.unwrap();
547            let json: serde_json::Value =
548                serde_json::from_slice(&req_body).expect("patch object is json");
549            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
550            let response = serde_json::to_vec(&ingress).unwrap();
551            // pass through kanidm "patch accepted"
552            send.send_response(Response::builder().body(Body::from(response)).unwrap());
553            Ok(self)
554        }
555    }
556
557    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
558        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
559        let mock_client = Client::new(mock_service, "default");
560        let stores = Stores {
561            stateful_set_store: Writer::default().as_reader(),
562            service_store: Writer::default().as_reader(),
563            ingress_store: Writer::default().as_reader(),
564            secret_store: Writer::default().as_reader(),
565        };
566        let controller_id = "test";
567        let state = State::new(
568            Default::default(),
569            &[controller_id],
570            Writer::default().as_reader(),
571            Writer::default().as_reader(),
572        );
573        let ctx = Arc::new(Context::new(
574            state.to_context(mock_client, controller_id),
575            stores,
576        ));
577        (ctx, ApiServerVerifier(handle))
578    }
579
580    #[tokio::test]
581    async fn kanidm_create() {
582        let (testctx, fakeserver) = get_test_context();
583        let kanidm = Kanidm::test();
584        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
585        reconcile_kanidm(Arc::new(kanidm), testctx)
586            .await
587            .expect("reconciler");
588        timeout_after_1s(mocksrv).await;
589    }
590
591    #[tokio::test]
592    async fn kanidm_create_with_two_replicas() {
593        let (testctx, fakeserver) = get_test_context();
594        let kanidm = Kanidm::test().with_replicas(2);
595        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
596        reconcile_kanidm(Arc::new(kanidm), testctx)
597            .await
598            .expect("reconciler");
599        timeout_after_1s(mocksrv).await;
600    }
601
602    #[tokio::test]
603    async fn kanidm_create_with_ingress() {
604        let (testctx, fakeserver) = get_test_context();
605        let kanidm = Kanidm::test().with_ingress();
606        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
607        reconcile_kanidm(Arc::new(kanidm), testctx)
608            .await
609            .expect("reconciler");
610        timeout_after_1s(mocksrv).await;
611    }
612
613    #[tokio::test]
614    async fn kanidm_create_with_ingress_with_two_replicas() {
615        let (testctx, fakeserver) = get_test_context();
616        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
617        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
618        reconcile_kanidm(Arc::new(kanidm), testctx)
619            .await
620            .expect("reconciler");
621        timeout_after_1s(mocksrv).await;
622    }
623}