Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod ingress;
2pub mod secret;
3mod service;
4pub mod statefulset;
5mod status;
6
7use super::controller::{CONTROLLER_ID, context::Context};
8
9use self::ingress::IngressExt;
10use self::secret::SecretExt;
11use self::service::ServiceExt;
12use self::statefulset::StatefulSetExt;
13use self::status::StatusExt;
14use self::status::{is_kanidm_available, is_kanidm_initialized};
15
16use crate::controller::context::KubeOperations;
17use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
18use crate::kanidm::crd::{
19    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
20};
21use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
22use crate::telemetry;
23
24use kaniop_k8s_util::client::get_output;
25use kaniop_k8s_util::error::{Error, Result};
26use kaniop_k8s_util::parse::parse_semver;
27use kaniop_k8s_util::resources::get_image_tag;
28
29use std::collections::BTreeMap;
30use std::fmt::Debug;
31use std::sync::{Arc, LazyLock};
32
33use futures::future::{TryJoinAll, join_all, try_join_all};
34use futures::stream::{self, StreamExt};
35use futures::try_join;
36use k8s_openapi::NamespaceResourceScope;
37use k8s_openapi::api::apps::v1::StatefulSet;
38use k8s_openapi::api::core::v1::Pod;
39use kube::Resource;
40use kube::ResourceExt;
41use kube::api::{Api, AttachParams};
42use kube::runtime::controller::Action;
43use kube::runtime::events::{Event, EventType};
44use kube::runtime::finalizer::{Event as Finalizer, finalizer};
45use serde::{Deserialize, Serialize};
46use tokio::time::Duration;
47use tracing::{Span, debug, field, info, instrument, trace, warn};
48
49pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
50const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
51pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
52
53const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
54
55static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
56    BTreeMap::from([
57        (NAME_LABEL.to_string(), "kanidm".to_string()),
58        (
59            MANAGED_BY_LABEL.to_string(),
60            format!("kaniop-{CONTROLLER_ID}"),
61        ),
62    ])
63});
64
65pub async fn reconcile_admins_secret(
66    kanidm: Arc<Kanidm>,
67    ctx: Arc<Context>,
68    status: &KanidmStatus,
69) -> Result<()> {
70    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
71        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
72        kanidm.patch(&ctx, admins_secret).await?;
73    }
74    Ok(())
75}
76
77pub async fn reconcile_replication_secrets(
78    kanidm: Arc<Kanidm>,
79    ctx: Arc<Context>,
80    status: &KanidmStatus,
81) -> Result<()> {
82    let expected_secret_names = if kanidm.is_replication_enabled() {
83        status
84            .replica_statuses
85            .iter()
86            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
87            .collect::<Vec<_>>()
88    } else {
89        Vec::new()
90    };
91    let deprecated_secrets = ctx
92        .stores
93        .secret_store
94        .state()
95        .into_iter()
96        .filter(|secret| {
97            secret.namespace() == kanidm.namespace()
98                && kanidm.admins_secret_name() != secret.name_any()
99                && !expected_secret_names.contains(&secret.name_any())
100                && secret
101                    .metadata
102                    .labels
103                    .as_ref()
104                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
105        })
106        .collect::<Vec<_>>();
107    let secret_delete_future = deprecated_secrets
108        .iter()
109        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
110        .collect::<TryJoinAll<_>>();
111    try_join!(secret_delete_future)?;
112
113    if kanidm.is_replication_enabled() {
114        // sequential renewal: concurrent certificate updates are not allowed; abort on first error
115        // https://github.com/kanidm/kanidm/issues/3917
116        for rs in status.replica_statuses.iter().filter(|rs| {
117            rs.state == KanidmReplicaState::CertificateExpiring
118                || rs.state == KanidmReplicaState::CertificateHostInvalid
119        }) {
120            let secret = kanidm
121                .update_replica_secret(ctx.clone(), &rs.pod_name)
122                .await?;
123            kanidm.patch(&ctx, secret.clone()).await?;
124        }
125
126        stream::iter(
127            status
128                .replica_statuses
129                .iter()
130                .filter(|rs| rs.state == KanidmReplicaState::Pending),
131        )
132        .then(|rs| async {
133            let secret = kanidm
134                .generate_replica_secret(ctx.clone(), &rs.pod_name)
135                .await?;
136            kanidm.patch(&ctx, secret).await
137        })
138        .collect::<Vec<_>>()
139        .await
140        .into_iter()
141        .collect::<Result<Vec<_>>>()?;
142
143        // If there are any replicas in pending or cert expiring state, trigger a restart
144        // of all statefulsets to pickup new certs
145        let has_pending_or_expiring = status
146            .replica_statuses
147            .iter()
148            .any(|rs| rs.state != KanidmReplicaState::Ready);
149
150        if has_pending_or_expiring {
151            let sts_api = Api::<StatefulSet>::namespaced(
152                ctx.kaniop_ctx.client.clone(),
153                &kanidm.get_namespace(),
154            );
155
156            // Check if any replica group has only 1 replica
157            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
158
159            let restart_futures = status
160                .replica_statuses
161                .iter()
162                .map(|rs| sts_api.restart(&rs.statefulset_name));
163
164            if has_single_replica {
165                // Restart sequentially to avoid downtime
166                let _ignore_errors = stream::iter(restart_futures)
167                    .then(|f| f)
168                    .collect::<Vec<_>>()
169                    .await;
170            } else {
171                // Restart concurrently for replica groups with multiple replicas
172                let _ignore_errors = join_all(restart_futures).await;
173            }
174        }
175    }
176
177    Ok(())
178}
179
180#[instrument(skip(ctx, kanidm))]
181pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
182    let trace_id = telemetry::get_trace_id();
183    Span::current().record("trace_id", field::display(&trace_id));
184    let _timer = ctx
185        .kaniop_ctx
186        .metrics
187        .reconcile_count_and_measure(&trace_id);
188    info!(msg = "reconciling Kanidm");
189
190    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
191        debug!(msg = "failed to reconcile status", %e);
192        ctx.kaniop_ctx.metrics.status_update_errors_inc();
193        e
194    })?;
195    let kanidm_api: Api<Kanidm> =
196        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
197    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
198        match event {
199            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
200            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
201        }
202    })
203    .await
204    .map_err(|e| {
205        Error::FinalizerError(
206            "failed on kanidm account finalizer".to_string(),
207            Box::new(e),
208        )
209    })
210}
211
212async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
213    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
214    let replication_secret_futures =
215        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
216
217    let sts_to_delete = {
218        let expected_sts_names = kanidm
219            .spec
220            .replica_groups
221            .iter()
222            .map(|rg| kanidm.statefulset_name(&rg.name))
223            .collect::<Vec<_>>();
224        ctx.stores
225            .stateful_set_store
226            .state()
227            .into_iter()
228            .filter(|sts| {
229                sts.namespace() == kanidm.namespace()
230                    && sts
231                        .metadata
232                        .labels
233                        .as_ref()
234                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
235                    && !expected_sts_names.contains(&sts.name_any())
236            })
237            .collect::<Vec<_>>()
238    };
239    let sts_delete_futures = sts_to_delete
240        .iter()
241        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
242        .collect::<TryJoinAll<_>>();
243
244    let sts_futures = match kanidm.is_updatable(&status) {
245        true => kanidm
246            .spec
247            .replica_groups
248            .iter()
249            .map(|rg| {
250                let sts = kanidm.create_statefulset(rg, &ctx)?;
251                Ok(kanidm.patch(&ctx, sts))
252            })
253            .collect::<Result<TryJoinAll<_>, _>>()?,
254        false => {
255            let note = match status.version.as_ref() {
256                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
257                    format!(
258                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
259                        v.image_tag,
260                        crate::version::KANIDM_CLIENT_VERSION
261                    )
262                }
263                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
264            };
265            let _ignore_error = ctx
266                .kaniop_ctx
267                .recorder
268                .publish(
269                    &Event {
270                        type_: EventType::Warning,
271                        reason: "UpgradeBlocked".to_string(),
272                        note: Some(note),
273                        action: "ReconcileStatefulSet".to_string(),
274                        secondary: None,
275                    },
276                    &kanidm.object_ref(&()),
277                )
278                .await
279                .map_err(|e| {
280                    warn!(msg = "failed to publish KanidmError event", %e);
281                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
282                });
283            try_join_all([].into_iter())
284        }
285    };
286    let service_future = kanidm.patch(&ctx, kanidm.create_service());
287
288    let deprecated_rg_svcs = {
289        let expected_rg_svcs_names = kanidm
290            .spec
291            .replica_groups
292            .iter()
293            .filter(|rg| rg.services.is_some())
294            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
295            .collect::<Vec<_>>();
296        ctx.stores
297            .service_store
298            .state()
299            .into_iter()
300            .filter(|svc| {
301                let labels = svc.metadata.labels.as_ref();
302                svc.namespace() == kanidm.namespace()
303                    && !expected_rg_svcs_names.contains(&svc.name_any())
304                    && labels.is_some_and(|l| {
305                        l.contains_key(REPLICA_LABEL)
306                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
307                    })
308            })
309            .collect::<Vec<_>>()
310    };
311    let rg_svcs_delete_futures = deprecated_rg_svcs
312        .iter()
313        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
314        .collect::<TryJoinAll<_>>();
315
316    let rg_services_futures = kanidm
317        .spec
318        .replica_groups
319        .iter()
320        .filter(|rg| rg.services.is_some())
321        .flat_map(|rg| {
322            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
323        })
324        .collect::<TryJoinAll<_>>();
325
326    let deprecated_ingresses = {
327        let names = [
328            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
329            kanidm.generate_region_ingress_name(),
330        ]
331        .into_iter()
332        .flatten()
333        .collect::<Vec<_>>();
334        ctx.stores
335            .ingress_store
336            .state()
337            .into_iter()
338            .filter(|ing| {
339                ing.namespace() == kanidm.namespace()
340                    && !names.contains(&ing.name_any())
341                    && ing.metadata.labels == Some(kanidm.generate_labels())
342            })
343            .collect::<Vec<_>>()
344    };
345    let ingress_delete_futures = deprecated_ingresses
346        .iter()
347        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
348        .collect::<TryJoinAll<_>>();
349
350    let ingress_futures = kanidm
351        .create_ingress()
352        .into_iter()
353        .chain(kanidm.create_region_ingress())
354        .map(|ingress| kanidm.patch(&ctx, ingress))
355        .collect::<TryJoinAll<_>>();
356
357    try_join!(
358        sts_delete_futures,
359        admin_secret_future,
360        replication_secret_futures,
361        sts_futures,
362        service_future,
363        rg_svcs_delete_futures,
364        rg_services_futures,
365        ingress_delete_futures,
366        ingress_futures
367    )?;
368    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
369}
370
371async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
372    debug!(msg = "cleanup");
373    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
374    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
375}
376
377impl Kanidm {
378    // Convenience methods that handle context and operator name
379    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
380    where
381        K: Resource<Scope = NamespaceResourceScope>
382            + Serialize
383            + Clone
384            + std::fmt::Debug
385            + for<'de> Deserialize<'de>,
386        <K as kube::Resource>::DynamicType: Default,
387        <K as Resource>::Scope: std::marker::Sized,
388    {
389        self.kube_delete(
390            ctx.kaniop_ctx.client.clone(),
391            &ctx.kaniop_ctx.metrics,
392            resource,
393        )
394        .await
395    }
396
397    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
398    where
399        K: Resource<Scope = NamespaceResourceScope>
400            + Serialize
401            + Clone
402            + std::fmt::Debug
403            + for<'de> Deserialize<'de>,
404        <K as kube::Resource>::DynamicType: Default,
405        <K as Resource>::Scope: std::marker::Sized,
406    {
407        self.kube_patch(
408            ctx.kaniop_ctx.client.clone(),
409            &ctx.kaniop_ctx.metrics,
410            resource,
411            KANIDM_OPERATOR_NAME,
412        )
413        .await
414    }
415
416    #[inline]
417    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
418        LABELS
419            .clone()
420            .into_iter()
421            .chain([
422                (INSTANCE_LABEL.to_string(), self.name_any()),
423                (CLUSTER_LABEL.to_string(), self.name_any()),
424            ])
425            .collect()
426    }
427
428    #[inline]
429    fn generate_labels(&self) -> BTreeMap<String, String> {
430        self.generate_resource_labels()
431            .clone()
432            .into_iter()
433            .chain(self.labels().clone())
434            .collect()
435    }
436
437    #[inline]
438    fn get_tls_secret_name(&self) -> String {
439        format!("{}-tls", self.name_any())
440    }
441
442    #[inline]
443    fn get_namespace(&self) -> String {
444        // safe unwrap: Kanidm is namespaced scoped
445        self.namespace().unwrap()
446    }
447
448    #[inline]
449    fn is_replication_enabled(&self) -> bool {
450        // safe unwrap: at least one replica group is required
451        self.spec.replica_groups.len() > 1
452            || self.spec.replica_groups.first().unwrap().replicas > 1
453            || !self.spec.external_replication_nodes.is_empty()
454    }
455
456    #[inline]
457    fn is_updatable(&self, status: &KanidmStatus) -> bool {
458        status
459            .version
460            .as_ref()
461            .map(|v| {
462                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
463                    return false;
464                }
465                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
466                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
467                match v.upgrade_check_result {
468                    KanidmUpgradeCheckResult::Passed => {
469                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
470                            minor <= v_minor + 1
471                        } else {
472                            true
473                        }
474                    }
475                    KanidmUpgradeCheckResult::Failed => {
476                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
477                            versions
478                        {
479                            major == v_major && minor == v_minor && patch >= v_patch
480                        } else {
481                            v.image_tag == current_tag
482                        }
483                    }
484                }
485            })
486            .unwrap_or(true)
487    }
488
489    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
490    where
491        I: IntoIterator<Item = T> + Debug,
492        T: Into<String>,
493    {
494        let namespace = &self.get_namespace();
495        trace!(
496            msg = "pod exec",
497            resource.name = &pod_name,
498            resource.namespace = &namespace,
499            ?command
500        );
501        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
502        let attached = pod
503            .exec(pod_name, command, &AttachParams::default())
504            .await
505            .map_err(|e| {
506                Error::KubeError(
507                    format!("failed to exec pod {namespace}/{pod_name}"),
508                    Box::new(e),
509                )
510            })?;
511        get_output(attached).await
512    }
513
514    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
515    where
516        I: IntoIterator<Item = T> + Debug,
517        T: Into<String>,
518    {
519        // TODO: if replicas > 1 and replicas initialized, exec on pod available
520        // safe unwrap: at least one replica group is required
521        let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
522        let pod_name = format!("{sts_name}-0");
523        self.exec(ctx, &pod_name, command).await
524    }
525}
526
527#[cfg(test)]
528mod test {
529    use super::statefulset::StatefulSetExt;
530    use super::{Kanidm, reconcile_kanidm};
531
532    use crate::controller::State;
533    use crate::kanidm::controller::context::{Context, Stores};
534    use crate::kanidm::crd::KanidmStatus;
535
536    use kaniop_k8s_util::error::Result;
537
538    use std::sync::Arc;
539
540    use http::{Request, Response};
541    use k8s_openapi::api::apps::v1::StatefulSet;
542    use k8s_openapi::api::core::v1::Service;
543    use k8s_openapi::api::networking::v1::Ingress;
544    use kube::runtime::reflector::store::Writer;
545    use kube::{Client, Resource, ResourceExt, client::Body};
546    use opentelemetry::metrics::MeterProvider;
547    use serde_json::json;
548
549    impl Kanidm {
550        /// A normal test kanidm with a given status
551        pub fn test() -> Self {
552            let mut e = Kanidm::new(
553                "test",
554                serde_json::from_value(json!({
555                    "domain": "idm.example.com",
556                    "replicaGroups": [
557                        {
558                            "name": "default",
559                            "replicas": 1
560                        }
561                    ]
562                }))
563                .unwrap(),
564            );
565            e.meta_mut().namespace = Some("default".into());
566            // Pre-populate finalizer so reconcile callback runs immediately
567            // (skips the "add finalizer + wait for next reconcile" step)
568            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
569            e
570        }
571
572        pub fn with_ingress(mut self) -> Self {
573            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
574            self
575        }
576
577        /// Modify kanidm replicas
578        pub fn with_replicas(mut self, replicas: i32) -> Self {
579            self.spec.replica_groups[0].replicas = replicas;
580            self
581        }
582
583        /// Modify kanidm to set a deletion timestamp
584        pub fn needs_delete(mut self) -> Self {
585            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
586            use k8s_openapi::jiff::Timestamp;
587            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
588            self.meta_mut().deletion_timestamp = Some(Time(now));
589            self
590        }
591
592        /// Modify a kanidm to have an expected status
593        pub fn with_status(mut self, status: KanidmStatus) -> Self {
594            self.status = Some(status);
595            self
596        }
597    }
598
599    // We wrap tower_test::mock::Handle
600    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
601    pub struct ApiServerVerifier(ApiServerHandle);
602
603    /// Scenarios we test for in ApiServerVerifier
604    pub enum Scenario {
605        Create(Kanidm),
606        CreateWithTwoReplicas(Kanidm),
607        CreateWithIngress(Kanidm),
608        CreateWithIngressWithTwoReplicas(Kanidm),
609    }
610
611    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
612        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
613            .await
614            .expect("timeout on mock apiserver")
615            .expect("scenario succeeded")
616    }
617
618    impl ApiServerVerifier {
619        /// Tests only get to run specific scenarios that has matching handlers
620        ///
621        /// This setup makes it easy to handle multiple requests by chaining handlers together.
622        ///
623        /// NB: If the controller is making more calls than we are handling in the scenario,
624        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
625        ///
626        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
627        /// scenario runs to completion (i.e. all expected calls were responded to),
628        /// using the timeout to catch missing api calls to Kubernetes.
629        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
630            tokio::spawn(async move {
631                // moving self => one scenario per test
632                match scenario {
633                    Scenario::Create(kanidm) => {
634                        self.handle_kanidm_status_patch(kanidm.clone())
635                            .await
636                            .unwrap()
637                            .handle_statefulset_patch(kanidm.clone())
638                            .await
639                            .unwrap()
640                            .handle_service_patch(kanidm.clone())
641                            .await
642                    }
643                    Scenario::CreateWithTwoReplicas(kanidm) => {
644                        self.handle_kanidm_status_patch(kanidm.clone())
645                            .await
646                            .unwrap()
647                            .handle_statefulset_patch(kanidm.clone())
648                            .await
649                            .unwrap()
650                            .handle_service_patch(kanidm.clone())
651                            .await
652                    }
653                    Scenario::CreateWithIngress(kanidm) => {
654                        self.handle_kanidm_status_patch(kanidm.clone())
655                            .await
656                            .unwrap()
657                            .handle_statefulset_patch(kanidm.clone())
658                            .await
659                            .unwrap()
660                            .handle_service_patch(kanidm.clone())
661                            .await
662                            .unwrap()
663                            .handle_ingress_patch(kanidm.clone())
664                            .await
665                    }
666                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
667                        self.handle_kanidm_status_patch(kanidm.clone())
668                            .await
669                            .unwrap()
670                            .handle_statefulset_patch(kanidm.clone())
671                            .await
672                            .unwrap()
673                            .handle_service_patch(kanidm.clone())
674                            .await
675                            .unwrap()
676                            .handle_ingress_patch(kanidm.clone())
677                            .await
678                    }
679                }
680                .expect("scenario completed without errors");
681            })
682        }
683
684        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
685            let (request, send) = self.0.next_request().await.expect("service not called");
686            assert_eq!(request.method(), http::Method::PATCH);
687            assert_eq!(
688                request.uri().to_string(),
689                format!(
690                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?&force=true&fieldManager=kanidms.kaniop.rs",
691                    kanidm.name_any()
692                )
693            );
694
695            let req_body = request.into_body().collect_bytes().await.unwrap();
696            let json: serde_json::Value =
697                serde_json::from_slice(&req_body).expect("patch object is json");
698            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
699                .expect("valid kanidm status");
700            let mut kanidm = Kanidm::test();
701            kanidm.status = Some(status.clone());
702            let response = serde_json::to_vec(&kanidm).unwrap();
703            // pass through kanidm "patch accepted"
704            send.send_response(Response::builder().body(Body::from(response)).unwrap());
705            Ok(self)
706        }
707
708        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
709            for rg in kanidm.spec.replica_groups.iter() {
710                let (request, send) = self.0.next_request().await.expect("service not called");
711                assert_eq!(request.method(), http::Method::PATCH);
712                assert_eq!(
713                    request.uri().to_string(),
714                    format!(
715                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
716                        kanidm.statefulset_name(&rg.name)
717                    )
718                );
719                let req_body = request.into_body().collect_bytes().await.unwrap();
720                let json: serde_json::Value =
721                    serde_json::from_slice(&req_body).expect("patch object is json");
722                let statefulset: StatefulSet =
723                    serde_json::from_value(json).expect("valid statefulset");
724                assert_eq!(
725                    statefulset.clone().spec.unwrap().replicas.unwrap(),
726                    rg.replicas
727                );
728                let response = serde_json::to_vec(&statefulset).unwrap();
729                // pass through kanidm "patch accepted"
730                send.send_response(Response::builder().body(Body::from(response)).unwrap());
731            }
732
733            Ok(self)
734        }
735
736        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
737            let (request, send) = self.0.next_request().await.expect("service not called");
738            assert_eq!(request.method(), http::Method::PATCH);
739            assert_eq!(
740                request.uri().to_string(),
741                format!(
742                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
743                    kanidm.name_any()
744                )
745            );
746
747            let req_body = request.into_body().collect_bytes().await.unwrap();
748            let json: serde_json::Value =
749                serde_json::from_slice(&req_body).expect("patch object is json");
750            let service: Service = serde_json::from_value(json).expect("valid service");
751            let response = serde_json::to_vec(&service).unwrap();
752            // pass through kanidm "patch accepted"
753            send.send_response(Response::builder().body(Body::from(response)).unwrap());
754            Ok(self)
755        }
756
757        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
758            let (request, send) = self.0.next_request().await.expect("service not called");
759            assert_eq!(request.method(), http::Method::PATCH);
760            assert_eq!(
761                request.uri().to_string(),
762                format!(
763                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
764                    kanidm.name_any()
765                )
766            );
767
768            let req_body = request.into_body().collect_bytes().await.unwrap();
769            let json: serde_json::Value =
770                serde_json::from_slice(&req_body).expect("patch object is json");
771            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
772            let response = serde_json::to_vec(&ingress).unwrap();
773            // pass through kanidm "patch accepted"
774            send.send_response(Response::builder().body(Body::from(response)).unwrap());
775            Ok(self)
776        }
777    }
778
779    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
780        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
781        let mock_client = Client::new(mock_service, "default");
782        let stores = Stores {
783            stateful_set_store: Writer::default().as_reader(),
784            service_store: Writer::default().as_reader(),
785            ingress_store: Writer::default().as_reader(),
786            secret_store: Writer::default().as_reader(),
787        };
788        let controller_id = "test";
789
790        // Create a test meter for metrics
791        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
792        let meter = provider.meter("test");
793        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
794
795        let state = State::new(
796            metrics,
797            Writer::default().as_reader(),
798            Writer::default().as_reader(),
799            None,
800        );
801        let ctx = Arc::new(Context::new(
802            state.to_context(mock_client, controller_id),
803            stores,
804        ));
805        (ctx, ApiServerVerifier(handle))
806    }
807
808    #[tokio::test]
809    async fn kanidm_create() {
810        let (testctx, fakeserver) = get_test_context();
811        let kanidm = Kanidm::test();
812        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
813        reconcile_kanidm(Arc::new(kanidm), testctx)
814            .await
815            .expect("reconciler");
816        timeout_after_1s(mocksrv).await;
817    }
818
819    #[tokio::test]
820    async fn kanidm_create_with_two_replicas() {
821        let (testctx, fakeserver) = get_test_context();
822        let kanidm = Kanidm::test().with_replicas(2);
823        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
824        reconcile_kanidm(Arc::new(kanidm), testctx)
825            .await
826            .expect("reconciler");
827        timeout_after_1s(mocksrv).await;
828    }
829
830    #[tokio::test]
831    async fn kanidm_create_with_ingress() {
832        let (testctx, fakeserver) = get_test_context();
833        let kanidm = Kanidm::test().with_ingress();
834        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
835        reconcile_kanidm(Arc::new(kanidm), testctx)
836            .await
837            .expect("reconciler");
838        timeout_after_1s(mocksrv).await;
839    }
840
841    #[tokio::test]
842    async fn kanidm_create_with_ingress_with_two_replicas() {
843        let (testctx, fakeserver) = get_test_context();
844        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
845        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
846        reconcile_kanidm(Arc::new(kanidm), testctx)
847            .await
848            .expect("reconciler");
849        timeout_after_1s(mocksrv).await;
850    }
851}