Skip to main content

kaniop_operator/kanidm/reconcile/
mod.rs

1mod domain_appearance;
2mod gateway;
3mod ingress;
4pub mod secret;
5mod service;
6pub mod statefulset;
7mod status;
8
9use self::domain_appearance::reconcile_domain_appearance;
10use super::controller::{CONTROLLER_ID, context::Context};
11
12use self::gateway::GatewayExt;
13use self::ingress::IngressExt;
14use self::secret::SecretExt;
15use self::service::ServiceExt;
16use self::statefulset::StatefulSetExt;
17use self::status::StatusExt;
18use self::status::{is_kanidm_available, is_kanidm_initialized};
19
20use crate::controller::context::KubeOperations;
21use crate::controller::{INSTANCE_LABEL, MANAGED_BY_LABEL, NAME_LABEL};
22use crate::kanidm::crd::{
23    Kanidm, KanidmReplicaState, KanidmStatus, KanidmUpgradeCheckResult, VersionCompatibilityResult,
24};
25use crate::kanidm::reconcile::statefulset::REPLICA_LABEL;
26use crate::telemetry;
27
28use kaniop_k8s_util::client::get_output;
29use kaniop_k8s_util::error::{Error, Result};
30use kaniop_k8s_util::parse::parse_semver;
31use kaniop_k8s_util::resources::get_image_tag;
32
33use std::collections::BTreeMap;
34use std::fmt::Debug;
35use std::sync::{Arc, LazyLock};
36
37use futures::future::{TryJoinAll, join_all, try_join_all};
38use futures::stream::{self, StreamExt};
39use futures::try_join;
40use k8s_openapi::NamespaceResourceScope;
41use k8s_openapi::api::apps::v1::StatefulSet;
42use k8s_openapi::api::core::v1::Pod;
43use kube::Resource;
44use kube::ResourceExt;
45use kube::api::{Api, AttachParams};
46use kube::runtime::controller::Action;
47use kube::runtime::events::{Event, EventType};
48use kube::runtime::finalizer::{Event as Finalizer, finalizer};
49use serde::{Deserialize, Serialize};
50use tokio::time::Duration;
51use tracing::{Span, debug, field, info, instrument, trace, warn};
52
53pub const CLUSTER_LABEL: &str = "kanidm.kaniop.rs/cluster";
54const KANIDM_OPERATOR_NAME: &str = "kanidms.kaniop.rs";
55pub static KANIDM_FINALIZER: &str = "kanidms.kaniop.rs/finalizer";
56
57const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
58
59static LABELS: LazyLock<BTreeMap<String, String>> = LazyLock::new(|| {
60    BTreeMap::from([
61        (NAME_LABEL.to_string(), "kanidm".to_string()),
62        (
63            MANAGED_BY_LABEL.to_string(),
64            format!("kaniop-{CONTROLLER_ID}"),
65        ),
66    ])
67});
68
69pub async fn reconcile_admins_secret(
70    kanidm: Arc<Kanidm>,
71    ctx: Arc<Context>,
72    status: &KanidmStatus,
73) -> Result<()> {
74    if is_kanidm_available(status.clone()) && !is_kanidm_initialized(status.clone()) {
75        let admins_secret = kanidm.generate_admins_secret(ctx.clone()).await?;
76        kanidm.patch(&ctx, admins_secret).await?;
77    }
78    Ok(())
79}
80
81pub async fn reconcile_replication_secrets(
82    kanidm: Arc<Kanidm>,
83    ctx: Arc<Context>,
84    status: &KanidmStatus,
85) -> Result<()> {
86    let expected_secret_names = if kanidm.is_replication_enabled() {
87        status
88            .replica_statuses
89            .iter()
90            .map(|rs| kanidm.replica_secret_name(&rs.pod_name))
91            .collect::<Vec<_>>()
92    } else {
93        Vec::new()
94    };
95    let deprecated_secrets = ctx
96        .stores
97        .secret_store
98        .state()
99        .into_iter()
100        .filter(|secret| {
101            secret.namespace() == kanidm.namespace()
102                && kanidm.admins_secret_name() != secret.name_any()
103                && !expected_secret_names.contains(&secret.name_any())
104                && secret
105                    .metadata
106                    .labels
107                    .as_ref()
108                    .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
109        })
110        .collect::<Vec<_>>();
111    let secret_delete_future = deprecated_secrets
112        .iter()
113        .map(|secret| kanidm.delete(&ctx, secret.as_ref()))
114        .collect::<TryJoinAll<_>>();
115    try_join!(secret_delete_future)?;
116
117    if kanidm.is_replication_enabled() {
118        // sequential renewal: concurrent certificate updates are not allowed; abort on first error
119        // https://github.com/kanidm/kanidm/issues/3917
120        for rs in status.replica_statuses.iter().filter(|rs| {
121            rs.state == KanidmReplicaState::CertificateExpiring
122                || rs.state == KanidmReplicaState::CertificateHostInvalid
123        }) {
124            let secret = kanidm
125                .update_replica_secret(ctx.clone(), &rs.pod_name)
126                .await?;
127            kanidm.patch(&ctx, secret.clone()).await?;
128        }
129
130        stream::iter(
131            status
132                .replica_statuses
133                .iter()
134                .filter(|rs| rs.state == KanidmReplicaState::Pending),
135        )
136        .then(|rs| async {
137            let secret = kanidm
138                .generate_replica_secret(ctx.clone(), &rs.pod_name)
139                .await?;
140            kanidm.patch(&ctx, secret).await
141        })
142        .collect::<Vec<_>>()
143        .await
144        .into_iter()
145        .collect::<Result<Vec<_>>>()?;
146
147        // If there are any replicas in pending or cert expiring state, trigger a restart
148        // of all statefulsets to pickup new certs
149        let has_pending_or_expiring = status
150            .replica_statuses
151            .iter()
152            .any(|rs| rs.state != KanidmReplicaState::Ready);
153
154        if has_pending_or_expiring {
155            let sts_api = Api::<StatefulSet>::namespaced(
156                ctx.kaniop_ctx.client.clone(),
157                &kanidm.get_namespace(),
158            );
159
160            // Check if any replica group has only 1 replica
161            let has_single_replica = kanidm.spec.replica_groups.iter().any(|rg| rg.replicas == 1);
162
163            let restart_futures = status
164                .replica_statuses
165                .iter()
166                .map(|rs| sts_api.restart(&rs.statefulset_name));
167
168            if has_single_replica {
169                // Restart sequentially to avoid downtime
170                let results = stream::iter(restart_futures)
171                    .then(|f| f)
172                    .collect::<Vec<_>>()
173                    .await;
174                for result in results.iter().filter(|r| r.is_err()) {
175                    warn!(msg = "failed to restart statefulset sequentially", error = ?result);
176                }
177            } else {
178                // Restart concurrently for replica groups with multiple replicas
179                let results = join_all(restart_futures).await;
180                for result in results.iter().filter(|r| r.is_err()) {
181                    warn!(msg = "failed to restart statefulset concurrently", error = ?result);
182                }
183            }
184        }
185    }
186
187    Ok(())
188}
189
190#[instrument(skip(ctx, kanidm))]
191pub async fn reconcile_kanidm(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
192    let trace_id = telemetry::get_trace_id();
193    Span::current().record("trace_id", field::display(&trace_id));
194    let _timer = ctx
195        .kaniop_ctx
196        .metrics
197        .reconcile_count_and_measure(&trace_id);
198    info!(msg = "reconciling Kanidm");
199
200    let status = kanidm.update_status(ctx.clone()).await.map_err(|e| {
201        debug!(msg = "failed to reconcile status", %e);
202        ctx.kaniop_ctx.metrics.status_update_errors_inc();
203        e
204    })?;
205    let kanidm_api: Api<Kanidm> =
206        Api::namespaced(ctx.kaniop_ctx.client.clone(), &kanidm.get_namespace());
207    finalizer(&kanidm_api, KANIDM_FINALIZER, kanidm, |event| async {
208        match event {
209            Finalizer::Apply(kanidm) => reconcile(kanidm, ctx, status).await,
210            Finalizer::Cleanup(kanidm) => cleanup(kanidm, ctx).await,
211        }
212    })
213    .await
214    .map_err(|e| {
215        Error::FinalizerError(
216            "failed on kanidm account finalizer".to_string(),
217            Box::new(e),
218        )
219    })
220}
221
222async fn reconcile(kanidm: Arc<Kanidm>, ctx: Arc<Context>, status: KanidmStatus) -> Result<Action> {
223    let admin_secret_future = reconcile_admins_secret(kanidm.clone(), ctx.clone(), &status);
224    let replication_secret_futures =
225        reconcile_replication_secrets(kanidm.clone(), ctx.clone(), &status);
226
227    let sts_to_delete = {
228        let expected_sts_names = kanidm
229            .spec
230            .replica_groups
231            .iter()
232            .map(|rg| kanidm.statefulset_name(&rg.name))
233            .collect::<Vec<_>>();
234        ctx.stores
235            .stateful_set_store
236            .state()
237            .into_iter()
238            .filter(|sts| {
239                sts.namespace() == kanidm.namespace()
240                    && sts
241                        .metadata
242                        .labels
243                        .as_ref()
244                        .is_some_and(|l| l.get(CLUSTER_LABEL) == Some(&kanidm.name_any()))
245                    && !expected_sts_names.contains(&sts.name_any())
246            })
247            .collect::<Vec<_>>()
248    };
249    let sts_delete_futures = sts_to_delete
250        .iter()
251        .map(|sts| kanidm.delete(&ctx, sts.as_ref()))
252        .collect::<TryJoinAll<_>>();
253
254    let sts_futures = match kanidm.is_updatable(&status) {
255        true => kanidm
256            .spec
257            .replica_groups
258            .iter()
259            .map(|rg| {
260                let sts = kanidm.create_statefulset(rg, &ctx)?;
261                Ok(kanidm.patch(&ctx, sts))
262            })
263            .collect::<Result<TryJoinAll<_>, _>>()?,
264        false => {
265            let note = match status.version.as_ref() {
266                Some(v) if v.compatibility_result == VersionCompatibilityResult::Incompatible => {
267                    format!(
268                        "Version change blocked: image version {} is not compatible with operator (uses Kanidm client v{}). Override with `.spec.disableUpgradeChecks: true` or use a compatible version.",
269                        v.image_tag,
270                        crate::version::KANIDM_CLIENT_VERSION
271                    )
272                }
273                _ => "Version change blocked: upgrade pre-check failed. Override with `.spec.disableUpgradeCheck: true` or update the resource to retry.".to_string(),
274            };
275            let _ignore_error = ctx
276                .kaniop_ctx
277                .recorder
278                .publish(
279                    &Event {
280                        type_: EventType::Warning,
281                        reason: "UpgradeBlocked".to_string(),
282                        note: Some(note),
283                        action: "ReconcileStatefulSet".to_string(),
284                        secondary: None,
285                    },
286                    &kanidm.object_ref(&()),
287                )
288                .await
289                .map_err(|e| {
290                    warn!(msg = "failed to publish KanidmError event", %e);
291                    Error::KubeError("failed to publish event".to_string(), Box::new(e))
292                });
293            try_join_all([])
294        }
295    };
296    let service_future = kanidm.patch(&ctx, kanidm.create_service());
297
298    let deprecated_rg_svcs = {
299        let expected_rg_svcs_names = kanidm
300            .spec
301            .replica_groups
302            .iter()
303            .filter(|rg| rg.services.is_some())
304            .flat_map(|rg| (0..rg.replicas).map(|i| kanidm.replica_group_service_name(&rg.name, i)))
305            .collect::<Vec<_>>();
306        ctx.stores
307            .service_store
308            .state()
309            .into_iter()
310            .filter(|svc| {
311                let labels = svc.metadata.labels.as_ref();
312                svc.namespace() == kanidm.namespace()
313                    && !expected_rg_svcs_names.contains(&svc.name_any())
314                    && labels.is_some_and(|l| {
315                        l.contains_key(REPLICA_LABEL)
316                            && l.get(CLUSTER_LABEL) == Some(&kanidm.name_any())
317                    })
318            })
319            .collect::<Vec<_>>()
320    };
321    let rg_svcs_delete_futures = deprecated_rg_svcs
322        .iter()
323        .map(|svc| kanidm.delete(&ctx, svc.as_ref()))
324        .collect::<TryJoinAll<_>>();
325
326    let rg_services_futures = kanidm
327        .spec
328        .replica_groups
329        .iter()
330        .filter(|rg| rg.services.is_some())
331        .flat_map(|rg| {
332            (0..rg.replicas).map(|i| kanidm.patch(&ctx, kanidm.create_replica_group_service(rg, i)))
333        })
334        .collect::<TryJoinAll<_>>();
335
336    let deprecated_ingresses = {
337        let names = [
338            kanidm.spec.ingress.as_ref().map(|_| kanidm.name_any()),
339            kanidm.generate_region_ingress_name(),
340        ]
341        .into_iter()
342        .flatten()
343        .collect::<Vec<_>>();
344        ctx.stores
345            .ingress_store
346            .state()
347            .into_iter()
348            .filter(|ing| {
349                ing.namespace() == kanidm.namespace()
350                    && !names.contains(&ing.name_any())
351                    && ing.metadata.labels == Some(kanidm.generate_labels())
352            })
353            .collect::<Vec<_>>()
354    };
355    let ingress_delete_futures = deprecated_ingresses
356        .iter()
357        .map(|ing| kanidm.delete(&ctx, ing.as_ref()))
358        .collect::<TryJoinAll<_>>();
359
360    let ingress_futures = kanidm
361        .create_ingress()
362        .into_iter()
363        .chain(kanidm.create_region_ingress())
364        .map(|ingress| kanidm.patch(&ctx, ingress))
365        .collect::<TryJoinAll<_>>();
366
367    let deprecated_http_routes = match &ctx.stores.http_route_store {
368        Some(store) => {
369            let expected_names = kanidm
370                .spec
371                .gateway
372                .as_ref()
373                .map(|_| vec![kanidm.name_any()])
374                .unwrap_or_default();
375            store
376                .state()
377                .into_iter()
378                .filter(|route| {
379                    route.namespace() == kanidm.namespace()
380                        && !expected_names.contains(&route.name_any())
381                        && route.metadata.labels == Some(kanidm.generate_labels())
382                })
383                .collect::<Vec<_>>()
384        }
385        None => Vec::new(),
386    };
387    let http_route_delete_futures = deprecated_http_routes
388        .iter()
389        .map(|route| kanidm.delete(&ctx, route.as_ref()))
390        .collect::<TryJoinAll<_>>();
391
392    let http_route_futures = if ctx.stores.http_route_store.is_some() {
393        kanidm
394            .create_http_route()
395            .into_iter()
396            .map(|route| kanidm.patch(&ctx, route))
397            .collect::<TryJoinAll<_>>()
398    } else {
399        try_join_all(Vec::new())
400    };
401
402    try_join!(
403        sts_delete_futures,
404        admin_secret_future,
405        replication_secret_futures,
406        sts_futures,
407        service_future,
408        rg_svcs_delete_futures,
409        rg_services_futures,
410        ingress_delete_futures,
411        ingress_futures,
412        http_route_delete_futures,
413        http_route_futures
414    )?;
415
416    if is_kanidm_available(status.clone()) {
417        let namespace = kanidm.namespace().unwrap();
418        let name = kanidm.name_any();
419        let kanidm_client = crate::controller::kanidm::KanidmClients::create_client(
420            &namespace,
421            &name,
422            crate::controller::kanidm::KanidmUser::Admin,
423            ctx.kaniop_ctx.client.clone(),
424        )
425        .await?;
426        reconcile_domain_appearance(&kanidm, kanidm_client, &status, ctx.clone()).await?;
427    }
428
429    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
430}
431
432async fn cleanup(kanidm: Arc<Kanidm>, ctx: Arc<Context>) -> Result<Action> {
433    debug!(msg = "cleanup");
434    ctx.kaniop_ctx.release_kanidm_clients(&kanidm).await;
435    Ok(Action::requeue(DEFAULT_RECONCILE_INTERVAL))
436}
437
438impl Kanidm {
439    // Convenience methods that handle context and operator name
440    pub async fn delete<K>(&self, ctx: &Context, resource: &K) -> Result<()>
441    where
442        K: Resource<Scope = NamespaceResourceScope>
443            + Serialize
444            + Clone
445            + std::fmt::Debug
446            + for<'de> Deserialize<'de>,
447        <K as kube::Resource>::DynamicType: Default,
448        <K as Resource>::Scope: std::marker::Sized,
449    {
450        self.kube_delete(
451            ctx.kaniop_ctx.client.clone(),
452            &ctx.kaniop_ctx.metrics,
453            resource,
454        )
455        .await
456    }
457
458    pub async fn patch<K>(&self, ctx: &Context, resource: K) -> Result<K>
459    where
460        K: Resource<Scope = NamespaceResourceScope>
461            + Serialize
462            + Clone
463            + std::fmt::Debug
464            + for<'de> Deserialize<'de>,
465        <K as kube::Resource>::DynamicType: Default,
466        <K as Resource>::Scope: std::marker::Sized,
467    {
468        self.kube_patch(
469            ctx.kaniop_ctx.client.clone(),
470            &ctx.kaniop_ctx.metrics,
471            resource,
472            KANIDM_OPERATOR_NAME,
473        )
474        .await
475    }
476
477    #[inline]
478    fn generate_resource_labels(&self) -> BTreeMap<String, String> {
479        LABELS
480            .clone()
481            .into_iter()
482            .chain([
483                (INSTANCE_LABEL.to_string(), self.name_any()),
484                (CLUSTER_LABEL.to_string(), self.name_any()),
485            ])
486            .collect()
487    }
488
489    #[inline]
490    fn generate_labels(&self) -> BTreeMap<String, String> {
491        self.generate_resource_labels()
492    }
493
494    #[inline]
495    fn get_tls_secret_name(&self) -> String {
496        format!("{}-tls", self.name_any())
497    }
498
499    #[inline]
500    fn get_namespace(&self) -> String {
501        // safe unwrap: Kanidm is namespaced scoped
502        self.namespace().unwrap()
503    }
504
505    #[inline]
506    fn is_replication_enabled(&self) -> bool {
507        // safe unwrap: at least one replica group is required
508        self.spec.replica_groups.len() > 1
509            || self.spec.replica_groups.first().unwrap().replicas > 1
510            || !self.spec.external_replication_nodes.is_empty()
511    }
512
513    #[inline]
514    fn is_updatable(&self, status: &KanidmStatus) -> bool {
515        status
516            .version
517            .as_ref()
518            .map(|v| {
519                if v.compatibility_result == VersionCompatibilityResult::Incompatible {
520                    return false;
521                }
522                let current_tag = get_image_tag(&self.spec.image).unwrap_or_default();
523                let versions = (parse_semver(&current_tag), parse_semver(&v.image_tag));
524                match v.upgrade_check_result {
525                    KanidmUpgradeCheckResult::Passed => {
526                        if let (Some((_, minor, _)), Some((_, v_minor, _))) = versions {
527                            minor <= v_minor + 1
528                        } else {
529                            true
530                        }
531                    }
532                    KanidmUpgradeCheckResult::Failed => {
533                        if let (Some((major, minor, patch)), Some((v_major, v_minor, v_patch))) =
534                            versions
535                        {
536                            major == v_major && minor == v_minor && patch >= v_patch
537                        } else {
538                            v.image_tag == current_tag
539                        }
540                    }
541                }
542            })
543            .unwrap_or(true)
544    }
545
546    async fn exec<I, T>(&self, ctx: Arc<Context>, pod_name: &str, command: I) -> Result<String>
547    where
548        I: IntoIterator<Item = T> + Debug,
549        T: Into<String>,
550    {
551        let namespace = &self.get_namespace();
552        trace!(
553            msg = "pod exec",
554            resource.name = &pod_name,
555            resource.namespace = &namespace,
556            ?command
557        );
558        let pod = Api::<Pod>::namespaced(ctx.kaniop_ctx.client.clone(), namespace);
559        let attached = pod
560            .exec(pod_name, command, &AttachParams::default())
561            .await
562            .map_err(|e| {
563                Error::KubeError(
564                    format!("failed to exec pod {namespace}/{pod_name}"),
565                    Box::new(e),
566                )
567            })?;
568        get_output(attached).await
569    }
570
571    async fn exec_any<I, T>(&self, ctx: Arc<Context>, command: I) -> Result<String>
572    where
573        I: IntoIterator<Item = T> + Debug,
574        T: Into<String>,
575    {
576        // TODO: if replicas > 1 and replicas initialized, exec on pod available
577        // safe unwrap: at least one replica group is required
578        let sts_name = self.statefulset_name(&self.spec.replica_groups.first().unwrap().name);
579        let pod_name = format!("{sts_name}-0");
580        self.exec(ctx, &pod_name, command).await
581    }
582}
583
584#[cfg(test)]
585mod test {
586    use super::statefulset::StatefulSetExt;
587    use super::{Kanidm, reconcile_kanidm};
588
589    use crate::controller::State;
590    use crate::kanidm::controller::context::{Context, Stores};
591    use crate::kanidm::crd::KanidmStatus;
592
593    use kaniop_k8s_util::error::Result;
594
595    use std::sync::Arc;
596
597    use http::{Request, Response};
598    use k8s_openapi::api::apps::v1::StatefulSet;
599    use k8s_openapi::api::core::v1::Service;
600    use k8s_openapi::api::networking::v1::Ingress;
601    use kube::runtime::reflector::store::Writer;
602    use kube::{Client, Resource, ResourceExt, client::Body};
603    use opentelemetry::metrics::MeterProvider;
604    use serde_json::json;
605
606    impl Kanidm {
607        /// A normal test kanidm with a given status
608        pub fn test() -> Self {
609            let mut e = Kanidm::new(
610                "test",
611                serde_json::from_value(json!({
612                    "domain": "idm.example.com",
613                    "replicaGroups": [
614                        {
615                            "name": "default",
616                            "replicas": 1
617                        }
618                    ]
619                }))
620                .unwrap(),
621            );
622            e.meta_mut().namespace = Some("default".into());
623            // Pre-populate finalizer so reconcile callback runs immediately
624            // (skips the "add finalizer + wait for next reconcile" step)
625            e.meta_mut().finalizers = Some(vec![super::KANIDM_FINALIZER.to_string()]);
626            e
627        }
628
629        pub fn with_ingress(mut self) -> Self {
630            self.spec.ingress = Some(serde_json::from_value(json!({})).unwrap());
631            self
632        }
633
634        /// Modify kanidm replicas
635        pub fn with_replicas(mut self, replicas: i32) -> Self {
636            self.spec.replica_groups[0].replicas = replicas;
637            self
638        }
639
640        /// Modify kanidm to set a deletion timestamp
641        pub fn needs_delete(mut self) -> Self {
642            use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
643            use k8s_openapi::jiff::Timestamp;
644            let now = Timestamp::from_second(1491138632).unwrap(); // 2017-04-02 12:50:32 UTC
645            self.meta_mut().deletion_timestamp = Some(Time(now));
646            self
647        }
648
649        /// Modify a kanidm to have an expected status
650        pub fn with_status(mut self, status: KanidmStatus) -> Self {
651            self.status = Some(status);
652            self
653        }
654    }
655
656    // We wrap tower_test::mock::Handle
657    type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
658    pub struct ApiServerVerifier(ApiServerHandle);
659
660    /// Scenarios we test for in ApiServerVerifier
661    pub enum Scenario {
662        Create(Kanidm),
663        CreateWithTwoReplicas(Kanidm),
664        CreateWithIngress(Kanidm),
665        CreateWithIngressWithTwoReplicas(Kanidm),
666    }
667
668    pub async fn timeout_after_1s(handle: tokio::task::JoinHandle<()>) {
669        tokio::time::timeout(std::time::Duration::from_secs(1), handle)
670            .await
671            .expect("timeout on mock apiserver")
672            .expect("scenario succeeded")
673    }
674
675    impl ApiServerVerifier {
676        /// Tests only get to run specific scenarios that has matching handlers
677        ///
678        /// This setup makes it easy to handle multiple requests by chaining handlers together.
679        ///
680        /// NB: If the controller is making more calls than we are handling in the scenario,
681        /// you then typically see a `KubeError(Service(Closed(())))` from the reconciler.
682        ///
683        /// You should await the `JoinHandle` (with a timeout) from this function to ensure that the
684        /// scenario runs to completion (i.e. all expected calls were responded to),
685        /// using the timeout to catch missing api calls to Kubernetes.
686        pub fn run(self, scenario: Scenario) -> tokio::task::JoinHandle<()> {
687            tokio::spawn(async move {
688                // moving self => one scenario per test
689                match scenario {
690                    Scenario::Create(kanidm) => {
691                        self.handle_kanidm_get(kanidm.clone())
692                            .await
693                            .unwrap()
694                            .handle_kanidm_status_patch(kanidm.clone())
695                            .await
696                            .unwrap()
697                            .handle_statefulset_patch(kanidm.clone())
698                            .await
699                            .unwrap()
700                            .handle_service_patch(kanidm.clone())
701                            .await
702                    }
703                    Scenario::CreateWithTwoReplicas(kanidm) => {
704                        self.handle_kanidm_get(kanidm.clone())
705                            .await
706                            .unwrap()
707                            .handle_kanidm_status_patch(kanidm.clone())
708                            .await
709                            .unwrap()
710                            .handle_statefulset_patch(kanidm.clone())
711                            .await
712                            .unwrap()
713                            .handle_service_patch(kanidm.clone())
714                            .await
715                    }
716                    Scenario::CreateWithIngress(kanidm) => {
717                        self.handle_kanidm_get(kanidm.clone())
718                            .await
719                            .unwrap()
720                            .handle_kanidm_status_patch(kanidm.clone())
721                            .await
722                            .unwrap()
723                            .handle_statefulset_patch(kanidm.clone())
724                            .await
725                            .unwrap()
726                            .handle_service_patch(kanidm.clone())
727                            .await
728                            .unwrap()
729                            .handle_ingress_patch(kanidm.clone())
730                            .await
731                    }
732                    Scenario::CreateWithIngressWithTwoReplicas(kanidm) => {
733                        self.handle_kanidm_get(kanidm.clone())
734                            .await
735                            .unwrap()
736                            .handle_kanidm_status_patch(kanidm.clone())
737                            .await
738                            .unwrap()
739                            .handle_statefulset_patch(kanidm.clone())
740                            .await
741                            .unwrap()
742                            .handle_service_patch(kanidm.clone())
743                            .await
744                            .unwrap()
745                            .handle_ingress_patch(kanidm.clone())
746                            .await
747                    }
748                }
749                .expect("scenario completed without errors");
750            })
751        }
752
753        async fn handle_kanidm_get(mut self, kanidm: Kanidm) -> Result<Self> {
754            let (request, send) = self.0.next_request().await.expect("service not called");
755            assert_eq!(request.method(), http::Method::GET);
756            let expected_uri_prefix = format!(
757                "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}",
758                kanidm.name_any()
759            );
760            let uri = request.uri().to_string();
761            assert!(
762                uri.starts_with(&expected_uri_prefix),
763                "expected uri to start with {}, got {}",
764                expected_uri_prefix,
765                uri
766            );
767            let response = serde_json::to_vec(&kanidm).unwrap();
768            send.send_response(Response::builder().body(Body::from(response)).unwrap());
769            Ok(self)
770        }
771
772        async fn handle_kanidm_status_patch(mut self, kanidm: Kanidm) -> Result<Self> {
773            let (request, send) = self.0.next_request().await.expect("service not called");
774            assert_eq!(request.method(), http::Method::PATCH);
775            assert_eq!(
776                request.uri().to_string(),
777                format!(
778                    "/apis/kaniop.rs/v1beta1/namespaces/default/kanidms/{}/status?",
779                    kanidm.name_any()
780                )
781            );
782
783            let req_body = request.into_body().collect_bytes().await.unwrap();
784            let json: serde_json::Value =
785                serde_json::from_slice(&req_body).expect("patch object is json");
786            let status: KanidmStatus = serde_json::from_value(json.get("status").unwrap().clone())
787                .expect("valid kanidm status");
788            let mut kanidm = Kanidm::test();
789            kanidm.status = Some(status.clone());
790            let response = serde_json::to_vec(&kanidm).unwrap();
791            // pass through kanidm "patch accepted"
792            send.send_response(Response::builder().body(Body::from(response)).unwrap());
793            Ok(self)
794        }
795
796        async fn handle_statefulset_patch(mut self, kanidm: Kanidm) -> Result<Self> {
797            for rg in kanidm.spec.replica_groups.iter() {
798                let (request, send) = self.0.next_request().await.expect("service not called");
799                assert_eq!(request.method(), http::Method::PATCH);
800                assert_eq!(
801                    request.uri().to_string(),
802                    format!(
803                        "/apis/apps/v1/namespaces/default/statefulsets/{}?&force=true&fieldManager=kanidms.kaniop.rs",
804                        kanidm.statefulset_name(&rg.name)
805                    )
806                );
807                let req_body = request.into_body().collect_bytes().await.unwrap();
808                let json: serde_json::Value =
809                    serde_json::from_slice(&req_body).expect("patch object is json");
810                let statefulset: StatefulSet =
811                    serde_json::from_value(json).expect("valid statefulset");
812                assert_eq!(
813                    statefulset.clone().spec.unwrap().replicas.unwrap(),
814                    rg.replicas
815                );
816                let response = serde_json::to_vec(&statefulset).unwrap();
817                // pass through kanidm "patch accepted"
818                send.send_response(Response::builder().body(Body::from(response)).unwrap());
819            }
820
821            Ok(self)
822        }
823
824        async fn handle_service_patch(mut self, kanidm: Kanidm) -> Result<Self> {
825            let (request, send) = self.0.next_request().await.expect("service not called");
826            assert_eq!(request.method(), http::Method::PATCH);
827            assert_eq!(
828                request.uri().to_string(),
829                format!(
830                    "/api/v1/namespaces/default/services/{}?&force=true&fieldManager=kanidms.kaniop.rs",
831                    kanidm.name_any()
832                )
833            );
834
835            let req_body = request.into_body().collect_bytes().await.unwrap();
836            let json: serde_json::Value =
837                serde_json::from_slice(&req_body).expect("patch object is json");
838            let service: Service = serde_json::from_value(json).expect("valid service");
839            let response = serde_json::to_vec(&service).unwrap();
840            // pass through kanidm "patch accepted"
841            send.send_response(Response::builder().body(Body::from(response)).unwrap());
842            Ok(self)
843        }
844
845        async fn handle_ingress_patch(mut self, kanidm: Kanidm) -> Result<Self> {
846            let (request, send) = self.0.next_request().await.expect("service not called");
847            assert_eq!(request.method(), http::Method::PATCH);
848            assert_eq!(
849                request.uri().to_string(),
850                format!(
851                    "/apis/networking.k8s.io/v1/namespaces/default/ingresses/{}?&force=true&fieldManager=kanidms.kaniop.rs",
852                    kanidm.name_any()
853                )
854            );
855
856            let req_body = request.into_body().collect_bytes().await.unwrap();
857            let json: serde_json::Value =
858                serde_json::from_slice(&req_body).expect("patch object is json");
859            let ingress: Ingress = serde_json::from_value(json).expect("valid service");
860            let response = serde_json::to_vec(&ingress).unwrap();
861            // pass through kanidm "patch accepted"
862            send.send_response(Response::builder().body(Body::from(response)).unwrap());
863            Ok(self)
864        }
865    }
866
867    pub fn get_test_context() -> (Arc<Context>, ApiServerVerifier) {
868        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
869        let mock_client = Client::new(mock_service, "default");
870        let stores = Stores {
871            stateful_set_store: Writer::default().as_reader(),
872            service_store: Writer::default().as_reader(),
873            ingress_store: Writer::default().as_reader(),
874            secret_store: Writer::default().as_reader(),
875            http_route_store: Some(Writer::default().as_reader()),
876        };
877        let controller_id = "test";
878
879        // Create a test meter for metrics
880        let provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder().build();
881        let meter = provider.meter("test");
882        let metrics = crate::metrics::Metrics::new(&meter, &[controller_id]);
883
884        let state = State::new(
885            metrics,
886            Writer::default().as_reader(),
887            Writer::default().as_reader(),
888            None,
889        );
890        let ctx = Arc::new(Context::new(
891            state.to_context(mock_client, controller_id),
892            stores,
893        ));
894        (ctx, ApiServerVerifier(handle))
895    }
896
897    #[tokio::test]
898    async fn kanidm_create() {
899        let (testctx, fakeserver) = get_test_context();
900        let kanidm = Kanidm::test();
901        let mocksrv = fakeserver.run(Scenario::Create(kanidm.clone()));
902        reconcile_kanidm(Arc::new(kanidm), testctx)
903            .await
904            .expect("reconciler");
905        timeout_after_1s(mocksrv).await;
906    }
907
908    #[tokio::test]
909    async fn kanidm_create_with_two_replicas() {
910        let (testctx, fakeserver) = get_test_context();
911        let kanidm = Kanidm::test().with_replicas(2);
912        let mocksrv = fakeserver.run(Scenario::CreateWithTwoReplicas(kanidm.clone()));
913        reconcile_kanidm(Arc::new(kanidm), testctx)
914            .await
915            .expect("reconciler");
916        timeout_after_1s(mocksrv).await;
917    }
918
919    #[tokio::test]
920    async fn kanidm_create_with_ingress() {
921        let (testctx, fakeserver) = get_test_context();
922        let kanidm = Kanidm::test().with_ingress();
923        let mocksrv = fakeserver.run(Scenario::CreateWithIngress(kanidm.clone()));
924        reconcile_kanidm(Arc::new(kanidm), testctx)
925            .await
926            .expect("reconciler");
927        timeout_after_1s(mocksrv).await;
928    }
929
930    #[tokio::test]
931    async fn kanidm_create_with_ingress_with_two_replicas() {
932        let (testctx, fakeserver) = get_test_context();
933        let kanidm = Kanidm::test().with_ingress().with_replicas(2);
934        let mocksrv = fakeserver.run(Scenario::CreateWithIngressWithTwoReplicas(kanidm.clone()));
935        reconcile_kanidm(Arc::new(kanidm), testctx)
936            .await
937            .expect("reconciler");
938        timeout_after_1s(mocksrv).await;
939    }
940}