Skip to main content

kaniop_operator/controller/
mod.rs

1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use kube::Resource;
22use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
23use kube::client::Client;
24use kube::runtime::controller::Action;
25use kube::runtime::events::Recorder;
26use kube::runtime::reflector::store::Writer;
27use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
28use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
29use serde::de::DeserializeOwned;
30use tokio::sync::RwLock;
31use tokio::time::Duration;
32use tracing::{debug, error, info, trace};
33
34use std::sync::OnceLock;
35
36const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
37const DEFAULT_CLUSTER_DOMAIN: &str = "cluster.local";
38const DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS: u64 = 300;
39const DEFAULT_BACKUP_DISCOVERY_STALE_SECS: u64 = 900;
40
41static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
42static CLUSTER_DOMAIN: OnceLock<String> = OnceLock::new();
43static BACKUP_DISCOVERY_SCAN_INTERVAL: OnceLock<Duration> = OnceLock::new();
44static BACKUP_DISCOVERY_STALE_THRESHOLD: OnceLock<Duration> = OnceLock::new();
45
46pub fn set_idm_reconcile_interval(duration: Duration) {
47    let _ = IDM_RECONCILE_INTERVAL.set(duration);
48}
49
50pub fn idm_reconcile_interval() -> Duration {
51    *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
52}
53
54pub fn set_cluster_domain(domain: String) {
55    let _ = CLUSTER_DOMAIN.set(domain);
56}
57
58pub fn cluster_domain() -> &'static str {
59    CLUSTER_DOMAIN.get_or_init(|| DEFAULT_CLUSTER_DOMAIN.to_string())
60}
61
62pub fn set_backup_discovery_scan_interval(duration: Duration) {
63    let _ = BACKUP_DISCOVERY_SCAN_INTERVAL.set(duration);
64}
65
66pub fn backup_discovery_scan_interval() -> Duration {
67    *BACKUP_DISCOVERY_SCAN_INTERVAL
68        .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_SCAN_INTERVAL_SECS))
69}
70
71pub fn set_backup_discovery_stale_threshold(duration: Duration) {
72    let _ = BACKUP_DISCOVERY_STALE_THRESHOLD.set(duration);
73}
74
75pub fn backup_discovery_stale_threshold() -> Duration {
76    *BACKUP_DISCOVERY_STALE_THRESHOLD
77        .get_or_init(|| Duration::from_secs(DEFAULT_BACKUP_DISCOVERY_STALE_SECS))
78}
79pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
80pub const RELOAD_BUFFER_SIZE: usize = 16;
81pub const NAME_LABEL: &str = "app.kubernetes.io/name";
82pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
83pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
84
85pub type ControllerId = &'static str;
86
87/// State shared between the controller and the web server
88// Kanidm defined as a generic because it causes a cycle dependency with the kaniop_kanidm crate
89#[derive(Clone)]
90pub struct State {
91    /// Metrics
92    metrics: Arc<metrics::Metrics>,
93    /// Shared Kanidm cache clients with the ability to manage users and their groups
94    idm_clients: Arc<RwLock<KanidmClients>>,
95    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
96    /// database and service
97    system_clients: Arc<RwLock<KanidmClients>>,
98    /// Cache for Namespace resources
99    pub namespace_store: Store<Namespace>,
100    /// Cache for Kanidm resources
101    pub kanidm_store: Store<Kanidm>,
102    /// Kubernetes client
103    pub client: Option<Client>,
104}
105
106/// Shared state for a resource stream
107pub struct ResourceReflector<K>
108where
109    K: Resource + Lookup + Clone + 'static,
110    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
111{
112    pub store: Store<K>,
113    pub writer: Writer<K>,
114    pub subscriber: ReflectHandle<K>,
115}
116
117/// State wrapper around the controller outputs for the web server
118impl State {
119    pub fn new(
120        metrics: metrics::Metrics,
121        namespace_store: Store<Namespace>,
122        kanidm_store: Store<Kanidm>,
123        client: Option<Client>,
124    ) -> Self {
125        Self {
126            metrics: Arc::new(metrics),
127            idm_clients: Arc::default(),
128            system_clients: Arc::default(),
129            namespace_store,
130            kanidm_store,
131            client,
132        }
133    }
134
135    /// Metrics getter
136    pub fn metrics(&self) -> Result<String> {
137        prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
138            Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
139        })
140    }
141
142    /// Create a Controller Context that can update State
143    pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
144    where
145        K: Resource + Lookup + Clone + 'static,
146        <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
147    {
148        Context::new(
149            controller_id,
150            client.clone(),
151            self.metrics
152                .controllers
153                .get(controller_id)
154                .expect("all CONTROLLER_IDs have to be registered")
155                .clone(),
156            Recorder::new(client.clone(), controller_id.into()),
157            self.idm_clients.clone(),
158            self.system_clients.clone(),
159            self.namespace_store.clone(),
160            self.kanidm_store.clone(),
161        )
162    }
163}
164
165pub async fn check_api_queryable<K>(client: Client) -> Api<K>
166where
167    K: Resource + Clone + DeserializeOwned + Debug,
168    <K as Resource>::DynamicType: Default,
169{
170    let api = Api::<K>::all(client.clone());
171    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
172        error!(
173            "{} is not queryable; {e:?}. Check controller permissions",
174            short_type_name::<K>().unwrap_or("Unknown resource"),
175        );
176        std::process::exit(1);
177    }
178    api
179}
180
181pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
182where
183    K: Resource + Clone + DeserializeOwned + Debug,
184    <K as Resource>::DynamicType: Default,
185{
186    let api = Api::<K>::all(client.clone());
187    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
188        info!(
189            "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
190            short_type_name::<K>().unwrap_or("Unknown resource"),
191        );
192        None
193    } else {
194        Some(api)
195    }
196}
197
198pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
199where
200    K: Resource + Lookup + Clone + 'static,
201    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
202    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
203{
204    let (store, writer) = reflector::store_shared::<K>(buffer_size);
205    let subscriber = writer
206        .subscribe()
207        .expect("subscribers can only be created from shared stores");
208
209    ResourceReflector {
210        store,
211        writer,
212        subscriber,
213    }
214}
215
216fn create_generic_watcher<K, W, T, S, StreamT>(
217    api: Api<K>,
218    writer: Writer<W>,
219    reload_tx: mpsc::Sender<()>,
220    controller_id: ControllerId,
221    ctx: Arc<Context<T>>,
222    stream_fn: S,
223) -> BoxFuture<'static, ()>
224where
225    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
226    W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
227    <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
228    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
229    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
230    <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
231    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
232    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
233    S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
234    StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
235        + Send
236        + 'static,
237{
238    let resource_name = short_type_name::<K>().unwrap_or("Unknown");
239
240    stream_fn(
241        api,
242        watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
243    )
244    .default_backoff()
245    .reflect_shared(writer)
246    .for_each(move |res| {
247        let mut reload_tx_clone = reload_tx.clone();
248        let ctx = ctx.clone();
249        async move {
250            match res {
251                Ok(event) => {
252                    trace!(?event, "watched event");
253                    match event {
254                        watcher::Event::Delete(d) => {
255                            debug!(
256                                namespace = ResourceExt::namespace(&d).unwrap(),
257                                name = d.name_any(),
258                                "delete event for {resource_name} trigger reconcile"
259                            );
260
261                            // TODO: remove for each trigger on delete logic when
262                            // (dispatch delete events issue)[https://github.com/kube-rs/kube/issues/1590]
263                            // is solved
264                            let _ignore_errors = reload_tx_clone
265                                .try_send(())
266                                .map_err(|e| error!(%e, "failed to trigger reconcile on delete"));
267                            ctx.metrics
268                                .triggered_inc(metrics::Action::Delete, resource_name);
269                        }
270                        watcher::Event::Apply(d) => {
271                            debug!(
272                                namespace = ResourceExt::namespace(&d).unwrap(),
273                                name = d.name_any(),
274                                "apply event for {resource_name} trigger reconcile"
275                            );
276                            ctx.metrics
277                                .triggered_inc(metrics::Action::Apply, resource_name);
278                        }
279                        _ => {}
280                    }
281                }
282                Err(e) => {
283                    error!(%e, "unexpected error when watching {resource_name}");
284                    ctx.metrics.watch_operations_failed_inc();
285                }
286            }
287        }
288    })
289    .boxed()
290}
291
292pub fn create_watcher<K, T>(
293    api: Api<K>,
294    writer: Writer<K>,
295    reload_tx: mpsc::Sender<()>,
296    controller_id: ControllerId,
297    ctx: Arc<Context<T>>,
298) -> BoxFuture<'static, ()>
299where
300    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
301    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
302    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
303    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
304    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
305{
306    create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
307}
308
309pub fn create_metadata_watcher<K, T>(
310    api: Api<K>,
311    writer: Writer<PartialObjectMeta<K>>,
312    reload_tx: mpsc::Sender<()>,
313    controller_id: ControllerId,
314    ctx: Arc<Context<T>>,
315) -> BoxFuture<'static, ()>
316where
317    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
318    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
319    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
320    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
321    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
322{
323    create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
324        api,
325        writer,
326        reload_tx,
327        controller_id,
328        ctx,
329        metadata_watcher,
330    )
331}
332
333pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
334where
335    K: Resource + Lookup + Clone + 'static,
336    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
337{
338    tracing::warn!(
339        "error_policy called unexpectedly - all controllers should use backoff_reconciler! macro"
340    );
341    Action::requeue(Duration::from_secs(300))
342}
343
344#[macro_export]
345macro_rules! backoff_reconciler {
346    ($inner_reconciler:ident) => {
347        |obj, ctx| async move {
348            use $crate::controller::context::BackoffContext;
349            let _active_guard = ctx.metrics().active_reconcile_inc();
350            match $inner_reconciler(obj.clone(), ctx.clone()).await {
351                Ok((action, changed)) => {
352                    ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
353                        .await;
354                    if changed {
355                        ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_CHANGED);
356                    } else {
357                        ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_UNCHANGED);
358                    }
359                    Ok(action)
360                }
361                Err(error) => {
362                    // safe unwrap: all resources in the operator are namespace scoped resources
363                    let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
364                    let name = kube::ResourceExt::name_any(obj.as_ref());
365                    tracing::error!(%namespace, %name, %error, "failed reconciliation");
366                    ctx.metrics().reconcile_failure_inc();
367                    ctx.metrics().reconcile_outcome_record($crate::metrics::KANIDM_OUTCOME_ERROR);
368                    let backoff_duration = ctx
369                        .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
370                        .await;
371                    tracing::trace!(%namespace, %name, "backoff duration: {backoff_duration:?}");
372                    Ok(kube::runtime::controller::Action::requeue(backoff_duration))
373                }
374            }
375        }
376    };
377}