Skip to main content

kaniop_operator/controller/
mod.rs

1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use crate::prometheus_exporter;
9
10use kaniop_k8s_util::error::{Error, Result};
11
12use kaniop_k8s_util::types::short_type_name;
13
14use std::fmt::Debug;
15use std::sync::Arc;
16
17use futures::channel::mpsc;
18use futures::future::BoxFuture;
19use futures::{FutureExt, StreamExt};
20use k8s_openapi::api::core::v1::Namespace;
21use kube::Resource;
22use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
23use kube::client::Client;
24use kube::runtime::controller::Action;
25use kube::runtime::events::Recorder;
26use kube::runtime::reflector::store::Writer;
27use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
28use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
29use serde::de::DeserializeOwned;
30use tokio::sync::RwLock;
31use tokio::time::Duration;
32use tracing::{debug, error, info, trace};
33
34use std::sync::OnceLock;
35
36const DEFAULT_IDM_RECONCILE_INTERVAL_SECS: u64 = 60;
37
38static IDM_RECONCILE_INTERVAL: OnceLock<Duration> = OnceLock::new();
39
40pub fn set_idm_reconcile_interval(duration: Duration) {
41    let _ = IDM_RECONCILE_INTERVAL.set(duration);
42}
43
44pub fn idm_reconcile_interval() -> Duration {
45    *IDM_RECONCILE_INTERVAL.get_or_init(|| Duration::from_secs(DEFAULT_IDM_RECONCILE_INTERVAL_SECS))
46}
47pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
48pub const RELOAD_BUFFER_SIZE: usize = 16;
49pub const NAME_LABEL: &str = "app.kubernetes.io/name";
50pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
51pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
52
53pub type ControllerId = &'static str;
54
55/// State shared between the controller and the web server
56// Kanidm defined as a generic because it causes a cycle dependency with the kaniop_kanidm crate
57#[derive(Clone)]
58pub struct State {
59    /// Metrics
60    metrics: Arc<metrics::Metrics>,
61    /// Shared Kanidm cache clients with the ability to manage users and their groups
62    idm_clients: Arc<RwLock<KanidmClients>>,
63    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
64    /// database and service
65    system_clients: Arc<RwLock<KanidmClients>>,
66    /// Cache for Namespace resources
67    pub namespace_store: Store<Namespace>,
68    /// Cache for Kanidm resources
69    pub kanidm_store: Store<Kanidm>,
70    /// Kubernetes client
71    pub client: Option<Client>,
72}
73
74/// Shared state for a resource stream
75pub struct ResourceReflector<K>
76where
77    K: Resource + Lookup + Clone + 'static,
78    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
79{
80    pub store: Store<K>,
81    pub writer: Writer<K>,
82    pub subscriber: ReflectHandle<K>,
83}
84
85/// State wrapper around the controller outputs for the web server
86impl State {
87    pub fn new(
88        metrics: metrics::Metrics,
89        namespace_store: Store<Namespace>,
90        kanidm_store: Store<Kanidm>,
91        client: Option<Client>,
92    ) -> Self {
93        Self {
94            metrics: Arc::new(metrics),
95            idm_clients: Arc::default(),
96            system_clients: Arc::default(),
97            namespace_store,
98            kanidm_store,
99            client,
100        }
101    }
102
103    /// Metrics getter
104    pub fn metrics(&self) -> Result<String> {
105        prometheus_exporter::format_prometheus_metrics("kaniop").map_err(|e| {
106            Error::FormattingError(format!("failed to export metrics: {}", e), std::fmt::Error)
107        })
108    }
109
110    /// Create a Controller Context that can update State
111    pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
112    where
113        K: Resource + Lookup + Clone + 'static,
114        <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
115    {
116        Context::new(
117            controller_id,
118            client.clone(),
119            self.metrics
120                .controllers
121                .get(controller_id)
122                .expect("all CONTROLLER_IDs have to be registered")
123                .clone(),
124            Recorder::new(client.clone(), controller_id.into()),
125            self.idm_clients.clone(),
126            self.system_clients.clone(),
127            self.namespace_store.clone(),
128            self.kanidm_store.clone(),
129        )
130    }
131}
132
133pub async fn check_api_queryable<K>(client: Client) -> Api<K>
134where
135    K: Resource + Clone + DeserializeOwned + Debug,
136    <K as Resource>::DynamicType: Default,
137{
138    let api = Api::<K>::all(client.clone());
139    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
140        error!(
141            "{} is not queryable; {e:?}. Check controller permissions",
142            short_type_name::<K>().unwrap_or("Unknown resource"),
143        );
144        std::process::exit(1);
145    }
146    api
147}
148
149pub async fn check_api_queryable_optional<K>(client: Client) -> Option<Api<K>>
150where
151    K: Resource + Clone + DeserializeOwned + Debug,
152    <K as Resource>::DynamicType: Default,
153{
154    let api = Api::<K>::all(client.clone());
155    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
156        info!(
157            "{} is not queryable (optional resource); {e:?}. Skipping optional resource support.",
158            short_type_name::<K>().unwrap_or("Unknown resource"),
159        );
160        None
161    } else {
162        Some(api)
163    }
164}
165
166pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
167where
168    K: Resource + Lookup + Clone + 'static,
169    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
170    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
171{
172    let (store, writer) = reflector::store_shared::<K>(buffer_size);
173    let subscriber = writer
174        .subscribe()
175        .expect("subscribers can only be created from shared stores");
176
177    ResourceReflector {
178        store,
179        writer,
180        subscriber,
181    }
182}
183
184fn create_generic_watcher<K, W, T, S, StreamT>(
185    api: Api<K>,
186    writer: Writer<W>,
187    reload_tx: mpsc::Sender<()>,
188    controller_id: ControllerId,
189    ctx: Arc<Context<T>>,
190    stream_fn: S,
191) -> BoxFuture<'static, ()>
192where
193    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
194    W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
195    <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
196    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
197    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
198    <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
199    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
200    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
201    S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
202    StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
203        + Send
204        + 'static,
205{
206    let resource_name = short_type_name::<K>().unwrap_or("Unknown");
207
208    stream_fn(
209        api,
210        watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
211    )
212    .default_backoff()
213    .reflect_shared(writer)
214    .for_each(move |res| {
215        let mut reload_tx_clone = reload_tx.clone();
216        let ctx = ctx.clone();
217        async move {
218            match res {
219                Ok(event) => {
220                    trace!(msg = "watched event", ?event);
221                    match event {
222                        watcher::Event::Delete(d) => {
223                            debug!(
224                                msg = format!("delete event for {resource_name} trigger reconcile"),
225                                namespace = ResourceExt::namespace(&d).unwrap(),
226                                name = d.name_any()
227                            );
228
229                            // TODO: remove for each trigger on delete logic when
230                            // (dispatch delete events issue)[https://github.com/kube-rs/kube/issues/1590]
231                            // is solved
232                            let _ignore_errors = reload_tx_clone.try_send(()).map_err(
233                                |e| error!(msg = "failed to trigger reconcile on delete", %e),
234                            );
235                            ctx.metrics
236                                .triggered_inc(metrics::Action::Delete, resource_name);
237                        }
238                        watcher::Event::Apply(d) => {
239                            debug!(
240                                msg = format!("apply event for {resource_name} trigger reconcile"),
241                                namespace = ResourceExt::namespace(&d).unwrap(),
242                                name = d.name_any()
243                            );
244                            ctx.metrics
245                                .triggered_inc(metrics::Action::Apply, resource_name);
246                        }
247                        _ => {}
248                    }
249                }
250                Err(e) => {
251                    error!(msg = format!("unexpected error when watching {resource_name}"), %e);
252                    ctx.metrics.watch_operations_failed_inc();
253                }
254            }
255        }
256    })
257    .boxed()
258}
259
260pub fn create_watcher<K, T>(
261    api: Api<K>,
262    writer: Writer<K>,
263    reload_tx: mpsc::Sender<()>,
264    controller_id: ControllerId,
265    ctx: Arc<Context<T>>,
266) -> BoxFuture<'static, ()>
267where
268    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
269    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
270    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
271    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
272    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
273{
274    create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
275}
276
277pub fn create_metadata_watcher<K, T>(
278    api: Api<K>,
279    writer: Writer<PartialObjectMeta<K>>,
280    reload_tx: mpsc::Sender<()>,
281    controller_id: ControllerId,
282    ctx: Arc<Context<T>>,
283) -> BoxFuture<'static, ()>
284where
285    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
286    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
287    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
288    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
289    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
290{
291    create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
292        api,
293        writer,
294        reload_tx,
295        controller_id,
296        ctx,
297        metadata_watcher,
298    )
299}
300
301pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
302where
303    K: Resource + Lookup + Clone + 'static,
304    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
305{
306    unreachable!("Handle in backoff_reconciler macro")
307}
308
309#[macro_export]
310macro_rules! backoff_reconciler {
311    ($inner_reconciler:ident) => {
312        |obj, ctx| async move {
313            use $crate::controller::context::BackoffContext;
314            match $inner_reconciler(obj.clone(), ctx.clone()).await {
315                Ok(action) => {
316                    ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
317                        .await;
318                    Ok(action)
319                }
320                Err(error) => {
321                    // safe unwrap: all resources in the operator are namespace scoped resources
322                    let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
323                    let name = kube::ResourceExt::name_any(obj.as_ref());
324                    tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
325                    ctx.metrics().reconcile_failure_inc();
326                    let backoff_duration = ctx
327                        .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
328                        .await;
329                    tracing::trace!(
330                        msg = format!("backoff duration: {backoff_duration:?}"),
331                        %namespace,
332                        %name,
333                    );
334                    Ok(kube::runtime::controller::Action::requeue(backoff_duration))
335                }
336            }
337        }
338    };
339}