kaniop_operator/controller/
mod.rs

1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::error::{Error, Result};
7use crate::kanidm::crd::Kanidm;
8use crate::metrics;
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::fmt::Debug;
13use std::sync::Arc;
14
15use futures::channel::mpsc;
16use futures::future::BoxFuture;
17use futures::{FutureExt, StreamExt};
18use k8s_openapi::api::core::v1::Namespace;
19use kube::Resource;
20use kube::api::{Api, ListParams, ResourceExt};
21use kube::client::Client;
22use kube::runtime::controller::Action;
23use kube::runtime::events::Recorder;
24use kube::runtime::reflector::store::Writer;
25use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
26use kube::runtime::{WatchStreamExt, watcher};
27use prometheus_client::registry::Registry;
28use serde::de::DeserializeOwned;
29use tokio::sync::RwLock;
30use tokio::time::Duration;
31use tracing::{debug, error, trace};
32
33pub const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(5 * 60);
34pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
35pub const RELOAD_BUFFER_SIZE: usize = 16;
36pub const NAME_LABEL: &str = "app.kubernetes.io/name";
37pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
38pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
39
40pub type ControllerId = &'static str;
41
42/// State shared between the controller and the web server
43// Kanidm defined as a generic because it causes a cycle dependency with the kaniop_kanidm crate
44#[derive(Clone)]
45pub struct State {
46    /// Metrics
47    metrics: Arc<metrics::Metrics>,
48    /// Shared Kanidm cache clients with the ability to manage users and their groups
49    idm_clients: Arc<RwLock<KanidmClients>>,
50    /// Shared Kanidm cache clients with the ability to manage the operation of Kanidm as a
51    /// database and service
52    system_clients: Arc<RwLock<KanidmClients>>,
53    /// Cache for Namespace resources
54    pub namespace_store: Store<Namespace>,
55    /// Cache for Kanidm resources
56    pub kanidm_store: Store<Kanidm>,
57}
58
59/// Shared state for a resource stream
60pub struct ResourceReflector<K>
61where
62    K: Resource + Lookup + Clone + 'static,
63    <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
64{
65    pub store: Store<K>,
66    pub writer: Writer<K>,
67    pub subscriber: ReflectHandle<K>,
68}
69
70/// State wrapper around the controller outputs for the web server
71impl State {
72    pub fn new(
73        registry: Registry,
74        controller_names: &[&'static str],
75        namespace_store: Store<Namespace>,
76        kanidm_store: Store<Kanidm>,
77    ) -> Self {
78        Self {
79            metrics: Arc::new(metrics::Metrics::new(registry, controller_names)),
80            idm_clients: Arc::default(),
81            system_clients: Arc::default(),
82            namespace_store,
83            kanidm_store,
84        }
85    }
86
87    /// Metrics getter
88    pub fn metrics(&self) -> Result<String> {
89        let mut buffer = String::new();
90        let registry = &*self.metrics.registry;
91        prometheus_client::encoding::text::encode(&mut buffer, registry)
92            .map_err(|e| Error::FormattingError("failed to encode metrics".to_string(), e))?;
93        Ok(buffer)
94    }
95
96    /// Create a Controller Context that can update State
97    pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
98    where
99        K: Resource + Lookup + Clone + 'static,
100        <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
101    {
102        Context::new(
103            controller_id,
104            client.clone(),
105            self.metrics
106                .controllers
107                .get(controller_id)
108                .expect("all CONTROLLER_IDs have to be registered")
109                .clone(),
110            Recorder::new(client.clone(), controller_id.into()),
111            self.idm_clients.clone(),
112            self.system_clients.clone(),
113            self.namespace_store.clone(),
114            self.kanidm_store.clone(),
115        )
116    }
117}
118
119pub async fn check_api_queryable<K>(client: Client) -> Api<K>
120where
121    K: Resource + Clone + DeserializeOwned + Debug,
122    <K as Resource>::DynamicType: Default,
123{
124    let api = Api::<K>::all(client.clone());
125    if let Err(e) = api.list(&ListParams::default().limit(1)).await {
126        error!(
127            "{} is not queryable; {e:?}. Check controller permissions",
128            short_type_name::<K>().unwrap_or("Unknown resource"),
129        );
130        std::process::exit(1);
131    }
132    api
133}
134
135pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
136where
137    K: Resource + Lookup + Clone + 'static,
138    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
139{
140    let (store, writer) = reflector::store_shared(buffer_size);
141    let subscriber = writer
142        .subscribe()
143        .expect("subscribers can only be created from shared stores");
144
145    ResourceReflector {
146        store,
147        writer,
148        subscriber,
149    }
150}
151
152pub fn create_watcher<K, T>(
153    api: Api<K>,
154    writer: Writer<K>,
155    reload_tx: mpsc::Sender<()>,
156    controller_id: ControllerId,
157    ctx: Arc<Context<T>>,
158) -> BoxFuture<'static, ()>
159where
160    K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
161    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
162    <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
163    T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
164    <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
165{
166    let resource_name = short_type_name::<K>().unwrap_or("Unknown");
167
168    watcher(
169        api,
170        watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
171    )
172    .default_backoff()
173    .reflect_shared(writer)
174    .for_each(move |res| {
175        let mut reload_tx_clone = reload_tx.clone();
176        let ctx = ctx.clone();
177        async move {
178            match res {
179                Ok(event) => {
180                    trace!(msg = "watched event", ?event);
181                    match event {
182                        watcher::Event::Delete(d) => {
183                            debug!(
184                                msg = format!("delete event for {resource_name} trigger reconcile"),
185                                namespace = ResourceExt::namespace(&d).unwrap(),
186                                name = d.name_any()
187                            );
188
189                            // TODO: remove for each trigger on delete logic when
190                            // (dispatch delete events issue)[https://github.com/kube-rs/kube/issues/1590]
191                            // is solved
192                            let _ignore_errors = reload_tx_clone.try_send(()).map_err(
193                                |e| error!(msg = "failed to trigger reconcile on delete", %e),
194                            );
195                            ctx.metrics
196                                .triggered_inc(metrics::Action::Delete, resource_name);
197                        }
198                        watcher::Event::Apply(d) => {
199                            debug!(
200                                msg = format!("apply event for {resource_name} trigger reconcile"),
201                                namespace = ResourceExt::namespace(&d).unwrap(),
202                                name = d.name_any()
203                            );
204                            ctx.metrics
205                                .triggered_inc(metrics::Action::Apply, resource_name);
206                        }
207                        _ => {}
208                    }
209                }
210                Err(e) => {
211                    error!(msg = format!("unexpected error when watching {resource_name}"), %e);
212                    ctx.metrics.watch_operations_failed_inc();
213                }
214            }
215        }
216    })
217    .boxed()
218}
219
220pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
221where
222    K: Resource + Lookup + Clone + 'static,
223    <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
224{
225    unreachable!("Handle in backoff_reconciler macro")
226}
227
228#[macro_export]
229macro_rules! backoff_reconciler {
230    ($inner_reconciler:ident) => {
231        |obj, ctx| async move {
232            use $crate::controller::context::BackoffContext;
233            match $inner_reconciler(obj.clone(), ctx.clone()).await {
234                Ok(action) => {
235                    ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
236                        .await;
237                    Ok(action)
238                }
239                Err(error) => {
240                    // safe unwrap: all resources in the operator are namespace scoped resources
241                    let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
242                    let name = kube::ResourceExt::name_any(obj.as_ref());
243                    tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
244                    ctx.metrics().reconcile_failure_inc();
245                    let backoff_duration = ctx
246                        .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
247                        .await;
248                    tracing::trace!(
249                        msg = format!("backoff duration: {backoff_duration:?}"),
250                        %namespace,
251                        %name,
252                    );
253                    Ok(kube::runtime::controller::Action::requeue(backoff_duration))
254                }
255            }
256        }
257    };
258}