kaniop_operator/controller/
mod.rs1pub mod context;
2pub mod kanidm;
3
4use self::{context::Context, kanidm::KanidmClients};
5
6use crate::kanidm::crd::Kanidm;
7use crate::metrics;
8use kaniop_k8s_util::error::{Error, Result};
9
10use kaniop_k8s_util::types::short_type_name;
11
12use std::fmt::Debug;
13use std::sync::Arc;
14
15use futures::channel::mpsc;
16use futures::future::BoxFuture;
17use futures::{FutureExt, StreamExt};
18use k8s_openapi::api::core::v1::Namespace;
19use kube::Resource;
20use kube::api::{Api, ListParams, PartialObjectMeta, ResourceExt};
21use kube::client::Client;
22use kube::runtime::controller::Action;
23use kube::runtime::events::Recorder;
24use kube::runtime::reflector::store::Writer;
25use kube::runtime::reflector::{self, Lookup, ReflectHandle, Store};
26use kube::runtime::{WatchStreamExt, metadata_watcher, watcher};
27use prometheus_client::registry::Registry;
28use serde::de::DeserializeOwned;
29use tokio::sync::RwLock;
30use tokio::time::Duration;
31use tracing::{debug, error, trace};
32
33pub const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(60);
34pub const SUBSCRIBE_BUFFER_SIZE: usize = 256;
35pub const RELOAD_BUFFER_SIZE: usize = 16;
36pub const NAME_LABEL: &str = "app.kubernetes.io/name";
37pub const INSTANCE_LABEL: &str = "app.kubernetes.io/instance";
38pub const MANAGED_BY_LABEL: &str = "app.kubernetes.io/managed-by";
39
40pub type ControllerId = &'static str;
41
42#[derive(Clone)]
45pub struct State {
46 metrics: Arc<metrics::Metrics>,
48 idm_clients: Arc<RwLock<KanidmClients>>,
50 system_clients: Arc<RwLock<KanidmClients>>,
53 pub namespace_store: Store<Namespace>,
55 pub kanidm_store: Store<Kanidm>,
57}
58
59pub struct ResourceReflector<K>
61where
62 K: Resource + Lookup + Clone + 'static,
63 <K as Lookup>::DynamicType: Eq + std::hash::Hash + Clone,
64{
65 pub store: Store<K>,
66 pub writer: Writer<K>,
67 pub subscriber: ReflectHandle<K>,
68}
69
70impl State {
72 pub fn new(
73 registry: Registry,
74 controller_names: &[&'static str],
75 namespace_store: Store<Namespace>,
76 kanidm_store: Store<Kanidm>,
77 ) -> Self {
78 Self {
79 metrics: Arc::new(metrics::Metrics::new(registry, controller_names)),
80 idm_clients: Arc::default(),
81 system_clients: Arc::default(),
82 namespace_store,
83 kanidm_store,
84 }
85 }
86
87 pub fn metrics(&self) -> Result<String> {
89 let mut buffer = String::new();
90 let registry = &*self.metrics.registry;
91 prometheus_client::encoding::text::encode(&mut buffer, registry)
92 .map_err(|e| Error::FormattingError("failed to encode metrics".to_string(), e))?;
93 Ok(buffer)
94 }
95
96 pub fn to_context<K>(&self, client: Client, controller_id: ControllerId) -> Context<K>
98 where
99 K: Resource + Lookup + Clone + 'static,
100 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
101 {
102 Context::new(
103 controller_id,
104 client.clone(),
105 self.metrics
106 .controllers
107 .get(controller_id)
108 .expect("all CONTROLLER_IDs have to be registered")
109 .clone(),
110 Recorder::new(client.clone(), controller_id.into()),
111 self.idm_clients.clone(),
112 self.system_clients.clone(),
113 self.namespace_store.clone(),
114 self.kanidm_store.clone(),
115 )
116 }
117}
118
119pub async fn check_api_queryable<K>(client: Client) -> Api<K>
120where
121 K: Resource + Clone + DeserializeOwned + Debug,
122 <K as Resource>::DynamicType: Default,
123{
124 let api = Api::<K>::all(client.clone());
125 if let Err(e) = api.list(&ListParams::default().limit(1)).await {
126 error!(
127 "{} is not queryable; {e:?}. Check controller permissions",
128 short_type_name::<K>().unwrap_or("Unknown resource"),
129 );
130 std::process::exit(1);
131 }
132 api
133}
134
135pub fn create_subscriber<K>(buffer_size: usize) -> ResourceReflector<K>
136where
137 K: Resource + Lookup + Clone + 'static,
138 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
139 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
140{
141 let (store, writer) = reflector::store_shared::<K>(buffer_size);
142 let subscriber = writer
143 .subscribe()
144 .expect("subscribers can only be created from shared stores");
145
146 ResourceReflector {
147 store,
148 writer,
149 subscriber,
150 }
151}
152
153fn create_generic_watcher<K, W, T, S, StreamT>(
154 api: Api<K>,
155 writer: Writer<W>,
156 reload_tx: mpsc::Sender<()>,
157 controller_id: ControllerId,
158 ctx: Arc<Context<T>>,
159 stream_fn: S,
160) -> BoxFuture<'static, ()>
161where
162 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
163 W: Resource + ResourceExt + Lookup + Clone + Debug + Send + Sync + 'static,
164 <W as Resource>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
165 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
166 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
167 <W as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
168 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
169 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
170 S: Fn(Api<K>, watcher::Config) -> StreamT + 'static,
171 StreamT: futures::Stream<Item = Result<watcher::Event<W>, kube::runtime::watcher::Error>>
172 + Send
173 + 'static,
174{
175 let resource_name = short_type_name::<K>().unwrap_or("Unknown");
176
177 stream_fn(
178 api,
179 watcher::Config::default().labels(&format!("{MANAGED_BY_LABEL}=kaniop-{controller_id}")),
180 )
181 .default_backoff()
182 .reflect_shared(writer)
183 .for_each(move |res| {
184 let mut reload_tx_clone = reload_tx.clone();
185 let ctx = ctx.clone();
186 async move {
187 match res {
188 Ok(event) => {
189 trace!(msg = "watched event", ?event);
190 match event {
191 watcher::Event::Delete(d) => {
192 debug!(
193 msg = format!("delete event for {resource_name} trigger reconcile"),
194 namespace = ResourceExt::namespace(&d).unwrap(),
195 name = d.name_any()
196 );
197
198 let _ignore_errors = reload_tx_clone.try_send(()).map_err(
202 |e| error!(msg = "failed to trigger reconcile on delete", %e),
203 );
204 ctx.metrics
205 .triggered_inc(metrics::Action::Delete, resource_name);
206 }
207 watcher::Event::Apply(d) => {
208 debug!(
209 msg = format!("apply event for {resource_name} trigger reconcile"),
210 namespace = ResourceExt::namespace(&d).unwrap(),
211 name = d.name_any()
212 );
213 ctx.metrics
214 .triggered_inc(metrics::Action::Apply, resource_name);
215 }
216 _ => {}
217 }
218 }
219 Err(e) => {
220 error!(msg = format!("unexpected error when watching {resource_name}"), %e);
221 ctx.metrics.watch_operations_failed_inc();
222 }
223 }
224 }
225 })
226 .boxed()
227}
228
229pub fn create_watcher<K, T>(
230 api: Api<K>,
231 writer: Writer<K>,
232 reload_tx: mpsc::Sender<()>,
233 controller_id: ControllerId,
234 ctx: Arc<Context<T>>,
235) -> BoxFuture<'static, ()>
236where
237 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
238 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
239 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
240 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
241 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
242{
243 create_generic_watcher::<K, K, T, _, _>(api, writer, reload_tx, controller_id, ctx, watcher)
244}
245
246pub fn create_metadata_watcher<K, T>(
247 api: Api<K>,
248 writer: Writer<PartialObjectMeta<K>>,
249 reload_tx: mpsc::Sender<()>,
250 controller_id: ControllerId,
251 ctx: Arc<Context<T>>,
252) -> BoxFuture<'static, ()>
253where
254 K: Resource + Lookup + Clone + DeserializeOwned + Send + Sync + Debug + 'static,
255 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone + Send + Sync,
256 <K as Resource>::DynamicType: Default + Eq + std::hash::Hash + Clone,
257 T: Resource<DynamicType = ()> + ResourceExt + Lookup + Clone + 'static,
258 <T as Lookup>::DynamicType: Eq + std::hash::Hash + Clone + Send + Sync,
259{
260 create_generic_watcher::<K, PartialObjectMeta<K>, T, _, _>(
261 api,
262 writer,
263 reload_tx,
264 controller_id,
265 ctx,
266 metadata_watcher,
267 )
268}
269
270pub fn error_policy<K>(_obj: Arc<K>, _error: &Error, _ctx: Arc<Context<K>>) -> Action
271where
272 K: Resource + Lookup + Clone + 'static,
273 <K as Lookup>::DynamicType: Default + Eq + std::hash::Hash + Clone,
274{
275 unreachable!("Handle in backoff_reconciler macro")
276}
277
278#[macro_export]
279macro_rules! backoff_reconciler {
280 ($inner_reconciler:ident) => {
281 |obj, ctx| async move {
282 use $crate::controller::context::BackoffContext;
283 match $inner_reconciler(obj.clone(), ctx.clone()).await {
284 Ok(action) => {
285 ctx.reset_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
286 .await;
287 Ok(action)
288 }
289 Err(error) => {
290 let namespace = kube::ResourceExt::namespace(obj.as_ref()).unwrap();
292 let name = kube::ResourceExt::name_any(obj.as_ref());
293 tracing::error!(msg = "failed reconciliation", %namespace, %name, %error);
294 ctx.metrics().reconcile_failure_inc();
295 let backoff_duration = ctx
296 .get_backoff(kube::runtime::reflector::ObjectRef::from(obj.as_ref()))
297 .await;
298 tracing::trace!(
299 msg = format!("backoff duration: {backoff_duration:?}"),
300 %namespace,
301 %name,
302 );
303 Ok(kube::runtime::controller::Action::requeue(backoff_duration))
304 }
305 }
306 }
307 };
308}