kaniop_operator/
leader_election.rs1use k8s_openapi::api::coordination::v1::Lease;
2use k8s_openapi::apimachinery::pkg::apis::meta::v1::MicroTime;
3use k8s_openapi::jiff::Timestamp;
4use kube::Client;
5use kube::api::{Api, Patch, PatchParams, PostParams};
6use std::time::Duration;
7use tokio::time::sleep;
8use tracing::{debug, info, warn};
9
10#[derive(Clone)]
11pub struct LeaseLock {
12 client: Client,
13 namespace: String,
14 params: LeaseLockParams,
15}
16
17#[derive(Clone)]
18pub struct LeaseLockParams {
19 pub holder_id: String,
20 pub lease_name: String,
21 pub lease_ttl: Duration,
22}
23
24pub struct LeaseLockResult {
25 pub acquired_lease: bool,
26}
27
28impl LeaseLock {
29 pub fn new(client: Client, namespace: &str, params: LeaseLockParams) -> Self {
30 Self {
31 client,
32 namespace: namespace.to_string(),
33 params,
34 }
35 }
36
37 pub async fn try_acquire_or_renew(&self) -> anyhow::Result<LeaseLockResult> {
38 let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
39 let ttl_seconds = self.params.lease_ttl.as_secs() as i32;
40
41 let now = Timestamp::now();
42 let now_micros = MicroTime(now);
43
44 let lease = leases.get(&self.params.lease_name).await;
45
46 match lease {
47 Ok(existing) => {
48 let holder = existing
49 .spec
50 .as_ref()
51 .and_then(|s| s.holder_identity.as_ref());
52 let lease_ttl = existing
53 .spec
54 .as_ref()
55 .and_then(|s| s.lease_duration_seconds);
56
57 let is_current_holder =
58 holder.map(|h| h == &self.params.holder_id).unwrap_or(false);
59 let lease_expired = lease_ttl
60 .map(|ttl| {
61 let renew_time = existing.spec.as_ref().and_then(|s| s.renew_time.as_ref());
62 if let Some(renew) = renew_time {
63 let renew_ts = renew.0;
64 let elapsed = now - renew_ts;
65 elapsed
66 .total(k8s_openapi::jiff::Unit::Second)
67 .unwrap_or(0.0)
68 > (ttl as f64)
69 } else {
70 true
71 }
72 })
73 .unwrap_or(true);
74
75 if is_current_holder || lease_expired || holder.is_none() {
76 debug!(
77 lease_name = %self.params.lease_name,
78 holder_id = %self.params.holder_id,
79 is_current_holder = is_current_holder,
80 lease_expired = lease_expired,
81 "attempting to acquire/renew lease"
82 );
83
84 let acquire_time = if is_current_holder {
85 existing.spec.as_ref().and_then(|s| s.acquire_time.clone())
86 } else {
87 Some(now_micros.clone())
88 };
89
90 let patch = Lease {
91 metadata: kube::api::ObjectMeta {
92 name: Some(self.params.lease_name.clone()),
93 namespace: Some(self.namespace.clone()),
94 ..Default::default()
95 },
96 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
97 holder_identity: Some(self.params.holder_id.clone()),
98 lease_duration_seconds: Some(ttl_seconds),
99 acquire_time,
100 renew_time: Some(now_micros),
101 ..Default::default()
102 }),
103 };
104
105 let params = PatchParams::default();
106 leases
107 .patch(&self.params.lease_name, ¶ms, &Patch::Merge(&patch))
108 .await?;
109
110 info!(
111 lease_name = %self.params.lease_name,
112 holder_id = %self.params.holder_id,
113 "acquired/renewed lease"
114 );
115
116 Ok(LeaseLockResult {
117 acquired_lease: true,
118 })
119 } else {
120 debug!(
121 lease_name = %self.params.lease_name,
122 current_holder = ?holder,
123 "lease held by another instance"
124 );
125 Ok(LeaseLockResult {
126 acquired_lease: false,
127 })
128 }
129 }
130 Err(e) => {
131 if let kube::Error::Api(api_err) = &e {
132 if api_err.code == 404 {
133 debug!(
134 lease_name = %self.params.lease_name,
135 "lease does not exist, creating"
136 );
137
138 let new_lease = Lease {
139 metadata: kube::api::ObjectMeta {
140 name: Some(self.params.lease_name.clone()),
141 namespace: Some(self.namespace.clone()),
142 ..Default::default()
143 },
144 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
145 holder_identity: Some(self.params.holder_id.clone()),
146 lease_duration_seconds: Some(ttl_seconds),
147 acquire_time: Some(now_micros.clone()),
148 renew_time: Some(now_micros),
149 ..Default::default()
150 }),
151 };
152
153 let params = PostParams::default();
154 leases.create(¶ms, &new_lease).await?;
155
156 info!(
157 lease_name = %self.params.lease_name,
158 holder_id = %self.params.holder_id,
159 "created and acquired lease"
160 );
161
162 return Ok(LeaseLockResult {
163 acquired_lease: true,
164 });
165 }
166 }
167 warn!(
168 lease_name = %self.params.lease_name,
169 error = %e,
170 "failed to get lease"
171 );
172 Err(e.into())
173 }
174 }
175 }
176
177 pub async fn step_down(&self) -> anyhow::Result<()> {
178 let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
179
180 let lease = leases.get(&self.params.lease_name).await?;
181
182 let holder = lease.spec.as_ref().and_then(|s| s.holder_identity.as_ref());
183
184 if holder.map(|h| h == &self.params.holder_id).unwrap_or(false) {
185 debug!(
186 lease_name = %self.params.lease_name,
187 holder_id = %self.params.holder_id,
188 "stepping down from leadership"
189 );
190
191 let patch = Lease {
192 metadata: kube::api::ObjectMeta {
193 name: Some(self.params.lease_name.clone()),
194 namespace: Some(self.namespace.clone()),
195 ..Default::default()
196 },
197 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
198 lease_duration_seconds: Some(1),
199 holder_identity: None,
200 ..Default::default()
201 }),
202 };
203
204 let params = PatchParams::default();
205 leases
206 .patch(&self.params.lease_name, ¶ms, &Patch::Merge(&patch))
207 .await?;
208
209 info!(
210 lease_name = %self.params.lease_name,
211 "stepped down from leadership"
212 );
213 }
214
215 Ok(())
216 }
217}
218
219pub async fn acquire_lease_with_retry(
220 lease_lock: &LeaseLock,
221 retry_interval: Duration,
222 max_retries: Option<u32>,
223) -> anyhow::Result<()> {
224 let mut retry_count = 0u32;
225
226 loop {
227 if let Some(max) = max_retries {
228 if retry_count >= max {
229 return Err(anyhow::anyhow!(
230 "failed to acquire lease after {} retries",
231 max
232 ));
233 }
234 }
235
236 let result = lease_lock.try_acquire_or_renew().await?;
237
238 if result.acquired_lease {
239 return Ok(());
240 }
241
242 retry_count += 1;
243 sleep(retry_interval).await;
244 }
245}