kaniop_operator/
leader_election.rs1use jiff::Timestamp;
2use k8s_openapi::api::coordination::v1::Lease;
3use k8s_openapi::apimachinery::pkg::apis::meta::v1::MicroTime;
4use kube::Client;
5use kube::api::{Api, Patch, PatchParams, PostParams};
6use std::time::Duration;
7use tokio::time::sleep;
8use tracing::{debug, info, warn};
9
10#[derive(Clone)]
11pub struct LeaseLock {
12 client: Client,
13 namespace: String,
14 params: LeaseLockParams,
15}
16
17#[derive(Clone)]
18pub struct LeaseLockParams {
19 pub holder_id: String,
20 pub lease_name: String,
21 pub lease_ttl: Duration,
22}
23
24pub struct LeaseLockResult {
25 pub acquired_lease: bool,
26}
27
28impl LeaseLock {
29 pub fn new(client: Client, namespace: &str, params: LeaseLockParams) -> Self {
30 Self {
31 client,
32 namespace: namespace.to_string(),
33 params,
34 }
35 }
36
37 pub async fn try_acquire_or_renew(&self) -> anyhow::Result<LeaseLockResult> {
38 let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
39 let ttl_seconds = self.params.lease_ttl.as_secs() as i32;
40
41 let now = Timestamp::now();
42 let now_micros = MicroTime(now);
43
44 let lease = leases.get(&self.params.lease_name).await;
45
46 match lease {
47 Ok(existing) => {
48 let holder = existing
49 .spec
50 .as_ref()
51 .and_then(|s| s.holder_identity.as_ref());
52 let lease_ttl = existing
53 .spec
54 .as_ref()
55 .and_then(|s| s.lease_duration_seconds);
56
57 let is_current_holder =
58 holder.map(|h| h == &self.params.holder_id).unwrap_or(false);
59 let lease_expired = lease_ttl
60 .map(|ttl| {
61 let renew_time = existing.spec.as_ref().and_then(|s| s.renew_time.as_ref());
62 if let Some(renew) = renew_time {
63 let renew_ts = renew.0;
64 let elapsed = now - renew_ts;
65 elapsed.total(jiff::Unit::Second).unwrap_or(0.0) > (ttl as f64)
66 } else {
67 true
68 }
69 })
70 .unwrap_or(true);
71
72 if is_current_holder || lease_expired || holder.is_none() {
73 debug!(
74 lease_name = %self.params.lease_name,
75 holder_id = %self.params.holder_id,
76 is_current_holder = is_current_holder,
77 lease_expired = lease_expired,
78 "attempting to acquire/renew lease"
79 );
80
81 let acquire_time = if is_current_holder {
82 existing.spec.as_ref().and_then(|s| s.acquire_time.clone())
83 } else {
84 Some(now_micros.clone())
85 };
86
87 let patch = Lease {
88 metadata: kube::api::ObjectMeta {
89 name: Some(self.params.lease_name.clone()),
90 namespace: Some(self.namespace.clone()),
91 ..Default::default()
92 },
93 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
94 holder_identity: Some(self.params.holder_id.clone()),
95 lease_duration_seconds: Some(ttl_seconds),
96 acquire_time,
97 renew_time: Some(now_micros),
98 ..Default::default()
99 }),
100 };
101
102 let params = PatchParams::default();
103 leases
104 .patch(&self.params.lease_name, ¶ms, &Patch::Merge(&patch))
105 .await?;
106
107 info!(
108 lease_name = %self.params.lease_name,
109 holder_id = %self.params.holder_id,
110 "acquired/renewed lease"
111 );
112
113 Ok(LeaseLockResult {
114 acquired_lease: true,
115 })
116 } else {
117 debug!(
118 lease_name = %self.params.lease_name,
119 current_holder = ?holder,
120 "lease held by another instance"
121 );
122 Ok(LeaseLockResult {
123 acquired_lease: false,
124 })
125 }
126 }
127 Err(e) => {
128 if let kube::Error::Api(api_err) = &e {
129 if api_err.code == 404 {
130 debug!(
131 lease_name = %self.params.lease_name,
132 "lease does not exist, creating"
133 );
134
135 let new_lease = Lease {
136 metadata: kube::api::ObjectMeta {
137 name: Some(self.params.lease_name.clone()),
138 namespace: Some(self.namespace.clone()),
139 ..Default::default()
140 },
141 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
142 holder_identity: Some(self.params.holder_id.clone()),
143 lease_duration_seconds: Some(ttl_seconds),
144 acquire_time: Some(now_micros.clone()),
145 renew_time: Some(now_micros),
146 ..Default::default()
147 }),
148 };
149
150 let params = PostParams::default();
151 leases.create(¶ms, &new_lease).await?;
152
153 info!(
154 lease_name = %self.params.lease_name,
155 holder_id = %self.params.holder_id,
156 "created and acquired lease"
157 );
158
159 return Ok(LeaseLockResult {
160 acquired_lease: true,
161 });
162 }
163 }
164 warn!(
165 lease_name = %self.params.lease_name,
166 error = %e,
167 "failed to get lease"
168 );
169 Err(e.into())
170 }
171 }
172 }
173
174 pub async fn step_down(&self) -> anyhow::Result<()> {
175 let leases: Api<Lease> = Api::namespaced(self.client.clone(), &self.namespace);
176
177 let lease = leases.get(&self.params.lease_name).await?;
178
179 let holder = lease.spec.as_ref().and_then(|s| s.holder_identity.as_ref());
180
181 if holder.map(|h| h == &self.params.holder_id).unwrap_or(false) {
182 debug!(
183 lease_name = %self.params.lease_name,
184 holder_id = %self.params.holder_id,
185 "stepping down from leadership"
186 );
187
188 let patch = Lease {
189 metadata: kube::api::ObjectMeta {
190 name: Some(self.params.lease_name.clone()),
191 namespace: Some(self.namespace.clone()),
192 ..Default::default()
193 },
194 spec: Some(k8s_openapi::api::coordination::v1::LeaseSpec {
195 lease_duration_seconds: Some(1),
196 holder_identity: None,
197 ..Default::default()
198 }),
199 };
200
201 let params = PatchParams::default();
202 leases
203 .patch(&self.params.lease_name, ¶ms, &Patch::Merge(&patch))
204 .await?;
205
206 info!(
207 lease_name = %self.params.lease_name,
208 "stepped down from leadership"
209 );
210 }
211
212 Ok(())
213 }
214}
215
216pub async fn acquire_lease_with_retry(
217 lease_lock: &LeaseLock,
218 retry_interval: Duration,
219 max_retries: Option<u32>,
220) -> anyhow::Result<()> {
221 let mut retry_count = 0u32;
222
223 loop {
224 if let Some(max) = max_retries {
225 if retry_count >= max {
226 return Err(anyhow::anyhow!(
227 "failed to acquire lease after {} retries",
228 max
229 ));
230 }
231 }
232
233 let result = lease_lock.try_acquire_or_renew().await?;
234
235 if result.acquired_lease {
236 return Ok(());
237 }
238
239 retry_count += 1;
240 sleep(retry_interval).await;
241 }
242}