1mod events;
2pub use events::ServerBackupEvent;
3
4pub mod retention;
5
6use crate::{
7 jwt::BasePayload,
8 models::{InsertQueryBuilder, UpdateQueryBuilder, server_variable::ServerVariable},
9 prelude::*,
10 storage::StorageUrlRetriever,
11};
12use compact_str::ToCompactString;
13use futures_util::StreamExt;
14use garde::Validate;
15use reqwest::StatusCode;
16use serde::{Deserialize, Serialize};
17use sqlx::{Row, postgres::PgRow, prelude::Type};
18use std::{
19 collections::{BTreeMap, HashMap},
20 sync::{Arc, LazyLock},
21};
22use utoipa::ToSchema;
23
24#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
25#[serde(rename_all = "kebab-case")]
26#[sqlx(type_name = "backup_disk", rename_all = "SCREAMING_SNAKE_CASE")]
27pub enum BackupDisk {
28 Local,
29 S3,
30 DdupBak,
31 Btrfs,
32 Zfs,
33 Restic,
34 ProxmoxBackupServer,
35 Kopia,
36}
37
38impl BackupDisk {
39 #[inline]
40 pub fn from_wings_adapter(adapter: wings_api::BackupAdapter) -> Self {
41 match adapter {
42 wings_api::BackupAdapter::Wings => BackupDisk::Local,
43 wings_api::BackupAdapter::S3 => BackupDisk::S3,
44 wings_api::BackupAdapter::DdupBak => BackupDisk::DdupBak,
45 wings_api::BackupAdapter::Btrfs => BackupDisk::Btrfs,
46 wings_api::BackupAdapter::Zfs => BackupDisk::Zfs,
47 wings_api::BackupAdapter::Restic => BackupDisk::Restic,
48 wings_api::BackupAdapter::ProxmoxBackupServer => BackupDisk::ProxmoxBackupServer,
49 wings_api::BackupAdapter::Kopia => BackupDisk::Kopia,
50 }
51 }
52
53 #[inline]
54 pub fn to_wings_adapter(self) -> wings_api::BackupAdapter {
55 match self {
56 BackupDisk::Local => wings_api::BackupAdapter::Wings,
57 BackupDisk::S3 => wings_api::BackupAdapter::S3,
58 BackupDisk::DdupBak => wings_api::BackupAdapter::DdupBak,
59 BackupDisk::Btrfs => wings_api::BackupAdapter::Btrfs,
60 BackupDisk::Zfs => wings_api::BackupAdapter::Zfs,
61 BackupDisk::Restic => wings_api::BackupAdapter::Restic,
62 BackupDisk::ProxmoxBackupServer => wings_api::BackupAdapter::ProxmoxBackupServer,
63 BackupDisk::Kopia => wings_api::BackupAdapter::Kopia,
64 }
65 }
66}
67
68#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
69#[serde(rename_all = "snake_case")]
70#[sqlx(type_name = "server_backup_kind", rename_all = "SCREAMING_SNAKE_CASE")]
71pub enum ServerBackupKind {
72 Server,
73 DatabaseInstance,
74}
75
76pub struct ServerBackupFilter {
77 pub kind: Option<ServerBackupKind>,
78 pub database_instance_uuid: Option<uuid::Uuid>,
79 pub database_type: Option<db_agent_api::DatabaseAgentType>,
80}
81
82pub struct ServerBackupRestoreOptions {
83 pub truncate_directory: bool,
84 pub restore_startup: bool,
85}
86
87#[derive(Debug, Clone, Copy)]
89enum EvictionScope<'a> {
90 Server,
91 Group(&'a str),
92 Policy(&'a str),
93}
94
95impl<'a> EvictionScope<'a> {
96 #[inline]
97 fn group_name(self) -> Option<&'a str> {
98 match self {
99 Self::Group(name) => Some(name),
100 _ => None,
101 }
102 }
103
104 #[inline]
105 fn policy_name(self) -> Option<&'a str> {
106 match self {
107 Self::Policy(name) => Some(name),
108 _ => None,
109 }
110 }
111}
112
113#[derive(Serialize, Deserialize, Clone)]
114pub struct ServerBackup {
115 pub uuid: uuid::Uuid,
116 pub server: Option<Fetchable<super::server::Server>>,
117 pub node: Fetchable<super::node::Node>,
118 pub backup_configuration: Option<Fetchable<super::backup_configuration::BackupConfiguration>>,
119 pub backup_group_uuid: Option<uuid::Uuid>,
120 pub system_backup_policy_uuid: Option<uuid::Uuid>,
121 pub database_instance_uuid: Option<uuid::Uuid>,
122
123 pub kind: ServerBackupKind,
124 pub database_type: Option<db_agent_api::DatabaseAgentType>,
125 pub name: compact_str::CompactString,
126 pub successful: bool,
127 pub browsable: bool,
128 pub streaming: bool,
129 pub locked: bool,
130 pub shared: bool,
131
132 pub ignored_files: Vec<compact_str::CompactString>,
133 pub checksum: Option<compact_str::CompactString>,
134 pub bytes: i64,
135 pub files: i64,
136
137 pub disk: BackupDisk,
138 pub upload_id: Option<compact_str::CompactString>,
139 pub upload_path: Option<compact_str::CompactString>,
140 pub metadata: serde_json::Value,
141
142 pub completed: Option<chrono::NaiveDateTime>,
143 pub deleting: Option<chrono::NaiveDateTime>,
144 pub deletion_retries: i32,
145 pub deleted: Option<chrono::NaiveDateTime>,
146 pub created: chrono::NaiveDateTime,
147
148 extension_data: super::ModelExtensionData,
149}
150
151#[derive(Debug, ToSchema, Serialize, Clone, Copy, PartialEq, Eq)]
152#[serde(rename_all = "snake_case")]
153pub enum ServerBackupDeletionStatus {
154 Deleting,
155 Failed,
156}
157
158impl BaseModel for ServerBackup {
159 const NAME: &'static str = "server_backup";
160
161 fn get_extension_list() -> &'static super::ModelExtensionList {
162 static EXTENSIONS: LazyLock<super::ModelExtensionList> =
163 LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
164
165 &EXTENSIONS
166 }
167
168 fn get_extension_data(&self) -> &super::ModelExtensionData {
169 &self.extension_data
170 }
171
172 #[inline]
173 fn base_columns(prefix: Option<&str>) -> BTreeMap<&'static str, compact_str::CompactString> {
174 let prefix = prefix.unwrap_or_default();
175
176 BTreeMap::from([
177 (
178 "server_backups.uuid",
179 compact_str::format_compact!("{prefix}uuid"),
180 ),
181 (
182 "server_backups.server_uuid",
183 compact_str::format_compact!("{prefix}server_uuid"),
184 ),
185 (
186 "server_backups.node_uuid",
187 compact_str::format_compact!("{prefix}node_uuid"),
188 ),
189 (
190 "server_backups.backup_configuration_uuid",
191 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
192 ),
193 (
194 "server_backups.backup_group_uuid",
195 compact_str::format_compact!("{prefix}backup_group_uuid"),
196 ),
197 (
198 "server_backups.system_backup_policy_uuid",
199 compact_str::format_compact!("{prefix}system_backup_policy_uuid"),
200 ),
201 (
202 "server_backups.database_instance_uuid",
203 compact_str::format_compact!("{prefix}database_instance_uuid"),
204 ),
205 (
206 "server_backups.kind",
207 compact_str::format_compact!("{prefix}kind"),
208 ),
209 (
210 "server_backups.database_type",
211 compact_str::format_compact!("{prefix}database_type"),
212 ),
213 (
214 "server_backups.name",
215 compact_str::format_compact!("{prefix}name"),
216 ),
217 (
218 "server_backups.successful",
219 compact_str::format_compact!("{prefix}successful"),
220 ),
221 (
222 "server_backups.browsable",
223 compact_str::format_compact!("{prefix}browsable"),
224 ),
225 (
226 "server_backups.streaming",
227 compact_str::format_compact!("{prefix}streaming"),
228 ),
229 (
230 "server_backups.locked",
231 compact_str::format_compact!("{prefix}locked"),
232 ),
233 (
234 "server_backups.shared",
235 compact_str::format_compact!("{prefix}shared"),
236 ),
237 (
238 "server_backups.ignored_files",
239 compact_str::format_compact!("{prefix}ignored_files"),
240 ),
241 (
242 "server_backups.checksum",
243 compact_str::format_compact!("{prefix}checksum"),
244 ),
245 (
246 "server_backups.bytes",
247 compact_str::format_compact!("{prefix}bytes"),
248 ),
249 (
250 "server_backups.files",
251 compact_str::format_compact!("{prefix}files"),
252 ),
253 (
254 "server_backups.disk",
255 compact_str::format_compact!("{prefix}disk"),
256 ),
257 (
258 "server_backups.upload_id",
259 compact_str::format_compact!("{prefix}upload_id"),
260 ),
261 (
262 "server_backups.upload_path",
263 compact_str::format_compact!("{prefix}upload_path"),
264 ),
265 (
266 "server_backups.metadata",
267 compact_str::format_compact!("{prefix}metadata"),
268 ),
269 (
270 "server_backups.completed",
271 compact_str::format_compact!("{prefix}completed"),
272 ),
273 (
274 "server_backups.deleting",
275 compact_str::format_compact!("{prefix}deleting"),
276 ),
277 (
278 "server_backups.deletion_retries",
279 compact_str::format_compact!("{prefix}deletion_retries"),
280 ),
281 (
282 "server_backups.deleted",
283 compact_str::format_compact!("{prefix}deleted"),
284 ),
285 (
286 "server_backups.created",
287 compact_str::format_compact!("{prefix}created"),
288 ),
289 ])
290 }
291
292 #[inline]
293 fn map(prefix: Option<&str>, row: &PgRow) -> Result<Self, crate::database::DatabaseError> {
294 let prefix = prefix.unwrap_or_default();
295
296 Ok(Self {
297 uuid: row.try_get(compact_str::format_compact!("{prefix}uuid").as_str())?,
298 server: super::server::Server::get_fetchable_from_row(
299 row,
300 compact_str::format_compact!("{prefix}server_uuid"),
301 ),
302 backup_configuration:
303 super::backup_configuration::BackupConfiguration::get_fetchable_from_row(
304 row,
305 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
306 ),
307 node: super::node::Node::get_fetchable(
308 row.try_get(compact_str::format_compact!("{prefix}node_uuid").as_str())?,
309 ),
310 backup_group_uuid: row
311 .try_get(compact_str::format_compact!("{prefix}backup_group_uuid").as_str())?,
312 system_backup_policy_uuid: row.try_get(
313 compact_str::format_compact!("{prefix}system_backup_policy_uuid").as_str(),
314 )?,
315 database_instance_uuid: row
316 .try_get(compact_str::format_compact!("{prefix}database_instance_uuid").as_str())?,
317 kind: row.try_get(compact_str::format_compact!("{prefix}kind").as_str())?,
318 database_type: row
319 .try_get(compact_str::format_compact!("{prefix}database_type").as_str())?,
320 name: row.try_get(compact_str::format_compact!("{prefix}name").as_str())?,
321 successful: row.try_get(compact_str::format_compact!("{prefix}successful").as_str())?,
322 browsable: row.try_get(compact_str::format_compact!("{prefix}browsable").as_str())?,
323 streaming: row.try_get(compact_str::format_compact!("{prefix}streaming").as_str())?,
324 locked: row.try_get(compact_str::format_compact!("{prefix}locked").as_str())?,
325 shared: row.try_get(compact_str::format_compact!("{prefix}shared").as_str())?,
326 ignored_files: row
327 .try_get(compact_str::format_compact!("{prefix}ignored_files").as_str())?,
328 checksum: row.try_get(compact_str::format_compact!("{prefix}checksum").as_str())?,
329 bytes: row.try_get(compact_str::format_compact!("{prefix}bytes").as_str())?,
330 files: row.try_get(compact_str::format_compact!("{prefix}files").as_str())?,
331 disk: row.try_get(compact_str::format_compact!("{prefix}disk").as_str())?,
332 upload_id: row.try_get(compact_str::format_compact!("{prefix}upload_id").as_str())?,
333 upload_path: row
334 .try_get(compact_str::format_compact!("{prefix}upload_path").as_str())?,
335 metadata: row.try_get(compact_str::format_compact!("{prefix}metadata").as_str())?,
336 completed: row.try_get(compact_str::format_compact!("{prefix}completed").as_str())?,
337 deleting: row.try_get(compact_str::format_compact!("{prefix}deleting").as_str())?,
338 deletion_retries: row
339 .try_get(compact_str::format_compact!("{prefix}deletion_retries").as_str())?,
340 deleted: row.try_get(compact_str::format_compact!("{prefix}deleted").as_str())?,
341 created: row.try_get(compact_str::format_compact!("{prefix}created").as_str())?,
342 extension_data: Self::map_extensions(prefix, row)?,
343 })
344 }
345}
346
347impl ServerBackup {
348 pub async fn create_raw(
349 state: &crate::State,
350 mut options: CreateServerBackupOptions<'_>,
351 ) -> Result<Self, anyhow::Error> {
352 let backup_configuration = match options.backup_configuration.take() {
353 Some(backup_configuration) => backup_configuration,
354 None => options
355 .server
356 .backup_configuration(&state.database)
357 .await
358 .ok_or_else(|| {
359 crate::response::DisplayError::new(
360 "no backup configuration available, unable to create backup",
361 )
362 .with_status(StatusCode::EXPECTATION_FAILED)
363 })?,
364 };
365
366 if backup_configuration.maintenance_enabled {
367 return Err(crate::response::DisplayError::new(
368 "cannot create backup while backup configuration is in maintenance mode",
369 )
370 .with_status(StatusCode::EXPECTATION_FAILED)
371 .into());
372 }
373
374 if options.database_instance.is_some()
375 && matches!(
376 backup_configuration.backup_disk,
377 BackupDisk::Btrfs | BackupDisk::Zfs
378 )
379 {
380 return Err(crate::response::DisplayError::new(
381 "database backups cannot be created on a btrfs or zfs backup configuration",
382 )
383 .with_status(StatusCode::EXPECTATION_FAILED)
384 .into());
385 }
386
387 let mut transaction = state.database.write().begin().await?;
388
389 let mut query_builder = InsertQueryBuilder::new("server_backups");
390
391 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
392 .await?;
393
394 query_builder
395 .set("server_uuid", options.server.uuid)
396 .set("node_uuid", options.server.node.uuid)
397 .set("backup_configuration_uuid", backup_configuration.uuid)
398 .set("backup_group_uuid", options.backup_group_uuid)
399 .set(
400 "system_backup_policy_uuid",
401 options.system_backup_policy_uuid,
402 )
403 .set(
404 "database_instance_uuid",
405 options.database_instance.map(|instance| instance.uuid),
406 )
407 .set("kind", options.kind())
408 .set(
409 "database_type",
410 options.database_instance.map(|instance| instance.r#type),
411 )
412 .set("name", &options.name)
413 .set("ignored_files", &options.ignored_files)
414 .set("bytes", 0i64)
415 .set("disk", backup_configuration.backup_disk)
416 .set("shared", backup_configuration.shared)
417 .set("metadata", &options.metadata);
418
419 let row = query_builder
420 .returning(&Self::columns_sql(None))
421 .fetch_one(&mut *transaction)
422 .await?;
423 let mut backup = Self::map(None, &row)?;
424
425 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
426
427 transaction.commit().await?;
428
429 Ok(backup)
430 }
431
432 pub async fn by_server_uuid_uuid(
433 database: &crate::database::Database,
434 server_uuid: uuid::Uuid,
435 uuid: uuid::Uuid,
436 ) -> Result<Option<Self>, crate::database::DatabaseError> {
437 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
438 r#"
439 SELECT {}
440 FROM server_backups
441 WHERE server_backups.server_uuid = $1 AND server_backups.uuid = $2
442 "#,
443 Self::columns_sql(None)
444 )))
445 .bind(server_uuid)
446 .bind(uuid)
447 .fetch_optional(database.read())
448 .await?;
449
450 row.try_map(|row| Self::map(None, &row))
451 }
452
453 pub async fn select_completed_by_server_uuid(
454 database: &crate::database::Database,
455 server_uuid: uuid::Uuid,
456 name: Option<&str>,
457 backup_group_uuid: Option<uuid::Uuid>,
458 filter: &ServerBackupFilter,
459 oldest: bool,
460 ) -> Result<Option<Self>, crate::database::DatabaseError> {
461 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
462 r#"
463 SELECT {}
464 FROM server_backups
465 WHERE
466 server_backups.server_uuid = $1
467 AND server_backups.deleted IS NULL
468 AND server_backups.deleting IS NULL
469 AND server_backups.completed IS NOT NULL
470 AND server_backups.successful
471 AND server_backups.system_backup_policy_uuid IS NULL
472 AND ($2 IS NULL OR server_backups.name = $2)
473 AND ($3::uuid IS NULL OR server_backups.backup_group_uuid = $3)
474 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
475 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
476 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
477 ORDER BY server_backups.created {}
478 LIMIT 1
479 "#,
480 Self::columns_sql(None),
481 if oldest { "ASC" } else { "DESC" }
482 )))
483 .bind(server_uuid)
484 .bind(name)
485 .bind(backup_group_uuid)
486 .bind(filter.kind)
487 .bind(filter.database_instance_uuid)
488 .bind(filter.database_type)
489 .fetch_optional(database.read())
490 .await?;
491
492 row.try_map(|row| Self::map(None, &row))
493 }
494
495 pub async fn by_node_uuid_uuid(
496 database: &crate::database::Database,
497 node_uuid: uuid::Uuid,
498 uuid: uuid::Uuid,
499 ) -> Result<Option<Self>, crate::database::DatabaseError> {
500 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
501 r#"
502 SELECT {}
503 FROM server_backups
504 WHERE server_backups.node_uuid = $1 AND server_backups.uuid = $2
505 "#,
506 Self::columns_sql(None)
507 )))
508 .bind(node_uuid)
509 .bind(uuid)
510 .fetch_optional(database.read())
511 .await?;
512
513 row.try_map(|row| Self::map(None, &row))
514 }
515
516 pub async fn by_server_uuid_with_pagination(
517 database: &crate::database::Database,
518 server_uuid: uuid::Uuid,
519 page: i64,
520 per_page: i64,
521 search: Option<&str>,
522 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
523 let offset = (page - 1) * per_page;
524
525 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
526 r#"
527 SELECT {}, COUNT(*) OVER() AS total_count
528 FROM server_backups
529 WHERE
530 server_backups.server_uuid = $1
531 AND server_backups.deleted IS NULL
532 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
533 ORDER BY server_backups.created
534 LIMIT $3 OFFSET $4
535 "#,
536 Self::columns_sql(None)
537 )))
538 .bind(server_uuid)
539 .bind(search)
540 .bind(per_page)
541 .bind(offset)
542 .fetch_all(database.read())
543 .await?;
544
545 Ok(super::Pagination {
546 total: rows
547 .first()
548 .map_or(Ok(0), |row| row.try_get("total_count"))?,
549 per_page,
550 page,
551 data: rows
552 .into_iter()
553 .map(|row| Self::map(None, &row))
554 .try_collect_vec()?,
555 })
556 }
557
558 pub async fn by_server_uuid_node_uuid_with_pagination(
559 database: &crate::database::Database,
560 server_uuid: uuid::Uuid,
561 node_uuid: uuid::Uuid,
562 page: i64,
563 per_page: i64,
564 search: Option<&str>,
565 filter: &ServerBackupFilter,
566 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
567 let offset = (page - 1) * per_page;
568
569 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
570 r#"
571 SELECT {}, COUNT(*) OVER() AS total_count
572 FROM server_backups
573 WHERE
574 server_backups.server_uuid = $1
575 AND server_backups.node_uuid = $2
576 AND server_backups.system_backup_policy_uuid IS NULL
577 AND server_backups.deleted IS NULL
578 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
579 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
580 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
581 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
582 ORDER BY server_backups.created
583 LIMIT $7 OFFSET $8
584 "#,
585 Self::columns_sql(None)
586 )))
587 .bind(server_uuid)
588 .bind(node_uuid)
589 .bind(search)
590 .bind(filter.kind)
591 .bind(filter.database_instance_uuid)
592 .bind(filter.database_type)
593 .bind(per_page)
594 .bind(offset)
595 .fetch_all(database.read())
596 .await?;
597
598 Ok(super::Pagination {
599 total: rows
600 .first()
601 .map_or(Ok(0), |row| row.try_get("total_count"))?,
602 per_page,
603 page,
604 data: rows
605 .into_iter()
606 .map(|row| Self::map(None, &row))
607 .try_collect_vec()?,
608 })
609 }
610
611 pub async fn by_server_uuid_node_uuid_backup_group_uuid_with_pagination(
612 database: &crate::database::Database,
613 server_uuid: uuid::Uuid,
614 node_uuid: uuid::Uuid,
615 backup_group_uuid: uuid::Uuid,
616 page: i64,
617 per_page: i64,
618 search: Option<&str>,
619 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
620 let offset = (page - 1) * per_page;
621
622 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
623 r#"
624 SELECT {}, COUNT(*) OVER() AS total_count
625 FROM server_backups
626 WHERE
627 server_backups.server_uuid = $1
628 AND server_backups.node_uuid = $2
629 AND server_backups.backup_group_uuid = $3
630 AND server_backups.deleted IS NULL
631 AND ($4 IS NULL OR server_backups.name ILIKE '%' || $4 || '%')
632 ORDER BY server_backups.created
633 LIMIT $5 OFFSET $6
634 "#,
635 Self::columns_sql(None)
636 )))
637 .bind(server_uuid)
638 .bind(node_uuid)
639 .bind(backup_group_uuid)
640 .bind(search)
641 .bind(per_page)
642 .bind(offset)
643 .fetch_all(database.read())
644 .await?;
645
646 Ok(super::Pagination {
647 total: rows
648 .first()
649 .map_or(Ok(0), |row| row.try_get("total_count"))?,
650 per_page,
651 page,
652 data: rows
653 .into_iter()
654 .map(|row| Self::map(None, &row))
655 .try_collect_vec()?,
656 })
657 }
658
659 pub async fn by_ungrouped_server_uuid_node_uuid_with_pagination(
660 database: &crate::database::Database,
661 server_uuid: uuid::Uuid,
662 node_uuid: uuid::Uuid,
663 page: i64,
664 per_page: i64,
665 search: Option<&str>,
666 filter: &ServerBackupFilter,
667 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
668 let offset = (page - 1) * per_page;
669
670 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
671 r#"
672 SELECT {}, COUNT(*) OVER() AS total_count
673 FROM server_backups
674 WHERE
675 server_backups.server_uuid = $1
676 AND server_backups.node_uuid = $2
677 AND server_backups.backup_group_uuid IS NULL
678 AND server_backups.system_backup_policy_uuid IS NULL
679 AND server_backups.deleted IS NULL
680 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
681 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
682 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
683 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
684 ORDER BY server_backups.created
685 LIMIT $7 OFFSET $8
686 "#,
687 Self::columns_sql(None)
688 )))
689 .bind(server_uuid)
690 .bind(node_uuid)
691 .bind(search)
692 .bind(filter.kind)
693 .bind(filter.database_instance_uuid)
694 .bind(filter.database_type)
695 .bind(per_page)
696 .bind(offset)
697 .fetch_all(database.read())
698 .await?;
699
700 Ok(super::Pagination {
701 total: rows
702 .first()
703 .map_or(Ok(0), |row| row.try_get("total_count"))?,
704 per_page,
705 page,
706 data: rows
707 .into_iter()
708 .map(|row| Self::map(None, &row))
709 .try_collect_vec()?,
710 })
711 }
712
713 pub async fn usage_by_server_uuid(
714 database: &crate::database::Database,
715 server_uuid: uuid::Uuid,
716 ) -> Result<ServerBackupUsage, sqlx::Error> {
717 let row = sqlx::query!(
718 r#"
719 SELECT
720 COUNT(*) FILTER (WHERE server_backups.kind = 'SERVER') AS "server!",
721 COUNT(*) FILTER (WHERE server_backups.kind = 'DATABASE_INSTANCE') AS "database_instance!"
722 FROM server_backups
723 WHERE
724 server_backups.server_uuid = $1
725 AND server_backups.system_backup_policy_uuid IS NULL
726 AND server_backups.deleted IS NULL
727 "#,
728 server_uuid
729 )
730 .fetch_one(database.read())
731 .await?;
732
733 Ok(ServerBackupUsage {
734 server: row.server,
735 database_instance: row.database_instance,
736 })
737 }
738
739 pub async fn by_system_server_uuid_node_uuid_with_pagination(
740 database: &crate::database::Database,
741 server_uuid: uuid::Uuid,
742 node_uuid: uuid::Uuid,
743 page: i64,
744 per_page: i64,
745 search: Option<&str>,
746 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
747 let offset = (page - 1) * per_page;
748
749 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
750 r#"
751 SELECT {}, COUNT(*) OVER() AS total_count
752 FROM server_backups
753 WHERE
754 server_backups.server_uuid = $1
755 AND server_backups.node_uuid = $2
756 AND server_backups.system_backup_policy_uuid IS NOT NULL
757 AND server_backups.deleted IS NULL
758 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
759 ORDER BY server_backups.created
760 LIMIT $4 OFFSET $5
761 "#,
762 Self::columns_sql(None)
763 )))
764 .bind(server_uuid)
765 .bind(node_uuid)
766 .bind(search)
767 .bind(per_page)
768 .bind(offset)
769 .fetch_all(database.read())
770 .await?;
771
772 Ok(super::Pagination {
773 total: rows
774 .first()
775 .map_or(Ok(0), |row| row.try_get("total_count"))?,
776 per_page,
777 page,
778 data: rows
779 .into_iter()
780 .map(|row| Self::map(None, &row))
781 .try_collect_vec()?,
782 })
783 }
784
785 pub async fn by_system_backup_policy_uuid_with_pagination(
786 database: &crate::database::Database,
787 system_backup_policy_uuid: uuid::Uuid,
788 page: i64,
789 per_page: i64,
790 search: Option<&str>,
791 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
792 let offset = (page - 1) * per_page;
793
794 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
795 r#"
796 SELECT {}, COUNT(*) OVER() AS total_count
797 FROM server_backups
798 WHERE
799 server_backups.system_backup_policy_uuid = $1
800 AND server_backups.deleted IS NULL
801 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
802 ORDER BY server_backups.created
803 LIMIT $3 OFFSET $4
804 "#,
805 Self::columns_sql(None)
806 )))
807 .bind(system_backup_policy_uuid)
808 .bind(search)
809 .bind(per_page)
810 .bind(offset)
811 .fetch_all(database.read())
812 .await?;
813
814 Ok(super::Pagination {
815 total: rows
816 .first()
817 .map_or(Ok(0), |row| row.try_get("total_count"))?,
818 per_page,
819 page,
820 data: rows
821 .into_iter()
822 .map(|row| Self::map(None, &row))
823 .try_collect_vec()?,
824 })
825 }
826
827 pub async fn all_by_system_backup_policy_uuid(
828 database: &crate::database::Database,
829 system_backup_policy_uuid: uuid::Uuid,
830 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
831 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
832 r#"
833 SELECT {}
834 FROM server_backups
835 WHERE
836 server_backups.system_backup_policy_uuid = $1
837 AND server_backups.deleted IS NULL
838 "#,
839 Self::columns_sql(None)
840 )))
841 .bind(system_backup_policy_uuid)
842 .fetch_all(database.read())
843 .await?;
844
845 rows.into_iter()
846 .map(|row| Self::map(None, &row))
847 .try_collect_vec()
848 }
849
850 pub async fn by_partially_detached_server_uuid_node_uuid_with_pagination(
851 database: &crate::database::Database,
852 server_uuid: uuid::Uuid,
853 node_uuid: uuid::Uuid,
854 page: i64,
855 per_page: i64,
856 search: Option<&str>,
857 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
858 let offset = (page - 1) * per_page;
859
860 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
861 r#"
862 SELECT {}, COUNT(*) OVER() AS total_count
863 FROM server_backups
864 WHERE
865 server_backups.server_uuid = $1
866 AND server_backups.node_uuid != $2
867 AND server_backups.deleted IS NULL
868 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
869 ORDER BY server_backups.created
870 LIMIT $4 OFFSET $5
871 "#,
872 Self::columns_sql(None)
873 )))
874 .bind(server_uuid)
875 .bind(node_uuid)
876 .bind(search)
877 .bind(per_page)
878 .bind(offset)
879 .fetch_all(database.read())
880 .await?;
881
882 Ok(super::Pagination {
883 total: rows
884 .first()
885 .map_or(Ok(0), |row| row.try_get("total_count"))?,
886 per_page,
887 page,
888 data: rows
889 .into_iter()
890 .map(|row| Self::map(None, &row))
891 .try_collect_vec()?,
892 })
893 }
894
895 pub async fn by_node_uuid_with_pagination(
896 database: &crate::database::Database,
897 node_uuid: uuid::Uuid,
898 page: i64,
899 per_page: i64,
900 search: Option<&str>,
901 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
902 let offset = (page - 1) * per_page;
903
904 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
905 r#"
906 SELECT {}, COUNT(*) OVER() AS total_count
907 FROM server_backups
908 WHERE
909 server_backups.node_uuid = $1
910 AND server_backups.deleted IS NULL
911 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
912 ORDER BY server_backups.created
913 LIMIT $3 OFFSET $4
914 "#,
915 Self::columns_sql(None)
916 )))
917 .bind(node_uuid)
918 .bind(search)
919 .bind(per_page)
920 .bind(offset)
921 .fetch_all(database.read())
922 .await?;
923
924 Ok(super::Pagination {
925 total: rows
926 .first()
927 .map_or(Ok(0), |row| row.try_get("total_count"))?,
928 per_page,
929 page,
930 data: rows
931 .into_iter()
932 .map(|row| Self::map(None, &row))
933 .try_collect_vec()?,
934 })
935 }
936
937 pub async fn by_database_agent_host_uuid_with_pagination(
938 database: &crate::database::Database,
939 database_agent_host_uuid: uuid::Uuid,
940 page: i64,
941 per_page: i64,
942 search: Option<&str>,
943 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
944 let offset = (page - 1) * per_page;
945
946 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
947 r#"
948 SELECT {}, COUNT(*) OVER() AS total_count
949 FROM server_backups
950 JOIN server_database_instances ON server_database_instances.uuid = server_backups.database_instance_uuid
951 WHERE
952 server_database_instances.database_agent_host_uuid = $1
953 AND server_backups.deleted IS NULL
954 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
955 ORDER BY server_backups.created
956 LIMIT $3 OFFSET $4
957 "#,
958 Self::columns_sql(None)
959 )))
960 .bind(database_agent_host_uuid)
961 .bind(search)
962 .bind(per_page)
963 .bind(offset)
964 .fetch_all(database.read())
965 .await?;
966
967 Ok(super::Pagination {
968 total: rows
969 .first()
970 .map_or(Ok(0), |row| row.try_get("total_count"))?,
971 per_page,
972 page,
973 data: rows
974 .into_iter()
975 .map(|row| Self::map(None, &row))
976 .try_collect_vec()?,
977 })
978 }
979
980 pub async fn by_backup_configuration_uuid_with_pagination(
981 database: &crate::database::Database,
982 backup_configuration_uuid: uuid::Uuid,
983 page: i64,
984 per_page: i64,
985 search: Option<&str>,
986 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
987 let offset = (page - 1) * per_page;
988
989 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
990 r#"
991 SELECT {}, COUNT(*) OVER() AS total_count
992 FROM server_backups
993 WHERE
994 server_backups.backup_configuration_uuid = $1
995 AND server_backups.deleted IS NULL
996 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
997 ORDER BY server_backups.created
998 LIMIT $3 OFFSET $4
999 "#,
1000 Self::columns_sql(None)
1001 )))
1002 .bind(backup_configuration_uuid)
1003 .bind(search)
1004 .bind(per_page)
1005 .bind(offset)
1006 .fetch_all(database.read())
1007 .await?;
1008
1009 Ok(super::Pagination {
1010 total: rows
1011 .first()
1012 .map_or(Ok(0), |row| row.try_get("total_count"))?,
1013 per_page,
1014 page,
1015 data: rows
1016 .into_iter()
1017 .map(|row| Self::map(None, &row))
1018 .try_collect_vec()?,
1019 })
1020 }
1021
1022 pub async fn by_detached_node_uuid_with_pagination(
1023 database: &crate::database::Database,
1024 node_uuid: uuid::Uuid,
1025 page: i64,
1026 per_page: i64,
1027 search: Option<&str>,
1028 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
1029 let offset = (page - 1) * per_page;
1030
1031 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1032 r#"
1033 SELECT {}, COUNT(*) OVER() AS total_count
1034 FROM server_backups
1035 WHERE
1036 server_backups.node_uuid = $1
1037 AND server_backups.server_uuid IS NULL
1038 AND server_backups.deleted IS NULL
1039 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
1040 ORDER BY server_backups.created
1041 LIMIT $3 OFFSET $4
1042 "#,
1043 Self::columns_sql(None)
1044 )))
1045 .bind(node_uuid)
1046 .bind(search)
1047 .bind(per_page)
1048 .bind(offset)
1049 .fetch_all(database.read())
1050 .await?;
1051
1052 Ok(super::Pagination {
1053 total: rows
1054 .first()
1055 .map_or(Ok(0), |row| row.try_get("total_count"))?,
1056 per_page,
1057 page,
1058 data: rows
1059 .into_iter()
1060 .map(|row| Self::map(None, &row))
1061 .try_collect_vec()?,
1062 })
1063 }
1064
1065 pub async fn all_uuids_by_server_uuid(
1066 database: &crate::database::Database,
1067 server_uuid: uuid::Uuid,
1068 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
1069 let rows = sqlx::query(
1070 r#"
1071 SELECT server_backups.uuid
1072 FROM server_backups
1073 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
1074 "#,
1075 )
1076 .bind(server_uuid)
1077 .fetch_all(database.read())
1078 .await?;
1079
1080 Ok(rows
1081 .into_iter()
1082 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
1083 .collect())
1084 }
1085
1086 pub async fn all_uuids_by_server_uuid_not_shared(
1087 database: &crate::database::Database,
1088 server_uuid: uuid::Uuid,
1089 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
1090 let rows = sqlx::query(
1091 r#"
1092 SELECT server_backups.uuid
1093 FROM server_backups
1094 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL AND server_backups.shared = false
1095 "#,
1096 )
1097 .bind(server_uuid)
1098 .fetch_all(database.read())
1099 .await?;
1100
1101 Ok(rows
1102 .into_iter()
1103 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
1104 .collect())
1105 }
1106
1107 pub async fn all_by_server_uuid(
1108 database: &crate::database::Database,
1109 server_uuid: uuid::Uuid,
1110 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
1111 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1112 r#"
1113 SELECT {}
1114 FROM server_backups
1115 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
1116 "#,
1117 Self::columns_sql(None)
1118 )))
1119 .bind(server_uuid)
1120 .fetch_all(database.read())
1121 .await?;
1122
1123 rows.into_iter()
1124 .map(|row| Self::map(None, &row))
1125 .try_collect_vec()
1126 }
1127
1128 pub async fn count_by_server_uuid(
1129 database: &crate::database::Database,
1130 server_uuid: uuid::Uuid,
1131 ) -> Result<i64, sqlx::Error> {
1132 sqlx::query_scalar(
1133 r#"
1134 SELECT COUNT(*)
1135 FROM server_backups
1136 WHERE
1137 server_backups.server_uuid = $1
1138 AND server_backups.system_backup_policy_uuid IS NULL
1139 AND server_backups.deleted IS NULL
1140 "#,
1141 )
1142 .bind(server_uuid)
1143 .fetch_one(database.read())
1144 .await
1145 }
1146
1147 pub async fn count_system_inflight_by_system_backup_policy_uuid_node_uuid(
1151 database: &crate::database::Database,
1152 system_backup_policy_uuid: uuid::Uuid,
1153 node_uuid: uuid::Uuid,
1154 ) -> Result<i64, sqlx::Error> {
1155 sqlx::query_scalar(
1156 r#"
1157 SELECT COUNT(*)
1158 FROM server_backups
1159 WHERE
1160 server_backups.system_backup_policy_uuid = $1
1161 AND server_backups.node_uuid = $2
1162 AND server_backups.completed IS NULL
1163 AND server_backups.deleted IS NULL
1164 AND server_backups.created >= NOW() - INTERVAL '1 day'
1165 "#,
1166 )
1167 .bind(system_backup_policy_uuid)
1168 .bind(node_uuid)
1169 .fetch_one(database.read())
1170 .await
1171 }
1172
1173 pub async fn count_system_inflight_by_system_backup_policy_uuid_database_agent_host_uuid(
1176 database: &crate::database::Database,
1177 system_backup_policy_uuid: uuid::Uuid,
1178 database_agent_host_uuid: uuid::Uuid,
1179 ) -> Result<i64, sqlx::Error> {
1180 sqlx::query_scalar(
1181 r#"
1182 SELECT COUNT(*)
1183 FROM server_backups
1184 JOIN server_database_instances ON server_database_instances.uuid = server_backups.database_instance_uuid
1185 WHERE
1186 server_backups.system_backup_policy_uuid = $1
1187 AND server_database_instances.database_agent_host_uuid = $2
1188 AND server_backups.completed IS NULL
1189 AND server_backups.deleted IS NULL
1190 AND server_backups.created >= NOW() - INTERVAL '1 day'
1191 "#,
1192 )
1193 .bind(system_backup_policy_uuid)
1194 .bind(database_agent_host_uuid)
1195 .fetch_one(database.read())
1196 .await
1197 }
1198
1199 pub async fn generate_metadata(
1200 state: &crate::State,
1201 server: &super::server::Server,
1202 ) -> Result<serde_json::Value, anyhow::Error> {
1203 let mut variables = serde_json::Map::new();
1204
1205 for variable in ServerVariable::all_by_server_uuid_egg_uuid(
1206 &state.database,
1207 server.uuid,
1208 server.egg.uuid,
1209 )
1210 .await?
1211 {
1212 variables.insert(variable.variable.env_variable.into(), variable.value.into());
1213 }
1214
1215 Ok(serde_json::json!({
1216 "startup": server.startup,
1217 "image": server.image,
1218 "variables": variables,
1219 }))
1220 }
1221
1222 #[inline]
1223 pub fn generate_database_metadata(
1224 database_instance: &super::server_database_instance::ServerDatabaseInstance,
1225 ) -> serde_json::Value {
1226 serde_json::json!({
1227 "source_instance": {
1228 "uuid": database_instance.uuid,
1229 "name": database_instance.name,
1230 },
1231 "image": database_instance.image,
1232 "template_version": database_instance.template_version,
1233 })
1234 }
1235
1236 #[inline]
1238 pub fn database_dump_name(&self) -> compact_str::CompactString {
1239 compact_str::format_compact!(
1240 "{}.{}",
1241 self.uuid,
1242 self.database_type
1243 .map_or("dump", db_agent_api::DatabaseAgentType::dump_extension)
1244 )
1245 }
1246
1247 pub async fn download_url(
1248 &self,
1249 state: &crate::State,
1250 user: &super::user::User,
1251 node: &super::node::Node,
1252 archive_format: wings_api::StreamableArchiveFormat,
1253 ) -> Result<String, anyhow::Error> {
1254 let backup_configuration = self
1255 .backup_configuration
1256 .as_ref()
1257 .ok_or_else(|| {
1258 crate::response::DisplayError::new(
1259 "no backup configuration available, unable to restore backup",
1260 )
1261 .with_status(StatusCode::EXPECTATION_FAILED)
1262 })?
1263 .fetch_cached(&state.database)
1264 .await?;
1265
1266 if backup_configuration.maintenance_enabled {
1267 return Err(crate::response::DisplayError::new(
1268 "cannot restore backup while backup configuration is in maintenance mode",
1269 )
1270 .with_status(StatusCode::EXPECTATION_FAILED)
1271 .into());
1272 }
1273
1274 if matches!(self.disk, BackupDisk::S3)
1275 && let Some(mut s3_configuration) = backup_configuration.backup_configs.s3
1276 {
1277 s3_configuration.decrypt(&state.database).await?;
1278
1279 let (client, bucket) = s3_configuration.into_client();
1280
1281 let file_path = match &self.upload_path {
1282 Some(path) => path,
1283 None => {
1284 return Err(crate::response::DisplayError::new(
1285 "backup does not have an upload path",
1286 )
1287 .with_status(StatusCode::EXPECTATION_FAILED)
1288 .into());
1289 }
1290 };
1291
1292 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1293 std::time::Duration::from_mins(15),
1294 )?;
1295 let presigned = client
1296 .get_object()
1297 .bucket(bucket)
1298 .key(&**file_path)
1299 .presigned(presigning_config)
1300 .await?;
1301
1302 return Ok(presigned.uri().to_string());
1303 }
1304
1305 #[derive(Serialize)]
1306 struct BackupDownloadJwt {
1307 #[serde(flatten)]
1308 base: BasePayload,
1309
1310 backup_uuid: uuid::Uuid,
1311 unique_id: uuid::Uuid,
1312 database: bool,
1313 }
1314
1315 let token = node.create_jwt(
1316 &state.database,
1317 &state.jwt,
1318 &BackupDownloadJwt {
1319 base: BasePayload {
1320 scope: "backup-download".into(),
1321 issuer: "panel".into(),
1322 subject: None,
1323 audience: Vec::new(),
1324 expiration_time: Some(chrono::Utc::now().timestamp() + 900),
1325 not_before: None,
1326 issued_at: Some(chrono::Utc::now().timestamp()),
1327 jwt_id: user.uuid.to_compact_string(),
1328 },
1329 backup_uuid: self.uuid,
1330 unique_id: uuid::Uuid::new_v4(),
1331 database: self.kind == ServerBackupKind::DatabaseInstance,
1332 },
1333 )?;
1334
1335 let mut url = node.public_url(state, "/download/backup").await?;
1336 url.set_query(Some(&format!(
1337 "token={}&archive_format={}",
1338 urlencoding::encode(&token),
1339 archive_format
1340 )));
1341
1342 Ok(url.to_string())
1343 }
1344
1345 pub async fn restore(
1346 self,
1347 state: &crate::State,
1348 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1349 mut server: super::server::Server,
1350 options: ServerBackupRestoreOptions,
1351 ) -> Result<(), anyhow::Error> {
1352 if self.kind != ServerBackupKind::Server {
1353 return Err(crate::response::DisplayError::new(
1354 "database instance backups cannot be restored to server files",
1355 )
1356 .with_status(StatusCode::EXPECTATION_FAILED)
1357 .into());
1358 }
1359
1360 let backup_configuration = self
1361 .backup_configuration
1362 .as_ref()
1363 .ok_or_else(|| {
1364 crate::response::DisplayError::new(
1365 "no backup configuration available, unable to restore backup",
1366 )
1367 .with_status(StatusCode::EXPECTATION_FAILED)
1368 })?
1369 .fetch_cached(&state.database)
1370 .await?;
1371
1372 if backup_configuration.maintenance_enabled {
1373 return Err(crate::response::DisplayError::new(
1374 "cannot restore backup while backup configuration is in maintenance mode",
1375 )
1376 .with_status(StatusCode::EXPECTATION_FAILED)
1377 .into());
1378 }
1379
1380 if options.restore_startup {
1381 self.restore_startup(state, transaction, &mut server)
1382 .await?;
1383 }
1384
1385 server
1386 .node
1387 .fetch_cached(&state.database)
1388 .await?
1389 .api_client(&state.database)
1390 .await?
1391 .post_servers_server_backup_backup_restore(
1392 server.uuid,
1393 self.uuid,
1394 &wings_api::servers_server_backup_backup_restore::post::RequestBody {
1395 adapter: self.disk.to_wings_adapter(),
1396 download_url: self.wings_restore_download_url(state, server.uuid).await?,
1397 truncate_directory: options.truncate_directory,
1398 },
1399 )
1400 .await?;
1401
1402 Self::get_event_emitter().emit(
1403 state.clone(),
1404 ServerBackupEvent::RestoreStarted {
1405 backup: Box::new(self),
1406 server: Box::new(server),
1407 },
1408 );
1409
1410 Ok(())
1411 }
1412
1413 pub async fn restore_database(
1414 self,
1415 state: &crate::State,
1416 server: super::server::Server,
1417 database_instance: &super::server_database_instance::ServerDatabaseInstance,
1418 request_uuid: Option<uuid::Uuid>,
1419 ) -> Result<(), anyhow::Error> {
1420 if self.kind != ServerBackupKind::DatabaseInstance {
1421 return Err(crate::response::DisplayError::new(
1422 "only database instance backups can be restored into a database instance",
1423 )
1424 .with_status(StatusCode::EXPECTATION_FAILED)
1425 .into());
1426 }
1427
1428 if self.server.as_ref().map(|server| server.uuid) != Some(database_instance.server.uuid) {
1429 return Err(crate::response::DisplayError::new(
1430 "backup does not belong to this database instance's server",
1431 )
1432 .with_status(StatusCode::EXPECTATION_FAILED)
1433 .into());
1434 }
1435
1436 if self.database_type != Some(database_instance.r#type) {
1437 return Err(crate::response::DisplayError::new(
1438 "backup was taken from a different database engine",
1439 )
1440 .with_status(StatusCode::EXPECTATION_FAILED)
1441 .into());
1442 }
1443
1444 let backup_configuration = self
1445 .backup_configuration
1446 .as_ref()
1447 .ok_or_else(|| {
1448 crate::response::DisplayError::new(
1449 "no backup configuration available, unable to restore backup",
1450 )
1451 .with_status(StatusCode::EXPECTATION_FAILED)
1452 })?
1453 .fetch_cached(&state.database)
1454 .await?;
1455
1456 if backup_configuration.maintenance_enabled {
1457 return Err(crate::response::DisplayError::new(
1458 "cannot restore backup while backup configuration is in maintenance mode",
1459 )
1460 .with_status(StatusCode::EXPECTATION_FAILED)
1461 .into());
1462 }
1463
1464 server
1465 .node
1466 .fetch_cached(&state.database)
1467 .await?
1468 .api_client(&state.database)
1469 .await?
1470 .post_servers_server_database_backup_backup_restore(
1471 server.uuid,
1472 self.uuid,
1473 &wings_api::servers_server_database_backup_backup_restore::post::RequestBody {
1474 adapter: self.disk.to_wings_adapter(),
1475 database_instance: database_instance.uuid,
1476 download_url: self.wings_restore_download_url(state, server.uuid).await?,
1477 request_uuid,
1478 },
1479 )
1480 .await?;
1481
1482 Self::get_event_emitter().emit(
1483 state.clone(),
1484 ServerBackupEvent::RestoreStarted {
1485 backup: Box::new(self),
1486 server: Box::new(server),
1487 },
1488 );
1489
1490 Ok(())
1491 }
1492
1493 pub async fn restore_startup(
1494 &self,
1495 state: &crate::State,
1496 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1497 server: &mut super::server::Server,
1498 ) -> Result<(), anyhow::Error> {
1499 let startup_cmd = self
1500 .metadata
1501 .get("startup")
1502 .and_then(|v| v.as_str())
1503 .map(|s| s.to_compact_string());
1504 let image_str = self
1505 .metadata
1506 .get("image")
1507 .and_then(|v| v.as_str())
1508 .map(|s| s.to_compact_string());
1509 let variables = self
1510 .metadata
1511 .get("variables")
1512 .and_then(|v| v.as_object())
1513 .cloned()
1514 .unwrap_or_default();
1515
1516 if let Some(startup) = startup_cmd
1517 && let Ok(egg_config) = server.egg.configuration(&state.database).await
1518 {
1519 let is_predefined = server
1520 .egg
1521 .startup_commands
1522 .values()
1523 .any(|cmd| cmd == startup.as_str());
1524 let custom_allowed = egg_config
1525 .config_startup
1526 .as_ref()
1527 .is_some_and(|c| c.allow_custom_startup_command);
1528 if is_predefined || custom_allowed {
1529 server
1530 .update_with_transaction(
1531 state,
1532 super::server::UpdateServerOptions {
1533 startup: Some(startup),
1534 ..Default::default()
1535 },
1536 transaction,
1537 )
1538 .await?;
1539 }
1540 }
1541
1542 if let Some(image) = image_str {
1543 let is_valid_image = server
1544 .egg
1545 .docker_images
1546 .values()
1547 .any(|img| img == image.as_str());
1548 if is_valid_image {
1549 let current_is_custom = !server
1550 .egg
1551 .docker_images
1552 .values()
1553 .any(|img| img == server.image.as_str());
1554 let allow_overwrite = state
1555 .settings
1556 .get()
1557 .await
1558 .map(|s| s.server.allow_overwriting_custom_docker_image)
1559 .unwrap_or(false);
1560 if !current_is_custom || allow_overwrite {
1561 server
1562 .update_with_transaction(
1563 state,
1564 super::server::UpdateServerOptions {
1565 image: Some(image),
1566 ..Default::default()
1567 },
1568 transaction,
1569 )
1570 .await?;
1571 }
1572 }
1573 }
1574
1575 if !variables.is_empty() {
1576 let existing_variables = ServerVariable::all_by_server_uuid_egg_uuid(
1577 &state.database,
1578 server.uuid,
1579 server.egg.uuid,
1580 )
1581 .await?;
1582
1583 let mut validator_variables = HashMap::new();
1584 for variable in existing_variables.iter() {
1585 validator_variables.insert(
1586 variable.variable.env_variable.as_str(),
1587 (
1588 variable.variable.rules.as_slice(),
1589 if let Some(value) = variables
1590 .iter()
1591 .find(|v| v.0 == variable.variable.env_variable)
1592 && variable.variable.user_editable
1593 && let Some(value) = value.1.as_str()
1594 {
1595 value
1596 } else {
1597 variable.value.as_str()
1598 },
1599 ),
1600 );
1601 }
1602
1603 let validator = match rule_validator::Validator::new(validator_variables) {
1604 Ok(validator) => validator,
1605 Err(error) => {
1606 return Err(crate::response::DisplayError::new(error)
1607 .with_status(StatusCode::EXPECTATION_FAILED)
1608 .into());
1609 }
1610 };
1611 if let Err(error) = validator.validate() {
1612 return Err(crate::response::DisplayError::new(error)
1613 .with_status(StatusCode::EXPECTATION_FAILED)
1614 .into());
1615 }
1616
1617 for (env_var, value) in &variables {
1618 let Some(value) = value.as_str() else {
1619 continue;
1620 };
1621 let variable_uuid = match existing_variables
1622 .iter()
1623 .find(|v| v.variable.env_variable == env_var)
1624 {
1625 Some(variable) if variable.variable.user_editable => variable.variable.uuid,
1626 _ => continue,
1627 };
1628
1629 ServerVariable::create_with_transaction(
1630 transaction,
1631 server.uuid,
1632 variable_uuid,
1633 value,
1634 )
1635 .await?;
1636 }
1637 }
1638
1639 Ok(())
1640 }
1641
1642 pub async fn wings_restore_download_url(
1643 &self,
1644 state: &crate::State,
1645 server_uuid: uuid::Uuid,
1646 ) -> Result<Option<compact_str::CompactString>, anyhow::Error> {
1647 if !matches!(self.disk, BackupDisk::S3) {
1648 return Ok(None);
1649 }
1650
1651 let backup_configuration = self
1652 .backup_configuration
1653 .as_ref()
1654 .ok_or_else(|| {
1655 crate::response::DisplayError::new(
1656 "no backup configuration available, unable to restore backup",
1657 )
1658 .with_status(StatusCode::EXPECTATION_FAILED)
1659 })?
1660 .fetch_cached(&state.database)
1661 .await?;
1662
1663 let Some(mut s3_configuration) = backup_configuration.backup_configs.s3 else {
1664 return Ok(None);
1665 };
1666
1667 s3_configuration.decrypt(&state.database).await?;
1668
1669 let compression_type = s3_configuration.compression_type;
1670 let (client, bucket) = s3_configuration.into_client();
1671
1672 let file_path = match &self.upload_path {
1673 Some(path) => path.as_str(),
1674 None => &self.s3_path(server_uuid, compression_type),
1675 };
1676
1677 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1678 std::time::Duration::from_mins(60),
1679 )?;
1680 let presigned = client
1681 .get_object()
1682 .bucket(bucket)
1683 .key(file_path)
1684 .presigned(presigning_config)
1685 .await?;
1686
1687 Ok(Some(presigned.uri().to_compact_string()))
1688 }
1689
1690 pub async fn export(
1691 &self,
1692 state: &crate::State,
1693 server: &super::server::Server,
1694 path: compact_str::CompactString,
1695 archive_format: wings_api::StreamableArchiveFormat,
1696 foreground: bool,
1697 ) -> Result<wings_api::backups_backup_export::post::Response, anyhow::Error> {
1698 let backup_configuration = self
1699 .backup_configuration
1700 .as_ref()
1701 .ok_or_else(|| {
1702 crate::response::DisplayError::new(
1703 "no backup configuration available, unable to export backup",
1704 )
1705 .with_status(StatusCode::EXPECTATION_FAILED)
1706 })?
1707 .fetch_cached(&state.database)
1708 .await?;
1709
1710 if backup_configuration.maintenance_enabled {
1711 return Err(crate::response::DisplayError::new(
1712 "cannot export backup while backup configuration is in maintenance mode",
1713 )
1714 .with_status(StatusCode::EXPECTATION_FAILED)
1715 .into());
1716 }
1717
1718 let client = server
1719 .node
1720 .fetch_cached(&state.database)
1721 .await?
1722 .api_client(&state.database)
1723 .await?
1724 .ignoring(server.subuser_ignored_files.clone().unwrap_or_default());
1725
1726 match client
1727 .post_backups_backup_export(
1728 self.uuid,
1729 &wings_api::backups_backup_export::post::RequestBody {
1730 adapter: self.disk.to_wings_adapter(),
1731 server: server.uuid,
1732 path,
1733 archive_format,
1734 foreground,
1735 },
1736 )
1737 .await
1738 {
1739 Ok(response) => Ok(response),
1740 Err(wings_api::client::ApiHttpError::Http(
1741 status @ (StatusCode::NOT_FOUND | StatusCode::EXPECTATION_FAILED),
1742 err,
1743 )) => Err(crate::response::DisplayError::new(
1744 crate::ApiError::new_wings_value(err).to_string(),
1745 )
1746 .with_status(status)
1747 .into()),
1748 Err(err) => Err(err.into()),
1749 }
1750 }
1751
1752 pub async fn query(
1753 &self,
1754 state: &crate::State,
1755 node: &super::node::Node,
1756 ) -> Result<wings_api::backups_backup_query::get::Response, anyhow::Error> {
1757 let client = node.api_client(&state.database).await?;
1758
1759 match client
1760 .get_backups_backup_query(
1761 self.uuid,
1762 &wings_api::backups_backup_query::get::Query {
1763 adapter: Some(self.disk.to_wings_adapter()),
1764 __priv: (),
1765 },
1766 )
1767 .await
1768 {
1769 Ok(response) => Ok(response),
1770 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, err)) => {
1771 Err(crate::response::DisplayError::new(
1772 crate::ApiError::new_wings_value(err).to_string(),
1773 )
1774 .with_status(StatusCode::NOT_FOUND)
1775 .into())
1776 }
1777 Err(err) => Err(err.into()),
1778 }
1779 }
1780
1781 async fn log_eviction_activity(
1782 state: &crate::State,
1783 server_uuid: uuid::Uuid,
1784 backup: &Self,
1785 rule: &str,
1786 scope: EvictionScope<'_>,
1787 ) {
1788 if let Err(err) = super::server_activity::ServerActivity::create(
1789 state,
1790 super::server_activity::CreateServerActivityOptions {
1791 server_uuid,
1792 user_uuid: None,
1793 impersonator_uuid: None,
1794 api_key_uuid: None,
1795 schedule_uuid: None,
1796 event: match backup.kind {
1797 ServerBackupKind::Server => "server:backup.delete".into(),
1798 ServerBackupKind::DatabaseInstance => "server:database-backup.delete".into(),
1799 },
1800 ip: None,
1801 data: serde_json::json!({
1802 "source": "eviction",
1803 "uuid": backup.uuid,
1804 "name": backup.name,
1805 "database_instance_uuid": backup.database_instance_uuid,
1806 "rule": rule,
1807 "group": scope.group_name(),
1808 "policy": scope.policy_name(),
1809 }),
1810 created: None,
1811 },
1812 )
1813 .await
1814 {
1815 tracing::warn!(
1816 server = %server_uuid,
1817 "failed to log backup eviction activity: {:#?}",
1818 err
1819 );
1820 }
1821 }
1822
1823 #[inline]
1824 pub fn default_name() -> compact_str::CompactString {
1825 let now = chrono::Local::now();
1826
1827 now.format("%Y-%m-%d %H:%M:%S %z").to_compact_string()
1828 }
1829
1830 #[inline]
1831 pub fn s3_path(
1832 &self,
1833 server_uuid: uuid::Uuid,
1834 compression_type: wings_api::CompressionType,
1835 ) -> compact_str::CompactString {
1836 let base_name = match self.kind {
1837 ServerBackupKind::Server => compact_str::format_compact!("{}.tar", self.uuid),
1838 ServerBackupKind::DatabaseInstance => self.database_dump_name(),
1839 };
1840
1841 compact_str::format_compact!(
1842 "{server_uuid}/{base_name}{}",
1843 match compression_type {
1844 wings_api::CompressionType::None => "",
1845 wings_api::CompressionType::Gz => ".gz",
1846 wings_api::CompressionType::Xz => ".xz",
1847 wings_api::CompressionType::Lzip => ".lz",
1848 wings_api::CompressionType::Bz2 => ".bz2",
1849 wings_api::CompressionType::Lz4 => ".lz4",
1850 wings_api::CompressionType::Zstd => ".zst",
1851 }
1852 )
1853 }
1854
1855 #[inline]
1856 pub fn s3_content_type(name: &str) -> &'static str {
1857 if name.ends_with(".tar") {
1858 "application/x-tar"
1859 } else if name.ends_with(".gz") {
1860 "application/x-gzip"
1861 } else if name.ends_with(".xz") {
1862 "application/x-xz"
1863 } else if name.ends_with(".lz") {
1864 "application/x-lzip"
1865 } else if name.ends_with(".bz2") {
1866 "application/x-bzip2"
1867 } else if name.ends_with(".lz4") {
1868 "application/x-lz4"
1869 } else if name.ends_with(".zst") {
1870 "application/zstd"
1871 } else {
1872 "application/octet-stream"
1873 }
1874 }
1875
1876 pub async fn into_admin_node_api_object(
1877 self,
1878 state: &crate::State,
1879 storage_url_retriever: &StorageUrlRetriever<'_>,
1880 ) -> Result<AdminApiNodeServerBackup, crate::database::DatabaseError> {
1881 let deletion_status = self.deletion_status();
1882
1883 Ok(AdminApiNodeServerBackup {
1884 uuid: self.uuid,
1885 server: match self.server {
1886 Some(server) => Some(
1887 server
1888 .fetch_cached(&state.database)
1889 .await?
1890 .into_admin_api_object(state, storage_url_retriever)
1891 .await?,
1892 ),
1893 None => None,
1894 },
1895 node: self
1896 .node
1897 .fetch_cached(&state.database)
1898 .await?
1899 .into_admin_api_object(state, ())
1900 .await?,
1901 backup_group_uuid: self.backup_group_uuid,
1902 system_backup_policy_uuid: self.system_backup_policy_uuid,
1903 database_instance_uuid: self.database_instance_uuid,
1904 kind: self.kind,
1905 database_type: self.database_type,
1906 name: self.name,
1907 ignored_files: self.ignored_files,
1908 is_successful: self.successful,
1909 is_locked: self.locked,
1910 is_browsable: self.browsable,
1911 is_streaming: self.streaming,
1912 is_shared: self.shared,
1913 checksum: self.checksum,
1914 bytes: self.bytes,
1915 files: self.files,
1916 deletion_status,
1917 metadata: self.metadata,
1918 completed: self.completed.map(|dt| dt.and_utc()),
1919 created: self.created.and_utc(),
1920 })
1921 }
1922}
1923
1924#[async_trait::async_trait]
1925impl IntoAdminApiObject for ServerBackup {
1926 type AdminApiObject = AdminApiServerBackup;
1927 type ExtraArgs<'a> = &'a crate::storage::StorageUrlRetriever<'a>;
1928
1929 async fn into_admin_api_object<'a>(
1930 self,
1931 state: &crate::State,
1932 storage_url_retriever: Self::ExtraArgs<'a>,
1933 ) -> Result<Self::AdminApiObject, crate::database::DatabaseError> {
1934 let deletion_status = self.deletion_status();
1935 let api_object = AdminApiServerBackup::init_hooks(&self, state).await?;
1936
1937 let api_object = finish_extendible!(
1938 AdminApiServerBackup {
1939 uuid: self.uuid,
1940 server: match self.server {
1941 Some(server) => Some(
1942 server
1943 .fetch_cached(&state.database)
1944 .await?
1945 .into_admin_api_object(state, storage_url_retriever)
1946 .await?,
1947 ),
1948 None => None,
1949 },
1950 backup_group_uuid: self.backup_group_uuid,
1951 system_backup_policy_uuid: self.system_backup_policy_uuid,
1952 database_instance_uuid: self.database_instance_uuid,
1953 kind: self.kind,
1954 database_type: self.database_type,
1955 name: self.name,
1956 ignored_files: self.ignored_files,
1957 is_successful: self.successful,
1958 is_locked: self.locked,
1959 is_browsable: self.browsable,
1960 is_streaming: self.streaming,
1961 is_shared: self.shared,
1962 checksum: self.checksum,
1963 bytes: self.bytes,
1964 files: self.files,
1965 deletion_status,
1966 metadata: self.metadata,
1967 completed: self.completed.map(|dt| dt.and_utc()),
1968 created: self.created.and_utc(),
1969 },
1970 api_object,
1971 state
1972 )?;
1973
1974 Ok(api_object)
1975 }
1976}
1977
1978#[async_trait::async_trait]
1979impl IntoApiObject for ServerBackup {
1980 type ApiObject = ApiServerBackup;
1981 type ExtraArgs<'a> = ();
1982
1983 async fn into_api_object<'a>(
1984 self,
1985 state: &crate::State,
1986 _args: Self::ExtraArgs<'a>,
1987 ) -> Result<Self::ApiObject, crate::database::DatabaseError> {
1988 let deletion_status = self.deletion_status();
1989 let api_object = ApiServerBackup::init_hooks(&self, state).await?;
1990
1991 let api_object = finish_extendible!(
1992 ApiServerBackup {
1993 uuid: self.uuid,
1994 backup_group_uuid: self.backup_group_uuid,
1995 database_instance_uuid: self.database_instance_uuid,
1996 kind: self.kind,
1997 database_type: self.database_type,
1998 name: self.name,
1999 ignored_files: self.ignored_files,
2000 is_successful: self.successful,
2001 is_locked: self.locked,
2002 is_browsable: self.browsable,
2003 is_streaming: self.streaming,
2004 checksum: self.checksum,
2005 bytes: self.bytes,
2006 files: self.files,
2007 deletion_status,
2008 metadata: self.metadata,
2009 completed: self.completed.map(|dt| dt.and_utc()),
2010 created: self.created.and_utc(),
2011 },
2012 api_object,
2013 state
2014 )?;
2015
2016 Ok(api_object)
2017 }
2018}
2019
2020#[derive(Validate)]
2021pub struct CreateServerBackupOptions<'a> {
2022 #[garde(skip)]
2023 pub server: &'a super::server::Server,
2024 #[garde(length(chars, min = 1, max = 255))]
2025 pub name: compact_str::CompactString,
2026 #[garde(skip)]
2027 pub backup_group_uuid: Option<uuid::Uuid>,
2028 #[garde(skip)]
2029 pub system_backup_policy_uuid: Option<uuid::Uuid>,
2030 #[garde(skip)]
2031 pub database_instance: Option<&'a super::server_database_instance::ServerDatabaseInstance>,
2032 #[garde(skip)]
2033 pub backup_configuration: Option<super::backup_configuration::BackupConfiguration>,
2034 #[garde(skip)]
2035 pub ignored_files: Vec<compact_str::CompactString>,
2036 #[garde(skip)]
2037 pub metadata: serde_json::Value,
2038}
2039
2040impl CreateServerBackupOptions<'_> {
2041 #[inline]
2042 pub fn kind(&self) -> ServerBackupKind {
2043 if self.database_instance.is_some() {
2044 ServerBackupKind::DatabaseInstance
2045 } else {
2046 ServerBackupKind::Server
2047 }
2048 }
2049}
2050
2051#[async_trait::async_trait]
2052impl CreatableModel for ServerBackup {
2053 type CreateOptions<'a> = CreateServerBackupOptions<'a>;
2054 type CreateResult = Self;
2055
2056 fn get_create_handlers() -> &'static LazyLock<CreateListenerList<Self>> {
2057 static CREATE_LISTENERS: LazyLock<CreateListenerList<ServerBackup>> =
2058 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2059
2060 &CREATE_LISTENERS
2061 }
2062
2063 async fn create_with_transaction(
2064 _state: &crate::State,
2065 _options: Self::CreateOptions<'_>,
2066 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2067 ) -> Result<Self, crate::database::DatabaseError> {
2068 Err(anyhow::anyhow!("create_with_transaction is not supported for ServerBackup").into())
2069 }
2070
2071 async fn create(
2072 state: &crate::State,
2073 mut options: Self::CreateOptions<'_>,
2074 ) -> Result<Self, crate::database::DatabaseError> {
2075 options.validate()?;
2076
2077 let backup_configuration = match options.backup_configuration.take() {
2078 Some(backup_configuration) => backup_configuration,
2079 None => options
2080 .server
2081 .backup_configuration(&state.database)
2082 .await
2083 .ok_or_else(|| {
2084 anyhow::Error::new(
2085 crate::response::DisplayError::new(
2086 "no backup configuration available, unable to create backup",
2087 )
2088 .with_status(StatusCode::EXPECTATION_FAILED),
2089 )
2090 })?,
2091 };
2092
2093 if backup_configuration.maintenance_enabled {
2094 return Err(anyhow::Error::new(
2095 crate::response::DisplayError::new(
2096 "cannot create backup while backup configuration is in maintenance mode",
2097 )
2098 .with_status(StatusCode::EXPECTATION_FAILED),
2099 )
2100 .into());
2101 }
2102
2103 if options.database_instance.is_some()
2104 && matches!(
2105 backup_configuration.backup_disk,
2106 BackupDisk::Btrfs | BackupDisk::Zfs
2107 )
2108 {
2109 return Err(anyhow::Error::new(
2110 crate::response::DisplayError::new(
2111 "database backups cannot be created on a btrfs or zfs backup configuration",
2112 )
2113 .with_status(StatusCode::EXPECTATION_FAILED),
2114 )
2115 .into());
2116 }
2117
2118 let mut transaction = state.database.write().begin().await?;
2119
2120 let mut query_builder = InsertQueryBuilder::new("server_backups");
2121
2122 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
2123 .await?;
2124
2125 query_builder
2126 .set("server_uuid", options.server.uuid)
2127 .set("node_uuid", options.server.node.uuid)
2128 .set("backup_configuration_uuid", backup_configuration.uuid)
2129 .set("backup_group_uuid", options.backup_group_uuid)
2130 .set(
2131 "system_backup_policy_uuid",
2132 options.system_backup_policy_uuid,
2133 )
2134 .set(
2135 "database_instance_uuid",
2136 options.database_instance.map(|instance| instance.uuid),
2137 )
2138 .set("kind", options.kind())
2139 .set(
2140 "database_type",
2141 options.database_instance.map(|instance| instance.r#type),
2142 )
2143 .set("name", &options.name)
2144 .set("ignored_files", &options.ignored_files)
2145 .set("bytes", 0i64)
2146 .set("disk", backup_configuration.backup_disk)
2147 .set("shared", backup_configuration.shared)
2148 .set("metadata", &options.metadata);
2149
2150 let row = query_builder
2151 .returning(&Self::columns_sql(None))
2152 .fetch_one(&mut *transaction)
2153 .await?;
2154 let mut backup = Self::map(None, &row)?;
2155
2156 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
2157
2158 transaction.commit().await?;
2159
2160 let server = options.server.clone();
2161 let database = Arc::clone(&state.database);
2162 let backup_uuid = backup.uuid;
2163 let backup_disk = backup.disk;
2164 let database_dump = options
2165 .database_instance
2166 .map(|instance| (instance.uuid, instance.r#type.dump_extension()));
2167 let ignored_files_str = options
2168 .ignored_files
2169 .iter()
2170 .map(|s| s.as_str())
2171 .collect::<Vec<_>>()
2172 .join("\n");
2173
2174 tokio::spawn(async move {
2175 tracing::debug!(backup = %backup_uuid, "creating server backup");
2176
2177 let node = match server.node.fetch_cached(&database).await {
2178 Ok(node) => node,
2179 Err(err) => {
2180 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2181
2182 if let Err(err) = sqlx::query!(
2183 "UPDATE server_backups
2184 SET successful = false, completed = NOW()
2185 WHERE server_backups.uuid = $1",
2186 backup_uuid
2187 )
2188 .execute(database.write())
2189 .await
2190 {
2191 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2192 }
2193
2194 return;
2195 }
2196 };
2197
2198 let api_client = match node.api_client(&database).await {
2199 Ok(api_client) => api_client,
2200 Err(err) => {
2201 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2202
2203 if let Err(err) = sqlx::query!(
2204 "UPDATE server_backups
2205 SET successful = false, completed = NOW()
2206 WHERE server_backups.uuid = $1",
2207 backup_uuid
2208 )
2209 .execute(database.write())
2210 .await
2211 {
2212 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2213 }
2214
2215 return;
2216 }
2217 };
2218
2219 let result = match database_dump {
2220 Some((database_instance, extension)) => api_client
2221 .post_servers_server_database_backup(
2222 server.uuid,
2223 &wings_api::servers_server_database_backup::post::RequestBody {
2224 adapter: backup_disk.to_wings_adapter(),
2225 uuid: backup_uuid,
2226 database_instance,
2227 extension: extension.into(),
2228 },
2229 )
2230 .await
2231 .map(|_| ()),
2232 None => api_client
2233 .post_servers_server_backup(
2234 server.uuid,
2235 &wings_api::servers_server_backup::post::RequestBody {
2236 adapter: backup_disk.to_wings_adapter(),
2237 uuid: backup_uuid,
2238 ignore: ignored_files_str.into(),
2239 },
2240 )
2241 .await
2242 .map(|_| ()),
2243 };
2244
2245 if let Err(err) = result {
2246 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2247
2248 if let Err(err) = sqlx::query!(
2249 "UPDATE server_backups
2250 SET successful = false, completed = NOW()
2251 WHERE server_backups.uuid = $1",
2252 backup_uuid
2253 )
2254 .execute(database.write())
2255 .await
2256 {
2257 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2258 }
2259 }
2260 });
2261
2262 Ok(backup)
2263 }
2264}
2265
2266#[derive(ToSchema, Serialize, Deserialize, Validate, Default)]
2267pub struct UpdateServerBackupOptions {
2268 #[garde(length(chars, min = 1, max = 255))]
2269 #[schema(min_length = 1, max_length = 255)]
2270 pub name: Option<compact_str::CompactString>,
2271 #[garde(skip)]
2272 #[serde(default, with = "::serde_with::rust::double_option")]
2273 pub backup_group_uuid: Option<Option<uuid::Uuid>>,
2274 #[garde(skip)]
2275 pub locked: Option<bool>,
2276}
2277
2278#[async_trait::async_trait]
2279impl UpdatableModel for ServerBackup {
2280 type UpdateOptions = UpdateServerBackupOptions;
2281
2282 fn get_update_handlers() -> &'static LazyLock<UpdateHandlerList<Self>> {
2283 static UPDATE_LISTENERS: LazyLock<UpdateHandlerList<ServerBackup>> =
2284 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2285
2286 &UPDATE_LISTENERS
2287 }
2288
2289 async fn update_with_transaction(
2290 &mut self,
2291 state: &crate::State,
2292 mut options: Self::UpdateOptions,
2293 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2294 ) -> Result<(), crate::database::DatabaseError> {
2295 options.validate()?;
2296
2297 if let Some(Some(backup_group_uuid)) = options.backup_group_uuid {
2298 sqlx::query(
2299 r#"
2300 SELECT server_backup_groups.uuid
2301 FROM server_backup_groups
2302 WHERE server_backup_groups.uuid = $1
2303 FOR KEY SHARE
2304 "#,
2305 )
2306 .bind(backup_group_uuid)
2307 .fetch_optional(&mut **transaction)
2308 .await?;
2309
2310 let group = super::server_backup_group::ServerBackupGroup::by_uuid_with_transaction(
2311 transaction,
2312 backup_group_uuid,
2313 )
2314 .await?;
2315
2316 if Some(group.server_uuid) != self.server.as_ref().map(|server| server.uuid) {
2317 return Err(anyhow::Error::new(
2318 crate::response::DisplayError::new(
2319 "backup group does not belong to this backup's server",
2320 )
2321 .with_status(StatusCode::EXPECTATION_FAILED),
2322 )
2323 .into());
2324 }
2325 }
2326
2327 if sqlx::query(
2328 r#"
2329 SELECT server_backups.uuid
2330 FROM server_backups
2331 WHERE server_backups.uuid = $1
2332 AND server_backups.deleted IS NULL
2333 AND server_backups.deleting IS NULL
2334 FOR UPDATE
2335 "#,
2336 )
2337 .bind(self.uuid)
2338 .fetch_optional(&mut **transaction)
2339 .await?
2340 .is_none()
2341 {
2342 return Err(anyhow::Error::new(
2343 crate::response::DisplayError::new("backup is being deleted")
2344 .with_status(StatusCode::EXPECTATION_FAILED),
2345 )
2346 .into());
2347 }
2348
2349 let mut query_builder = UpdateQueryBuilder::new("server_backups");
2350
2351 self.run_update_handlers(&mut options, &mut query_builder, state, transaction)
2352 .await?;
2353
2354 query_builder
2355 .set("name", options.name.as_ref())
2356 .set("backup_group_uuid", options.backup_group_uuid)
2357 .set("locked", options.locked)
2358 .where_eq("uuid", self.uuid);
2359
2360 query_builder.execute(&mut **transaction).await?;
2361
2362 if let Some(name) = options.name {
2363 self.name = name;
2364 }
2365 if let Some(backup_group_uuid) = options.backup_group_uuid {
2366 self.backup_group_uuid = backup_group_uuid;
2367 }
2368 if let Some(locked) = options.locked {
2369 self.locked = locked;
2370 }
2371
2372 self.run_after_update_handlers(state, transaction).await?;
2373
2374 Ok(())
2375 }
2376}
2377
2378#[async_trait::async_trait]
2379impl ByUuid for ServerBackup {
2380 async fn by_uuid(
2381 database: &crate::database::Database,
2382 uuid: uuid::Uuid,
2383 ) -> Result<Self, crate::database::DatabaseError> {
2384 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2385 r#"
2386 SELECT {}
2387 FROM server_backups
2388 WHERE server_backups.uuid = $1
2389 "#,
2390 Self::columns_sql(None)
2391 )))
2392 .bind(uuid)
2393 .fetch_one(database.read())
2394 .await?;
2395
2396 Self::map(None, &row)
2397 }
2398
2399 async fn by_uuid_with_transaction(
2400 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2401 uuid: uuid::Uuid,
2402 ) -> Result<Self, crate::database::DatabaseError> {
2403 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2404 r#"
2405 SELECT {}
2406 FROM server_backups
2407 WHERE server_backups.uuid = $1
2408 "#,
2409 Self::columns_sql(None)
2410 )))
2411 .bind(uuid)
2412 .fetch_one(&mut **transaction)
2413 .await?;
2414
2415 Self::map(None, &row)
2416 }
2417}
2418
2419#[derive(Clone, Default)]
2420pub struct DeleteServerBackupOptions {
2421 pub force: bool,
2422 pub retention: Option<retention::RetentionDeletionGuard>,
2423}
2424
2425#[derive(Clone, Copy)]
2428pub enum FailedServerBackupScope {
2429 Node(uuid::Uuid),
2430 DetachedNode(uuid::Uuid),
2431 Server(uuid::Uuid),
2432 PartiallyDetachedServer {
2433 server_uuid: uuid::Uuid,
2434 node_uuid: uuid::Uuid,
2435 },
2436 BackupConfiguration(uuid::Uuid),
2437 SystemBackupPolicy(uuid::Uuid),
2438 DatabaseAgentHost(uuid::Uuid),
2439}
2440
2441impl FailedServerBackupScope {
2442 #[inline]
2443 fn condition(self) -> &'static str {
2444 match self {
2445 Self::Node(_) => "server_backups.node_uuid = $1",
2446 Self::DetachedNode(_) => {
2447 "server_backups.node_uuid = $1 AND server_backups.server_uuid IS NULL"
2448 }
2449 Self::Server(_) | Self::PartiallyDetachedServer { .. } => {
2450 "server_backups.server_uuid = $1"
2451 }
2452 Self::BackupConfiguration(_) => "server_backups.backup_configuration_uuid = $1",
2453 Self::SystemBackupPolicy(_) => "server_backups.system_backup_policy_uuid = $1",
2454 Self::DatabaseAgentHost(_) => {
2455 "server_backups.database_instance_uuid IN (
2456 SELECT server_database_instances.uuid
2457 FROM server_database_instances
2458 WHERE server_database_instances.database_agent_host_uuid = $1
2459 )"
2460 }
2461 }
2462 }
2463
2464 #[inline]
2466 fn bindings(self) -> (uuid::Uuid, Option<uuid::Uuid>) {
2467 match self {
2468 Self::Node(uuid)
2469 | Self::DetachedNode(uuid)
2470 | Self::Server(uuid)
2471 | Self::BackupConfiguration(uuid)
2472 | Self::SystemBackupPolicy(uuid)
2473 | Self::DatabaseAgentHost(uuid) => (uuid, None),
2474 Self::PartiallyDetachedServer {
2475 server_uuid,
2476 node_uuid,
2477 } => (server_uuid, Some(node_uuid)),
2478 }
2479 }
2480}
2481
2482impl ServerBackup {
2483 pub const MAX_DELETION_RETRIES: i32 = 8;
2484
2485 #[inline]
2486 pub fn deletion_status(&self) -> Option<ServerBackupDeletionStatus> {
2487 if self.deleted.is_some() || self.deleting.is_none() {
2488 return None;
2489 }
2490
2491 if self.deletion_retries >= Self::MAX_DELETION_RETRIES {
2492 Some(ServerBackupDeletionStatus::Failed)
2493 } else {
2494 Some(ServerBackupDeletionStatus::Deleting)
2495 }
2496 }
2497
2498 pub async fn dispatch_deletion(
2499 &self,
2500 state: &crate::State,
2501 options: &DeleteServerBackupOptions,
2502 ) -> Result<bool, anyhow::Error> {
2503 let node = self.node.fetch_cached(&state.database).await?;
2504
2505 let backup_configuration = match &self.backup_configuration {
2506 Some(backup_configuration) => {
2507 Some(backup_configuration.fetch_cached(&state.database).await?)
2508 }
2509 None if options.force => None,
2510 None => {
2511 return Err(crate::response::DisplayError::new(
2512 "no backup configuration available, unable to delete backup",
2513 )
2514 .with_status(StatusCode::EXPECTATION_FAILED)
2515 .into());
2516 }
2517 };
2518
2519 if let Some(backup_configuration) = &backup_configuration
2520 && backup_configuration.maintenance_enabled
2521 {
2522 return Err(crate::response::DisplayError::new(
2523 "cannot delete backup while backup configuration is in maintenance mode",
2524 )
2525 .with_status(StatusCode::EXPECTATION_FAILED)
2526 .into());
2527 }
2528
2529 if self.disk == BackupDisk::S3 {
2530 let Some(mut s3_configuration) =
2531 backup_configuration.and_then(|c| c.backup_configs.s3.clone())
2532 else {
2533 if options.force {
2534 tracing::warn!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "S3 backup deletion attempted but no S3 configuration found, ignoring");
2535
2536 return Ok(true);
2537 }
2538
2539 return Err(anyhow::anyhow!(
2540 "s3 backup deletion attempted but no S3 configuration found"
2541 ));
2542 };
2543
2544 s3_configuration.decrypt(&state.database).await?;
2545
2546 let compression_type = s3_configuration.compression_type;
2547 let (client, bucket) = s3_configuration.into_client();
2548
2549 let file_path = match &self.upload_path {
2550 Some(path) => path,
2551 None => {
2552 if let Some(server) = &self.server {
2553 &self.s3_path(server.uuid, compression_type)
2554 } else {
2555 return Err(anyhow::anyhow!("backup upload path not found"));
2556 }
2557 }
2558 };
2559
2560 if let Err(err) = client
2561 .delete_object()
2562 .bucket(bucket)
2563 .key(&**file_path)
2564 .send()
2565 .await
2566 {
2567 if options.force {
2568 tracing::error!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "failed to delete S3 backup, ignoring: {:?}", err);
2569 } else {
2570 return Err(err.into());
2571 }
2572 }
2573
2574 return Ok(true);
2575 }
2576
2577 match node
2578 .api_client(&state.database)
2579 .await?
2580 .delete_backups_backup(
2581 self.uuid,
2582 &wings_api::backups_backup::delete::RequestBody {
2583 adapter: self.disk.to_wings_adapter(),
2584 foreground: false,
2585 server: self.server.as_ref().map(|s| s.uuid),
2586 },
2587 )
2588 .await
2589 {
2590 Ok(_) => Ok(false),
2591 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, _)) => Ok(true),
2592 Err(err) if options.force => {
2593 tracing::error!(node = %node.uuid, backup = %self.uuid, "unable to delete backup on node, finalizing anyway: {:?}", err);
2594
2595 Ok(true)
2596 }
2597 Err(err) => Err(err.into()),
2598 }
2599 }
2600
2601 pub async fn finish_deletion(
2602 &self,
2603 state: &crate::State,
2604 options: &DeleteServerBackupOptions,
2605 ) -> Result<(), anyhow::Error> {
2606 let mut transaction = state.database.write().begin().await?;
2607
2608 let finalized = sqlx::query(
2609 r#"
2610 UPDATE server_backups
2611 SET deleted = NOW(), deleting = NULL
2612 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2613 "#,
2614 )
2615 .bind(self.uuid)
2616 .execute(&mut *transaction)
2617 .await?
2618 .rows_affected();
2619
2620 if finalized == 0 {
2621 return Ok(());
2622 }
2623
2624 self.run_after_delete_handlers(options, state, &mut transaction)
2625 .await?;
2626
2627 transaction.commit().await?;
2628
2629 Self::get_event_emitter().emit(
2630 state.clone(),
2631 ServerBackupEvent::DeletionCompleted {
2632 backup: Box::new(self.clone()),
2633 successful: true,
2634 },
2635 );
2636
2637 Ok(())
2638 }
2639
2640 pub async fn fail_deletion_attempt(&self, state: &crate::State) -> Result<i32, anyhow::Error> {
2641 let deletion_retries: Option<i32> = sqlx::query_scalar(
2642 r#"
2643 UPDATE server_backups
2644 SET deletion_retries = deletion_retries + 1
2645 WHERE
2646 server_backups.uuid = $1
2647 AND server_backups.deleted IS NULL
2648 AND server_backups.deleting IS NOT NULL
2649 RETURNING server_backups.deletion_retries
2650 "#,
2651 )
2652 .bind(self.uuid)
2653 .fetch_optional(state.database.write())
2654 .await?;
2655
2656 let Some(deletion_retries) = deletion_retries else {
2657 return Ok(0);
2658 };
2659
2660 if deletion_retries >= Self::MAX_DELETION_RETRIES {
2661 if let Some(server) = &self.server
2662 && let Err(err) = super::server_activity::ServerActivity::create(
2663 state,
2664 super::server_activity::CreateServerActivityOptions {
2665 server_uuid: server.uuid,
2666 user_uuid: None,
2667 impersonator_uuid: None,
2668 api_key_uuid: None,
2669 schedule_uuid: None,
2670 event: match self.kind {
2671 ServerBackupKind::Server => "server:backup.delete-failed",
2672 ServerBackupKind::DatabaseInstance => {
2673 "server:database-backup.delete-failed"
2674 }
2675 }
2676 .into(),
2677 ip: None,
2678 data: serde_json::json!({
2679 "uuid": self.uuid,
2680 "name": self.name,
2681 "database_instance_uuid": self.database_instance_uuid,
2682 }),
2683 created: None,
2684 },
2685 )
2686 .await
2687 {
2688 tracing::warn!(
2689 backup = %self.uuid,
2690 "failed to log backup deletion failure activity: {:#?}",
2691 err
2692 );
2693 }
2694
2695 Self::get_event_emitter().emit(
2696 state.clone(),
2697 ServerBackupEvent::DeletionCompleted {
2698 backup: Box::new(self.clone()),
2699 successful: false,
2700 },
2701 );
2702 }
2703
2704 Ok(deletion_retries)
2705 }
2706
2707 pub async fn redispatch_stale_deletions(state: &crate::State) -> Result<u64, anyhow::Error> {
2708 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2709 r#"
2710 SELECT {}
2711 FROM server_backups
2712 WHERE
2713 server_backups.deleted IS NULL
2714 AND server_backups.deleting IS NOT NULL
2715 AND server_backups.deletion_retries < $1
2716 AND server_backups.deleting < NOW() - make_interval(mins => LEAST(60.0, 5.0 * POWER(2.0, server_backups.deletion_retries))::int)
2717 ORDER BY server_backups.deleting
2718 LIMIT 32
2719 "#,
2720 Self::columns_sql(None)
2721 )))
2722 .bind(Self::MAX_DELETION_RETRIES)
2723 .fetch_all(state.database.read())
2724 .await?;
2725
2726 let mut redispatched = 0;
2727 for row in rows {
2728 let backup = Self::map(None, &row)?;
2729
2730 if let Some(backup_configuration) = &backup.backup_configuration
2731 && let Ok(backup_configuration) =
2732 backup_configuration.fetch_cached(&state.database).await
2733 && backup_configuration.maintenance_enabled
2734 {
2735 sqlx::query(
2736 "UPDATE server_backups
2737 SET deleting = NOW()
2738 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL",
2739 )
2740 .bind(backup.uuid)
2741 .execute(state.database.write())
2742 .await?;
2743
2744 continue;
2745 }
2746
2747 let deletion_retries: i32 = sqlx::query_scalar(
2748 r#"
2749 UPDATE server_backups
2750 SET deleting = NOW(), deletion_retries = deletion_retries + 1
2751 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2752 RETURNING server_backups.deletion_retries
2753 "#,
2754 )
2755 .bind(backup.uuid)
2756 .fetch_one(state.database.write())
2757 .await?;
2758
2759 match backup
2760 .dispatch_deletion(state, &DeleteServerBackupOptions::default())
2761 .await
2762 {
2763 Ok(true) => {
2764 backup
2765 .finish_deletion(state, &DeleteServerBackupOptions::default())
2766 .await?;
2767 }
2768 Ok(false) => {}
2769 Err(err) => {
2770 tracing::error!(
2771 backup = %backup.uuid,
2772 deletion_retries,
2773 "failed to redispatch backup deletion: {:#?}",
2774 err
2775 );
2776
2777 if deletion_retries >= Self::MAX_DELETION_RETRIES {
2778 Self::get_event_emitter().emit(
2779 state.clone(),
2780 ServerBackupEvent::DeletionCompleted {
2781 backup: Box::new(backup.clone()),
2782 successful: false,
2783 },
2784 );
2785 }
2786
2787 continue;
2788 }
2789 }
2790
2791 redispatched += 1;
2792 }
2793
2794 Ok(redispatched)
2795 }
2796}
2797
2798#[async_trait::async_trait]
2799impl DeletableModel for ServerBackup {
2800 type DeleteOptions = DeleteServerBackupOptions;
2801
2802 fn get_delete_handlers() -> &'static LazyLock<DeleteHandlerList<Self>> {
2803 static DELETE_LISTENERS: LazyLock<DeleteHandlerList<ServerBackup>> =
2804 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2805
2806 &DELETE_LISTENERS
2807 }
2808
2809 async fn delete_with_transaction(
2810 &self,
2811 _state: &crate::State,
2812 _options: Self::DeleteOptions,
2813 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2814 ) -> Result<(), anyhow::Error> {
2815 Err(anyhow::anyhow!(
2816 "delete_with_transaction is not supported for ServerBackup"
2817 ))
2818 }
2819
2820 async fn delete(
2821 &self,
2822 state: &crate::State,
2823 options: Self::DeleteOptions,
2824 ) -> Result<(), anyhow::Error> {
2825 let mut transaction = state.database.write().begin().await?;
2826
2827 self.claim_deletion(state, &options, &mut transaction)
2828 .await?;
2829
2830 transaction.commit().await?;
2831
2832 self.dispatch_claimed_deletion(state, &options).await
2833 }
2834}
2835
2836impl ServerBackup {
2837 async fn backup_configuration_in_maintenance(
2838 &self,
2839 state: &crate::State,
2840 ) -> Result<bool, anyhow::Error> {
2841 let Some(configuration) = &self.backup_configuration else {
2842 return Ok(false);
2843 };
2844
2845 Ok(configuration
2846 .fetch_cached(&state.database)
2847 .await?
2848 .maintenance_enabled)
2849 }
2850
2851 async fn claim_deletion(
2852 &self,
2853 state: &crate::State,
2854 options: &DeleteServerBackupOptions,
2855 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2856 ) -> Result<(), anyhow::Error> {
2857 if self.backup_configuration_in_maintenance(state).await? {
2858 return Err(crate::response::DisplayError::new(
2859 "cannot delete backup while backup configuration is in maintenance mode",
2860 )
2861 .with_status(StatusCode::EXPECTATION_FAILED)
2862 .into());
2863 }
2864
2865 self.run_delete_handlers(options, state, transaction)
2866 .await?;
2867
2868 let retention_guard = options.retention.as_ref();
2869 let retention = retention_guard
2870 .map(|guard| serde_json::to_value(&guard.retention))
2871 .transpose()?;
2872
2873 let claimed = sqlx::query(
2874 r#"
2875 UPDATE server_backups
2876 SET deleting = NOW(), deletion_retries = 0
2877 WHERE
2878 server_backups.uuid = $1
2879 AND server_backups.deleted IS NULL
2880 AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $2)
2881 AND (NOT $3 OR (
2882 NOT server_backups.locked
2883 AND server_backups.completed = $4
2884 AND server_backups.successful = $5
2885 AND server_backups.backup_group_uuid IS NOT DISTINCT FROM $6
2886 AND server_backups.system_backup_policy_uuid IS NOT DISTINCT FROM $7
2887 AND server_backups.server_uuid IS NOT DISTINCT FROM $8
2888 AND server_backups.database_instance_uuid IS NOT DISTINCT FROM $9
2889 AND (
2890 EXISTS (
2891 SELECT 1 FROM server_backup_groups g
2892 WHERE g.uuid = $6
2893 AND g.retention IS NOT DISTINCT FROM $10
2894 ) OR EXISTS (
2895 SELECT 1 FROM system_backup_policies p
2896 WHERE p.uuid = $7
2897 AND p.retention IS NOT DISTINCT FROM $10
2898 )
2899 )
2900 ))
2901 "#,
2902 )
2903 .bind(self.uuid)
2904 .bind(Self::MAX_DELETION_RETRIES)
2905 .bind(retention_guard.is_some())
2906 .bind(retention_guard.map(|guard| guard.completed))
2907 .bind(retention_guard.map(|guard| guard.successful))
2908 .bind(retention_guard.and_then(|guard| guard.backup_group_uuid))
2909 .bind(retention_guard.and_then(|guard| guard.system_backup_policy_uuid))
2910 .bind(self.server.as_ref().map(|server| server.uuid))
2911 .bind(self.database_instance_uuid)
2912 .bind(retention)
2913 .execute(&mut **transaction)
2914 .await?
2915 .rows_affected();
2916
2917 if claimed == 0 {
2918 return Err(
2919 crate::response::DisplayError::new(if retention_guard.is_some() {
2920 "backup is no longer eligible for retention cleanup"
2921 } else {
2922 "backup is already being deleted"
2923 })
2924 .with_status(StatusCode::EXPECTATION_FAILED)
2925 .into(),
2926 );
2927 }
2928
2929 Ok(())
2930 }
2931
2932 async fn dispatch_claimed_deletion(
2933 &self,
2934 state: &crate::State,
2935 options: &DeleteServerBackupOptions,
2936 ) -> Result<(), anyhow::Error> {
2937 match self.dispatch_deletion(state, options).await {
2938 Ok(true) => self.finish_deletion(state, options).await,
2939 Ok(false) => Ok(()),
2940 Err(err) => {
2941 sqlx::query(
2942 r#"
2943 UPDATE server_backups
2944 SET deleting = NULL, deletion_retries = 0
2945 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2946 "#,
2947 )
2948 .bind(self.uuid)
2949 .execute(state.database.write())
2950 .await?;
2951
2952 Err(err)
2953 }
2954 }
2955 }
2956
2957 const FAILED_SWEEP_CONDITION: &'static str = r#"
2961 server_backups.deleted IS NULL
2962 AND server_backups.completed IS NOT NULL
2963 AND NOT server_backups.successful
2964 AND NOT server_backups.locked
2965 AND ($2::uuid IS NULL OR server_backups.node_uuid != $2)
2966 AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $3)
2967 AND NOT EXISTS (
2968 SELECT 1
2969 FROM backup_configurations
2970 WHERE backup_configurations.uuid = server_backups.backup_configuration_uuid
2971 AND backup_configurations.maintenance_enabled
2972 )
2973 "#;
2974
2975 const FAILED_SWEEP_CHUNK: i64 = 250;
2976 const FAILED_SWEEP_CONCURRENCY: usize = 5;
2977
2978 pub async fn count_failed(
2979 database: &crate::database::Database,
2980 scope: FailedServerBackupScope,
2981 ) -> Result<i64, sqlx::Error> {
2982 let (scope_uuid, excluded_node_uuid) = scope.bindings();
2983
2984 sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
2985 r#"
2986 SELECT COUNT(*)
2987 FROM server_backups
2988 WHERE {} AND {}
2989 "#,
2990 scope.condition(),
2991 Self::FAILED_SWEEP_CONDITION
2992 )))
2993 .bind(scope_uuid)
2994 .bind(excluded_node_uuid)
2995 .bind(Self::MAX_DELETION_RETRIES)
2996 .fetch_one(database.read())
2997 .await
2998 }
2999
3000 async fn claim_failed_deletions(
3004 state: &crate::State,
3005 scope: FailedServerBackupScope,
3006 options: &DeleteServerBackupOptions,
3007 after: Option<uuid::Uuid>,
3008 ) -> Result<(Vec<Self>, Option<uuid::Uuid>), anyhow::Error> {
3009 let (scope_uuid, excluded_node_uuid) = scope.bindings();
3010
3011 let mut transaction = state.database.write().begin().await?;
3012
3013 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
3014 r#"
3015 SELECT {}
3016 FROM server_backups
3017 WHERE {}
3018 AND {}
3019 AND ($4::uuid IS NULL OR server_backups.uuid > $4)
3020 ORDER BY server_backups.uuid
3021 LIMIT $5
3022 FOR UPDATE
3023 "#,
3024 Self::columns_sql(None),
3025 scope.condition(),
3026 Self::FAILED_SWEEP_CONDITION
3027 )))
3028 .bind(scope_uuid)
3029 .bind(excluded_node_uuid)
3030 .bind(Self::MAX_DELETION_RETRIES)
3031 .bind(after)
3032 .bind(Self::FAILED_SWEEP_CHUNK)
3033 .fetch_all(&mut *transaction)
3034 .await?;
3035
3036 let backups = rows
3037 .iter()
3038 .map(|row| Self::map(None, row))
3039 .try_collect_vec()?;
3040 let cursor = backups.last().map(|backup| backup.uuid);
3041
3042 let mut claimed = Vec::new();
3043 for backup in backups {
3044 if backup.backup_configuration_in_maintenance(state).await? {
3045 continue;
3046 }
3047
3048 backup
3049 .claim_deletion(state, options, &mut transaction)
3050 .await?;
3051 claimed.push(backup);
3052 }
3053
3054 transaction.commit().await?;
3055
3056 Ok((claimed, cursor))
3057 }
3058
3059 async fn dispatch_swept_deletion(
3060 &self,
3061 state: &crate::State,
3062 options: &DeleteServerBackupOptions,
3063 ) -> bool {
3064 if let Err(err) = self.dispatch_claimed_deletion(state, options).await {
3065 tracing::error!(backup = %self.uuid, "failed to delete failed backup: {err:#?}");
3066
3067 return false;
3068 }
3069
3070 true
3071 }
3072
3073 pub async fn delete_failed(
3077 state: &crate::State,
3078 scope: FailedServerBackupScope,
3079 force: bool,
3080 ) -> Result<u64, anyhow::Error> {
3081 let options = DeleteServerBackupOptions {
3082 force,
3083 ..Default::default()
3084 };
3085
3086 let mut deleted = 0;
3087 let mut after = None;
3088
3089 loop {
3090 let (claimed, cursor) =
3091 Self::claim_failed_deletions(state, scope, &options, after).await?;
3092
3093 let mut futures = Vec::with_capacity(claimed.len());
3094 for backup in &claimed {
3095 futures.push(backup.dispatch_swept_deletion(state, &options));
3096 }
3097
3098 let mut results_stream = futures_util::stream::iter(futures)
3099 .buffer_unordered(Self::FAILED_SWEEP_CONCURRENCY);
3100
3101 while let Some(dispatched) = results_stream.next().await {
3102 if dispatched {
3103 deleted += 1;
3104 }
3105 }
3106
3107 match cursor {
3108 Some(cursor) => after = Some(cursor),
3109 None => break,
3110 }
3111 }
3112
3113 Ok(deleted)
3114 }
3115}
3116
3117#[derive(ToSchema, Serialize)]
3118#[schema(title = "AdminNodeServerBackup")]
3119pub struct AdminApiNodeServerBackup {
3120 pub uuid: uuid::Uuid,
3121 pub server: Option<super::server::AdminApiServer>,
3122 pub node: super::node::AdminApiNode,
3123 pub backup_group_uuid: Option<uuid::Uuid>,
3124 pub system_backup_policy_uuid: Option<uuid::Uuid>,
3125 pub database_instance_uuid: Option<uuid::Uuid>,
3126
3127 pub kind: ServerBackupKind,
3128 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3129 pub name: compact_str::CompactString,
3130 pub ignored_files: Vec<compact_str::CompactString>,
3131
3132 pub is_successful: bool,
3133 pub is_locked: bool,
3134 pub is_browsable: bool,
3135 pub is_streaming: bool,
3136 pub is_shared: bool,
3137
3138 pub checksum: Option<compact_str::CompactString>,
3139 pub bytes: i64,
3140 pub files: i64,
3141
3142 pub metadata: serde_json::Value,
3143 pub deletion_status: Option<ServerBackupDeletionStatus>,
3144
3145 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3146 pub created: chrono::DateTime<chrono::Utc>,
3147}
3148
3149#[schema_extension_derive::extendible]
3150#[init_args(ServerBackup, crate::State)]
3151#[hook_args(crate::State)]
3152#[derive(ToSchema, Serialize)]
3153#[schema(title = "AdminServerBackup")]
3154pub struct AdminApiServerBackup {
3155 pub uuid: uuid::Uuid,
3156 pub server: Option<super::server::AdminApiServer>,
3157 pub backup_group_uuid: Option<uuid::Uuid>,
3158 pub system_backup_policy_uuid: Option<uuid::Uuid>,
3159 pub database_instance_uuid: Option<uuid::Uuid>,
3160
3161 pub kind: ServerBackupKind,
3162 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3163 pub name: compact_str::CompactString,
3164 pub ignored_files: Vec<compact_str::CompactString>,
3165
3166 pub is_successful: bool,
3167 pub is_locked: bool,
3168 pub is_browsable: bool,
3169 pub is_streaming: bool,
3170 pub is_shared: bool,
3171
3172 pub checksum: Option<compact_str::CompactString>,
3173 pub bytes: i64,
3174 pub files: i64,
3175
3176 pub metadata: serde_json::Value,
3177 pub deletion_status: Option<ServerBackupDeletionStatus>,
3178
3179 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3180 pub created: chrono::DateTime<chrono::Utc>,
3181}
3182
3183#[derive(ToSchema, Serialize)]
3184#[schema(title = "ServerBackupUsage")]
3185pub struct ServerBackupUsage {
3186 pub server: i64,
3187 pub database_instance: i64,
3188}
3189
3190#[schema_extension_derive::extendible]
3191#[init_args(ServerBackup, crate::State)]
3192#[hook_args(crate::State)]
3193#[derive(ToSchema, Serialize)]
3194#[schema(title = "ServerBackup")]
3195pub struct ApiServerBackup {
3196 pub uuid: uuid::Uuid,
3197 pub backup_group_uuid: Option<uuid::Uuid>,
3198 pub database_instance_uuid: Option<uuid::Uuid>,
3199
3200 pub kind: ServerBackupKind,
3201 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3202 pub name: compact_str::CompactString,
3203 pub ignored_files: Vec<compact_str::CompactString>,
3204
3205 pub is_successful: bool,
3206 pub is_locked: bool,
3207 pub is_browsable: bool,
3208 pub is_streaming: bool,
3209
3210 pub checksum: Option<compact_str::CompactString>,
3211 pub bytes: i64,
3212 pub files: i64,
3213
3214 pub metadata: serde_json::Value,
3215 pub deletion_status: Option<ServerBackupDeletionStatus>,
3216
3217 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3218 pub created: chrono::DateTime<chrono::Utc>,
3219}