1mod events;
2pub use events::ServerBackupEvent;
3
4pub mod retention;
5
6use crate::{
7 jwt::BasePayload,
8 models::{InsertQueryBuilder, UpdateQueryBuilder, server_variable::ServerVariable},
9 prelude::*,
10 storage::StorageUrlRetriever,
11};
12use compact_str::ToCompactString;
13use futures_util::StreamExt;
14use garde::Validate;
15use reqwest::StatusCode;
16use serde::{Deserialize, Serialize};
17use sqlx::{Row, postgres::PgRow, prelude::Type};
18use std::{
19 collections::{BTreeMap, HashMap},
20 sync::{Arc, LazyLock},
21};
22use utoipa::ToSchema;
23
24#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
25#[serde(rename_all = "kebab-case")]
26#[sqlx(type_name = "backup_disk", rename_all = "SCREAMING_SNAKE_CASE")]
27pub enum BackupDisk {
28 Local,
29 S3,
30 DdupBak,
31 Btrfs,
32 Zfs,
33 Restic,
34 ProxmoxBackupServer,
35 Kopia,
36}
37
38impl BackupDisk {
39 #[inline]
40 pub fn from_wings_adapter(adapter: wings_api::BackupAdapter) -> Self {
41 match adapter {
42 wings_api::BackupAdapter::Wings => BackupDisk::Local,
43 wings_api::BackupAdapter::S3 => BackupDisk::S3,
44 wings_api::BackupAdapter::DdupBak => BackupDisk::DdupBak,
45 wings_api::BackupAdapter::Btrfs => BackupDisk::Btrfs,
46 wings_api::BackupAdapter::Zfs => BackupDisk::Zfs,
47 wings_api::BackupAdapter::Restic => BackupDisk::Restic,
48 wings_api::BackupAdapter::ProxmoxBackupServer => BackupDisk::ProxmoxBackupServer,
49 wings_api::BackupAdapter::Kopia => BackupDisk::Kopia,
50 }
51 }
52
53 #[inline]
54 pub fn to_wings_adapter(self) -> wings_api::BackupAdapter {
55 match self {
56 BackupDisk::Local => wings_api::BackupAdapter::Wings,
57 BackupDisk::S3 => wings_api::BackupAdapter::S3,
58 BackupDisk::DdupBak => wings_api::BackupAdapter::DdupBak,
59 BackupDisk::Btrfs => wings_api::BackupAdapter::Btrfs,
60 BackupDisk::Zfs => wings_api::BackupAdapter::Zfs,
61 BackupDisk::Restic => wings_api::BackupAdapter::Restic,
62 BackupDisk::ProxmoxBackupServer => wings_api::BackupAdapter::ProxmoxBackupServer,
63 BackupDisk::Kopia => wings_api::BackupAdapter::Kopia,
64 }
65 }
66}
67
68#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
69#[serde(rename_all = "snake_case")]
70#[sqlx(type_name = "server_backup_kind", rename_all = "SCREAMING_SNAKE_CASE")]
71pub enum ServerBackupKind {
72 Server,
73 DatabaseInstance,
74}
75
76pub struct ServerBackupFilter {
77 pub kind: Option<ServerBackupKind>,
78 pub database_instance_uuid: Option<uuid::Uuid>,
79 pub database_type: Option<db_agent_api::DatabaseAgentType>,
80}
81
82pub struct ServerBackupRestoreOptions {
83 pub truncate_directory: bool,
84 pub restore_startup: bool,
85}
86
87#[derive(Debug, Clone, Copy)]
89enum EvictionScope<'a> {
90 Server,
91 Group(&'a str),
92 Policy(&'a str),
93}
94
95impl<'a> EvictionScope<'a> {
96 #[inline]
97 fn group_name(self) -> Option<&'a str> {
98 match self {
99 Self::Group(name) => Some(name),
100 _ => None,
101 }
102 }
103
104 #[inline]
105 fn policy_name(self) -> Option<&'a str> {
106 match self {
107 Self::Policy(name) => Some(name),
108 _ => None,
109 }
110 }
111}
112
113#[derive(Serialize, Deserialize, Clone)]
114pub struct ServerBackup {
115 pub uuid: uuid::Uuid,
116 pub server: Option<Fetchable<super::server::Server>>,
117 pub node: Fetchable<super::node::Node>,
118 pub backup_configuration: Option<Fetchable<super::backup_configuration::BackupConfiguration>>,
119 pub backup_group_uuid: Option<uuid::Uuid>,
120 pub system_backup_policy_uuid: Option<uuid::Uuid>,
121 pub database_instance_uuid: Option<uuid::Uuid>,
122
123 pub kind: ServerBackupKind,
124 pub database_type: Option<db_agent_api::DatabaseAgentType>,
125 pub name: compact_str::CompactString,
126 pub successful: bool,
127 pub browsable: bool,
128 pub streaming: bool,
129 pub locked: bool,
130 pub shared: bool,
131
132 pub ignored_files: Vec<compact_str::CompactString>,
133 pub checksum: Option<compact_str::CompactString>,
134 pub bytes: i64,
135 pub files: i64,
136
137 pub disk: BackupDisk,
138 pub upload_id: Option<compact_str::CompactString>,
139 pub upload_path: Option<compact_str::CompactString>,
140 pub metadata: serde_json::Value,
141
142 pub completed: Option<chrono::NaiveDateTime>,
143 pub deleting: Option<chrono::NaiveDateTime>,
144 pub deletion_retries: i32,
145 pub deleted: Option<chrono::NaiveDateTime>,
146 pub created: chrono::NaiveDateTime,
147
148 extension_data: super::ModelExtensionData,
149}
150
151#[derive(Debug, ToSchema, Serialize, Clone, Copy, PartialEq, Eq)]
152#[serde(rename_all = "snake_case")]
153pub enum ServerBackupDeletionStatus {
154 Deleting,
155 Failed,
156}
157
158impl BaseModel for ServerBackup {
159 const NAME: &'static str = "server_backup";
160
161 fn get_extension_list() -> &'static super::ModelExtensionList {
162 static EXTENSIONS: LazyLock<super::ModelExtensionList> =
163 LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
164
165 &EXTENSIONS
166 }
167
168 fn get_extension_data(&self) -> &super::ModelExtensionData {
169 &self.extension_data
170 }
171
172 #[inline]
173 fn base_columns(prefix: Option<&str>) -> BTreeMap<&'static str, compact_str::CompactString> {
174 let prefix = prefix.unwrap_or_default();
175
176 BTreeMap::from([
177 (
178 "server_backups.uuid",
179 compact_str::format_compact!("{prefix}uuid"),
180 ),
181 (
182 "server_backups.server_uuid",
183 compact_str::format_compact!("{prefix}server_uuid"),
184 ),
185 (
186 "server_backups.node_uuid",
187 compact_str::format_compact!("{prefix}node_uuid"),
188 ),
189 (
190 "server_backups.backup_configuration_uuid",
191 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
192 ),
193 (
194 "server_backups.backup_group_uuid",
195 compact_str::format_compact!("{prefix}backup_group_uuid"),
196 ),
197 (
198 "server_backups.system_backup_policy_uuid",
199 compact_str::format_compact!("{prefix}system_backup_policy_uuid"),
200 ),
201 (
202 "server_backups.database_instance_uuid",
203 compact_str::format_compact!("{prefix}database_instance_uuid"),
204 ),
205 (
206 "server_backups.kind",
207 compact_str::format_compact!("{prefix}kind"),
208 ),
209 (
210 "server_backups.database_type",
211 compact_str::format_compact!("{prefix}database_type"),
212 ),
213 (
214 "server_backups.name",
215 compact_str::format_compact!("{prefix}name"),
216 ),
217 (
218 "server_backups.successful",
219 compact_str::format_compact!("{prefix}successful"),
220 ),
221 (
222 "server_backups.browsable",
223 compact_str::format_compact!("{prefix}browsable"),
224 ),
225 (
226 "server_backups.streaming",
227 compact_str::format_compact!("{prefix}streaming"),
228 ),
229 (
230 "server_backups.locked",
231 compact_str::format_compact!("{prefix}locked"),
232 ),
233 (
234 "server_backups.shared",
235 compact_str::format_compact!("{prefix}shared"),
236 ),
237 (
238 "server_backups.ignored_files",
239 compact_str::format_compact!("{prefix}ignored_files"),
240 ),
241 (
242 "server_backups.checksum",
243 compact_str::format_compact!("{prefix}checksum"),
244 ),
245 (
246 "server_backups.bytes",
247 compact_str::format_compact!("{prefix}bytes"),
248 ),
249 (
250 "server_backups.files",
251 compact_str::format_compact!("{prefix}files"),
252 ),
253 (
254 "server_backups.disk",
255 compact_str::format_compact!("{prefix}disk"),
256 ),
257 (
258 "server_backups.upload_id",
259 compact_str::format_compact!("{prefix}upload_id"),
260 ),
261 (
262 "server_backups.upload_path",
263 compact_str::format_compact!("{prefix}upload_path"),
264 ),
265 (
266 "server_backups.metadata",
267 compact_str::format_compact!("{prefix}metadata"),
268 ),
269 (
270 "server_backups.completed",
271 compact_str::format_compact!("{prefix}completed"),
272 ),
273 (
274 "server_backups.deleting",
275 compact_str::format_compact!("{prefix}deleting"),
276 ),
277 (
278 "server_backups.deletion_retries",
279 compact_str::format_compact!("{prefix}deletion_retries"),
280 ),
281 (
282 "server_backups.deleted",
283 compact_str::format_compact!("{prefix}deleted"),
284 ),
285 (
286 "server_backups.created",
287 compact_str::format_compact!("{prefix}created"),
288 ),
289 ])
290 }
291
292 #[inline]
293 fn map(prefix: Option<&str>, row: &PgRow) -> Result<Self, crate::database::DatabaseError> {
294 let prefix = prefix.unwrap_or_default();
295
296 Ok(Self {
297 uuid: row.try_get(compact_str::format_compact!("{prefix}uuid").as_str())?,
298 server: super::server::Server::get_fetchable_from_row(
299 row,
300 compact_str::format_compact!("{prefix}server_uuid"),
301 ),
302 backup_configuration:
303 super::backup_configuration::BackupConfiguration::get_fetchable_from_row(
304 row,
305 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
306 ),
307 node: super::node::Node::get_fetchable(
308 row.try_get(compact_str::format_compact!("{prefix}node_uuid").as_str())?,
309 ),
310 backup_group_uuid: row
311 .try_get(compact_str::format_compact!("{prefix}backup_group_uuid").as_str())?,
312 system_backup_policy_uuid: row.try_get(
313 compact_str::format_compact!("{prefix}system_backup_policy_uuid").as_str(),
314 )?,
315 database_instance_uuid: row
316 .try_get(compact_str::format_compact!("{prefix}database_instance_uuid").as_str())?,
317 kind: row.try_get(compact_str::format_compact!("{prefix}kind").as_str())?,
318 database_type: row
319 .try_get(compact_str::format_compact!("{prefix}database_type").as_str())?,
320 name: row.try_get(compact_str::format_compact!("{prefix}name").as_str())?,
321 successful: row.try_get(compact_str::format_compact!("{prefix}successful").as_str())?,
322 browsable: row.try_get(compact_str::format_compact!("{prefix}browsable").as_str())?,
323 streaming: row.try_get(compact_str::format_compact!("{prefix}streaming").as_str())?,
324 locked: row.try_get(compact_str::format_compact!("{prefix}locked").as_str())?,
325 shared: row.try_get(compact_str::format_compact!("{prefix}shared").as_str())?,
326 ignored_files: row
327 .try_get(compact_str::format_compact!("{prefix}ignored_files").as_str())?,
328 checksum: row.try_get(compact_str::format_compact!("{prefix}checksum").as_str())?,
329 bytes: row.try_get(compact_str::format_compact!("{prefix}bytes").as_str())?,
330 files: row.try_get(compact_str::format_compact!("{prefix}files").as_str())?,
331 disk: row.try_get(compact_str::format_compact!("{prefix}disk").as_str())?,
332 upload_id: row.try_get(compact_str::format_compact!("{prefix}upload_id").as_str())?,
333 upload_path: row
334 .try_get(compact_str::format_compact!("{prefix}upload_path").as_str())?,
335 metadata: row.try_get(compact_str::format_compact!("{prefix}metadata").as_str())?,
336 completed: row.try_get(compact_str::format_compact!("{prefix}completed").as_str())?,
337 deleting: row.try_get(compact_str::format_compact!("{prefix}deleting").as_str())?,
338 deletion_retries: row
339 .try_get(compact_str::format_compact!("{prefix}deletion_retries").as_str())?,
340 deleted: row.try_get(compact_str::format_compact!("{prefix}deleted").as_str())?,
341 created: row.try_get(compact_str::format_compact!("{prefix}created").as_str())?,
342 extension_data: Self::map_extensions(prefix, row)?,
343 })
344 }
345
346 fn cache_invalidation_keys(&self) -> Vec<compact_str::CompactString> {
347 vec![compact_str::format_compact!(
348 "{}::{}",
349 Self::NAME,
350 self.uuid
351 )]
352 }
353}
354
355impl ServerBackup {
356 pub async fn create_raw(
357 state: &crate::State,
358 mut options: CreateServerBackupOptions<'_>,
359 ) -> Result<Self, anyhow::Error> {
360 let backup_configuration = match options.backup_configuration.take() {
361 Some(backup_configuration) => backup_configuration,
362 None => options
363 .server
364 .backup_configuration(&state.database)
365 .await
366 .ok_or_else(|| {
367 crate::response::DisplayError::new(
368 "no backup configuration available, unable to create backup",
369 )
370 .with_status(StatusCode::EXPECTATION_FAILED)
371 })?,
372 };
373
374 if backup_configuration.maintenance_enabled {
375 return Err(crate::response::DisplayError::new(
376 "cannot create backup while backup configuration is in maintenance mode",
377 )
378 .with_status(StatusCode::EXPECTATION_FAILED)
379 .into());
380 }
381
382 if options.database_instance.is_some()
383 && matches!(
384 backup_configuration.backup_disk,
385 BackupDisk::Btrfs | BackupDisk::Zfs
386 )
387 {
388 return Err(crate::response::DisplayError::new(
389 "database backups cannot be created on a btrfs or zfs backup configuration",
390 )
391 .with_status(StatusCode::EXPECTATION_FAILED)
392 .into());
393 }
394
395 let mut transaction = state.database.write().begin().await?;
396
397 let mut query_builder = InsertQueryBuilder::new("server_backups");
398
399 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
400 .await?;
401
402 query_builder
403 .set("server_uuid", options.server.uuid)
404 .set("node_uuid", options.server.node.uuid)
405 .set("backup_configuration_uuid", backup_configuration.uuid)
406 .set("backup_group_uuid", options.backup_group_uuid)
407 .set(
408 "system_backup_policy_uuid",
409 options.system_backup_policy_uuid,
410 )
411 .set(
412 "database_instance_uuid",
413 options.database_instance.map(|instance| instance.uuid),
414 )
415 .set("kind", options.kind())
416 .set(
417 "database_type",
418 options.database_instance.map(|instance| instance.r#type),
419 )
420 .set("name", &options.name)
421 .set("ignored_files", &options.ignored_files)
422 .set("bytes", 0i64)
423 .set("disk", backup_configuration.backup_disk)
424 .set("shared", backup_configuration.shared)
425 .set("metadata", &options.metadata);
426
427 let row = query_builder
428 .returning(&Self::columns_sql(None))
429 .fetch_one(&mut *transaction)
430 .await?;
431 let mut backup = Self::map(None, &row)?;
432
433 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
434
435 transaction.commit().await?;
436
437 Ok(backup)
438 }
439
440 pub async fn by_server_uuid_uuid(
441 database: &crate::database::Database,
442 server_uuid: uuid::Uuid,
443 uuid: uuid::Uuid,
444 ) -> Result<Option<Self>, crate::database::DatabaseError> {
445 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
446 r#"
447 SELECT {}
448 FROM server_backups
449 WHERE server_backups.server_uuid = $1 AND server_backups.uuid = $2
450 "#,
451 Self::columns_sql(None)
452 )))
453 .bind(server_uuid)
454 .bind(uuid)
455 .fetch_optional(database.read())
456 .await?;
457
458 row.try_map(|row| Self::map(None, &row))
459 }
460
461 pub async fn select_completed_by_server_uuid(
462 database: &crate::database::Database,
463 server_uuid: uuid::Uuid,
464 name: Option<&str>,
465 backup_group_uuid: Option<uuid::Uuid>,
466 filter: &ServerBackupFilter,
467 oldest: bool,
468 ) -> Result<Option<Self>, crate::database::DatabaseError> {
469 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
470 r#"
471 SELECT {}
472 FROM server_backups
473 WHERE
474 server_backups.server_uuid = $1
475 AND server_backups.deleted IS NULL
476 AND server_backups.deleting IS NULL
477 AND server_backups.completed IS NOT NULL
478 AND server_backups.successful
479 AND server_backups.system_backup_policy_uuid IS NULL
480 AND ($2 IS NULL OR server_backups.name = $2)
481 AND ($3::uuid IS NULL OR server_backups.backup_group_uuid = $3)
482 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
483 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
484 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
485 ORDER BY server_backups.created {1}, server_backups.uuid {1}
486 LIMIT 1
487 "#,
488 Self::columns_sql(None),
489 if oldest { "ASC" } else { "DESC" }
490 )))
491 .bind(server_uuid)
492 .bind(name)
493 .bind(backup_group_uuid)
494 .bind(filter.kind)
495 .bind(filter.database_instance_uuid)
496 .bind(filter.database_type)
497 .fetch_optional(database.read())
498 .await?;
499
500 row.try_map(|row| Self::map(None, &row))
501 }
502
503 pub async fn by_node_uuid_uuid(
504 database: &crate::database::Database,
505 node_uuid: uuid::Uuid,
506 uuid: uuid::Uuid,
507 ) -> Result<Option<Self>, crate::database::DatabaseError> {
508 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
509 r#"
510 SELECT {}
511 FROM server_backups
512 WHERE server_backups.node_uuid = $1 AND server_backups.uuid = $2
513 "#,
514 Self::columns_sql(None)
515 )))
516 .bind(node_uuid)
517 .bind(uuid)
518 .fetch_optional(database.read())
519 .await?;
520
521 row.try_map(|row| Self::map(None, &row))
522 }
523
524 pub async fn by_server_uuid_with_pagination(
525 database: &crate::database::Database,
526 server_uuid: uuid::Uuid,
527 page: i64,
528 per_page: i64,
529 search: Option<&str>,
530 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
531 super::PaginatedQueryBuilder::new(
532 Self::columns_sql(None),
533 format!(
534 r#"
535 FROM server_backups
536 WHERE
537 server_backups.server_uuid = $1
538 AND server_backups.deleted IS NULL
539 AND {search}
540 "#,
541 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
542 ),
543 "server_backups.created, server_backups.uuid",
544 )
545 .bind(server_uuid)
546 .bind(search)
547 .fetch(database, page, per_page)
548 .await
549 }
550
551 pub async fn by_server_uuid_node_uuid_with_pagination(
552 database: &crate::database::Database,
553 server_uuid: uuid::Uuid,
554 node_uuid: uuid::Uuid,
555 page: i64,
556 per_page: i64,
557 search: Option<&str>,
558 filter: &ServerBackupFilter,
559 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
560 super::PaginatedQueryBuilder::new(
561 Self::columns_sql(None),
562 format!(
563 r#"
564 FROM server_backups
565 WHERE
566 server_backups.server_uuid = $1
567 AND server_backups.node_uuid = $2
568 AND server_backups.system_backup_policy_uuid IS NULL
569 AND server_backups.deleted IS NULL
570 AND {search}
571 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
572 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
573 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
574 "#,
575 search = super::search_sql(3, &["server_backups.name"], &["server_backups.uuid"]),
576 ),
577 "server_backups.created, server_backups.uuid",
578 )
579 .bind(server_uuid)
580 .bind(node_uuid)
581 .bind(search)
582 .bind(filter.kind)
583 .bind(filter.database_instance_uuid)
584 .bind(filter.database_type)
585 .fetch(database, page, per_page)
586 .await
587 }
588
589 pub async fn by_server_uuid_node_uuid_backup_group_uuid_with_pagination(
590 database: &crate::database::Database,
591 server_uuid: uuid::Uuid,
592 node_uuid: uuid::Uuid,
593 backup_group_uuid: uuid::Uuid,
594 page: i64,
595 per_page: i64,
596 search: Option<&str>,
597 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
598 super::PaginatedQueryBuilder::new(
599 Self::columns_sql(None),
600 format!(
601 r#"
602 FROM server_backups
603 WHERE
604 server_backups.server_uuid = $1
605 AND server_backups.node_uuid = $2
606 AND server_backups.backup_group_uuid = $3
607 AND server_backups.deleted IS NULL
608 AND {search}
609 "#,
610 search = super::search_sql(4, &["server_backups.name"], &["server_backups.uuid"]),
611 ),
612 "server_backups.created, server_backups.uuid",
613 )
614 .bind(server_uuid)
615 .bind(node_uuid)
616 .bind(backup_group_uuid)
617 .bind(search)
618 .fetch(database, page, per_page)
619 .await
620 }
621
622 pub async fn by_ungrouped_server_uuid_node_uuid_with_pagination(
623 database: &crate::database::Database,
624 server_uuid: uuid::Uuid,
625 node_uuid: uuid::Uuid,
626 page: i64,
627 per_page: i64,
628 search: Option<&str>,
629 filter: &ServerBackupFilter,
630 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
631 super::PaginatedQueryBuilder::new(
632 Self::columns_sql(None),
633 format!(
634 r#"
635 FROM server_backups
636 WHERE
637 server_backups.server_uuid = $1
638 AND server_backups.node_uuid = $2
639 AND server_backups.backup_group_uuid IS NULL
640 AND server_backups.system_backup_policy_uuid IS NULL
641 AND server_backups.deleted IS NULL
642 AND {search}
643 AND ($4::server_backup_kind IS NULL OR server_backups.kind = $4)
644 AND ($5::uuid IS NULL OR server_backups.database_instance_uuid = $5)
645 AND ($6::database_agent_type IS NULL OR server_backups.database_type = $6)
646 "#,
647 search = super::search_sql(3, &["server_backups.name"], &["server_backups.uuid"]),
648 ),
649 "server_backups.created, server_backups.uuid",
650 )
651 .bind(server_uuid)
652 .bind(node_uuid)
653 .bind(search)
654 .bind(filter.kind)
655 .bind(filter.database_instance_uuid)
656 .bind(filter.database_type)
657 .fetch(database, page, per_page)
658 .await
659 }
660
661 pub async fn usage_by_server_uuid(
662 database: &crate::database::Database,
663 server_uuid: uuid::Uuid,
664 ) -> Result<ServerBackupUsage, sqlx::Error> {
665 let row = sqlx::query!(
666 r#"
667 SELECT
668 COUNT(*) FILTER (WHERE server_backups.kind = 'SERVER') AS "server!",
669 COUNT(*) FILTER (WHERE server_backups.kind = 'DATABASE_INSTANCE') AS "database_instance!"
670 FROM server_backups
671 WHERE
672 server_backups.server_uuid = $1
673 AND server_backups.system_backup_policy_uuid IS NULL
674 AND server_backups.deleted IS NULL
675 "#,
676 server_uuid
677 )
678 .fetch_one(database.read())
679 .await?;
680
681 Ok(ServerBackupUsage {
682 server: row.server,
683 database_instance: row.database_instance,
684 })
685 }
686
687 pub async fn by_system_server_uuid_node_uuid_with_pagination(
688 database: &crate::database::Database,
689 server_uuid: uuid::Uuid,
690 node_uuid: uuid::Uuid,
691 page: i64,
692 per_page: i64,
693 search: Option<&str>,
694 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
695 super::PaginatedQueryBuilder::new(
696 Self::columns_sql(None),
697 format!(
698 r#"
699 FROM server_backups
700 WHERE
701 server_backups.server_uuid = $1
702 AND server_backups.node_uuid = $2
703 AND server_backups.system_backup_policy_uuid IS NOT NULL
704 AND server_backups.deleted IS NULL
705 AND {search}
706 "#,
707 search = super::search_sql(3, &["server_backups.name"], &["server_backups.uuid"]),
708 ),
709 "server_backups.created, server_backups.uuid",
710 )
711 .bind(server_uuid)
712 .bind(node_uuid)
713 .bind(search)
714 .fetch(database, page, per_page)
715 .await
716 }
717
718 pub async fn by_system_backup_policy_uuid_with_pagination(
719 database: &crate::database::Database,
720 system_backup_policy_uuid: uuid::Uuid,
721 page: i64,
722 per_page: i64,
723 search: Option<&str>,
724 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
725 super::PaginatedQueryBuilder::new(
726 Self::columns_sql(None),
727 format!(
728 r#"
729 FROM server_backups
730 WHERE
731 server_backups.system_backup_policy_uuid = $1
732 AND server_backups.deleted IS NULL
733 AND {search}
734 "#,
735 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
736 ),
737 "server_backups.created, server_backups.uuid",
738 )
739 .bind(system_backup_policy_uuid)
740 .bind(search)
741 .fetch(database, page, per_page)
742 .await
743 }
744
745 pub async fn all_by_system_backup_policy_uuid(
746 database: &crate::database::Database,
747 system_backup_policy_uuid: uuid::Uuid,
748 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
749 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
750 r#"
751 SELECT {}
752 FROM server_backups
753 WHERE
754 server_backups.system_backup_policy_uuid = $1
755 AND server_backups.deleted IS NULL
756 "#,
757 Self::columns_sql(None)
758 )))
759 .bind(system_backup_policy_uuid)
760 .fetch_all(database.read())
761 .await?;
762
763 rows.into_iter()
764 .map(|row| Self::map(None, &row))
765 .try_collect_vec()
766 }
767
768 pub async fn by_partially_detached_server_uuid_node_uuid_with_pagination(
769 database: &crate::database::Database,
770 server_uuid: uuid::Uuid,
771 node_uuid: uuid::Uuid,
772 page: i64,
773 per_page: i64,
774 search: Option<&str>,
775 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
776 super::PaginatedQueryBuilder::new(
777 Self::columns_sql(None),
778 format!(
779 r#"
780 FROM server_backups
781 WHERE
782 server_backups.server_uuid = $1
783 AND server_backups.node_uuid != $2
784 AND server_backups.deleted IS NULL
785 AND {search}
786 "#,
787 search = super::search_sql(3, &["server_backups.name"], &["server_backups.uuid"]),
788 ),
789 "server_backups.created, server_backups.uuid",
790 )
791 .bind(server_uuid)
792 .bind(node_uuid)
793 .bind(search)
794 .fetch(database, page, per_page)
795 .await
796 }
797
798 pub async fn by_node_uuid_with_pagination(
799 database: &crate::database::Database,
800 node_uuid: uuid::Uuid,
801 page: i64,
802 per_page: i64,
803 search: Option<&str>,
804 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
805 super::PaginatedQueryBuilder::new(
806 Self::columns_sql(None),
807 format!(
808 r#"
809 FROM server_backups
810 WHERE
811 server_backups.node_uuid = $1
812 AND server_backups.deleted IS NULL
813 AND {search}
814 "#,
815 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
816 ),
817 "server_backups.created, server_backups.uuid",
818 )
819 .bind(node_uuid)
820 .bind(search)
821 .fetch(database, page, per_page)
822 .await
823 }
824
825 pub async fn by_database_agent_host_uuid_with_pagination(
826 database: &crate::database::Database,
827 database_agent_host_uuid: uuid::Uuid,
828 page: i64,
829 per_page: i64,
830 search: Option<&str>,
831 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
832 super::PaginatedQueryBuilder::new(
833 Self::columns_sql(None),
834 format!(
835 r#"
836 FROM server_backups
837 JOIN server_database_instances ON server_database_instances.uuid = server_backups.database_instance_uuid
838 WHERE
839 server_database_instances.database_agent_host_uuid = $1
840 AND server_backups.deleted IS NULL
841 AND {search}
842 "#,
843 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
844 ),
845 "server_backups.created, server_backups.uuid",
846 )
847 .bind(database_agent_host_uuid)
848 .bind(search)
849 .fetch(database, page, per_page)
850 .await
851 }
852
853 pub async fn by_backup_configuration_uuid_with_pagination(
854 database: &crate::database::Database,
855 backup_configuration_uuid: uuid::Uuid,
856 page: i64,
857 per_page: i64,
858 search: Option<&str>,
859 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
860 super::PaginatedQueryBuilder::new(
861 Self::columns_sql(None),
862 format!(
863 r#"
864 FROM server_backups
865 WHERE
866 server_backups.backup_configuration_uuid = $1
867 AND server_backups.deleted IS NULL
868 AND {search}
869 "#,
870 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
871 ),
872 "server_backups.created, server_backups.uuid",
873 )
874 .bind(backup_configuration_uuid)
875 .bind(search)
876 .fetch(database, page, per_page)
877 .await
878 }
879
880 pub async fn by_detached_node_uuid_with_pagination(
881 database: &crate::database::Database,
882 node_uuid: uuid::Uuid,
883 page: i64,
884 per_page: i64,
885 search: Option<&str>,
886 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
887 super::PaginatedQueryBuilder::new(
888 Self::columns_sql(None),
889 format!(
890 r#"
891 FROM server_backups
892 WHERE
893 server_backups.node_uuid = $1
894 AND server_backups.server_uuid IS NULL
895 AND server_backups.deleted IS NULL
896 AND {search}
897 "#,
898 search = super::search_sql(2, &["server_backups.name"], &["server_backups.uuid"]),
899 ),
900 "server_backups.created, server_backups.uuid",
901 )
902 .bind(node_uuid)
903 .bind(search)
904 .fetch(database, page, per_page)
905 .await
906 }
907
908 pub async fn all_uuids_by_server_uuid(
909 database: &crate::database::Database,
910 server_uuid: uuid::Uuid,
911 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
912 let rows = sqlx::query(
913 r#"
914 SELECT server_backups.uuid
915 FROM server_backups
916 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
917 "#,
918 )
919 .bind(server_uuid)
920 .fetch_all(database.read())
921 .await?;
922
923 Ok(rows
924 .into_iter()
925 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
926 .collect())
927 }
928
929 pub async fn all_uuids_by_server_uuid_not_shared(
930 database: &crate::database::Database,
931 server_uuid: uuid::Uuid,
932 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
933 let rows = sqlx::query(
934 r#"
935 SELECT server_backups.uuid
936 FROM server_backups
937 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL AND server_backups.shared = false
938 "#,
939 )
940 .bind(server_uuid)
941 .fetch_all(database.read())
942 .await?;
943
944 Ok(rows
945 .into_iter()
946 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
947 .collect())
948 }
949
950 pub async fn all_by_server_uuid(
951 database: &crate::database::Database,
952 server_uuid: uuid::Uuid,
953 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
954 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
955 r#"
956 SELECT {}
957 FROM server_backups
958 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
959 "#,
960 Self::columns_sql(None)
961 )))
962 .bind(server_uuid)
963 .fetch_all(database.read())
964 .await?;
965
966 rows.into_iter()
967 .map(|row| Self::map(None, &row))
968 .try_collect_vec()
969 }
970
971 pub async fn count_by_server_uuid(
972 database: &crate::database::Database,
973 server_uuid: uuid::Uuid,
974 ) -> Result<i64, sqlx::Error> {
975 sqlx::query_scalar(
976 r#"
977 SELECT COUNT(*)
978 FROM server_backups
979 WHERE
980 server_backups.server_uuid = $1
981 AND server_backups.system_backup_policy_uuid IS NULL
982 AND server_backups.deleted IS NULL
983 "#,
984 )
985 .bind(server_uuid)
986 .fetch_one(database.read())
987 .await
988 }
989
990 pub async fn count_system_inflight_by_system_backup_policy_uuid_node_uuid(
994 database: &crate::database::Database,
995 system_backup_policy_uuid: uuid::Uuid,
996 node_uuid: uuid::Uuid,
997 ) -> Result<i64, sqlx::Error> {
998 sqlx::query_scalar(
999 r#"
1000 SELECT COUNT(*)
1001 FROM server_backups
1002 WHERE
1003 server_backups.system_backup_policy_uuid = $1
1004 AND server_backups.node_uuid = $2
1005 AND server_backups.completed IS NULL
1006 AND server_backups.deleted IS NULL
1007 AND server_backups.created >= NOW() - INTERVAL '1 day'
1008 "#,
1009 )
1010 .bind(system_backup_policy_uuid)
1011 .bind(node_uuid)
1012 .fetch_one(database.read())
1013 .await
1014 }
1015
1016 pub async fn count_system_inflight_by_system_backup_policy_uuid_database_agent_host_uuid(
1019 database: &crate::database::Database,
1020 system_backup_policy_uuid: uuid::Uuid,
1021 database_agent_host_uuid: uuid::Uuid,
1022 ) -> Result<i64, sqlx::Error> {
1023 sqlx::query_scalar(
1024 r#"
1025 SELECT COUNT(*)
1026 FROM server_backups
1027 JOIN server_database_instances ON server_database_instances.uuid = server_backups.database_instance_uuid
1028 WHERE
1029 server_backups.system_backup_policy_uuid = $1
1030 AND server_database_instances.database_agent_host_uuid = $2
1031 AND server_backups.completed IS NULL
1032 AND server_backups.deleted IS NULL
1033 AND server_backups.created >= NOW() - INTERVAL '1 day'
1034 "#,
1035 )
1036 .bind(system_backup_policy_uuid)
1037 .bind(database_agent_host_uuid)
1038 .fetch_one(database.read())
1039 .await
1040 }
1041
1042 pub async fn generate_metadata(
1043 state: &crate::State,
1044 server: &super::server::Server,
1045 ) -> Result<serde_json::Value, anyhow::Error> {
1046 let mut variables = serde_json::Map::new();
1047
1048 for variable in ServerVariable::all_by_server_uuid_egg_uuid(
1049 &state.database,
1050 server.uuid,
1051 server.egg.uuid,
1052 )
1053 .await?
1054 {
1055 variables.insert(variable.variable.env_variable.into(), variable.value.into());
1056 }
1057
1058 Ok(serde_json::json!({
1059 "startup": server.startup,
1060 "image": server.image,
1061 "variables": variables,
1062 }))
1063 }
1064
1065 #[inline]
1066 pub fn generate_database_metadata(
1067 database_instance: &super::server_database_instance::ServerDatabaseInstance,
1068 ) -> serde_json::Value {
1069 serde_json::json!({
1070 "source_instance": {
1071 "uuid": database_instance.uuid,
1072 "name": database_instance.name,
1073 },
1074 "image": database_instance.image,
1075 "template_version": database_instance.template_version,
1076 })
1077 }
1078
1079 #[inline]
1081 pub fn database_dump_name(&self) -> compact_str::CompactString {
1082 compact_str::format_compact!(
1083 "{}.{}",
1084 self.uuid,
1085 self.database_type
1086 .map_or("dump", db_agent_api::DatabaseAgentType::dump_extension)
1087 )
1088 }
1089
1090 pub async fn download_url(
1091 &self,
1092 state: &crate::State,
1093 request_host: Option<&str>,
1094 user: &super::user::User,
1095 credential: Option<uuid::Uuid>,
1096 node: &super::node::Node,
1097 archive_format: wings_api::StreamableArchiveFormat,
1098 ) -> Result<String, anyhow::Error> {
1099 let backup_configuration = self
1100 .backup_configuration
1101 .as_ref()
1102 .ok_or_else(|| {
1103 crate::response::DisplayError::new(
1104 "no backup configuration available, unable to restore backup",
1105 )
1106 .with_status(StatusCode::EXPECTATION_FAILED)
1107 })?
1108 .fetch_cached(&state.database)
1109 .await?;
1110
1111 if backup_configuration.maintenance_enabled {
1112 return Err(crate::response::DisplayError::new(
1113 "cannot restore backup while backup configuration is in maintenance mode",
1114 )
1115 .with_status(StatusCode::EXPECTATION_FAILED)
1116 .into());
1117 }
1118
1119 if matches!(self.disk, BackupDisk::S3)
1120 && let Some(mut s3_configuration) = backup_configuration.backup_configs.s3
1121 {
1122 s3_configuration.decrypt(&state.database).await?;
1123
1124 let (client, bucket) = s3_configuration.into_client();
1125
1126 let file_path = match &self.upload_path {
1127 Some(path) => path,
1128 None => {
1129 return Err(crate::response::DisplayError::new(
1130 "backup does not have an upload path",
1131 )
1132 .with_status(StatusCode::EXPECTATION_FAILED)
1133 .into());
1134 }
1135 };
1136
1137 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1138 std::time::Duration::from_mins(15),
1139 )?;
1140 let presigned = client
1141 .get_object()
1142 .bucket(bucket)
1143 .key(&**file_path)
1144 .presigned(presigning_config)
1145 .await?;
1146
1147 return Ok(presigned.uri().to_string());
1148 }
1149
1150 #[derive(Serialize)]
1151 struct BackupDownloadJwt {
1152 #[serde(flatten)]
1153 base: BasePayload,
1154
1155 backup_uuid: uuid::Uuid,
1156 unique_id: uuid::Uuid,
1157 database: bool,
1158 credential: Option<uuid::Uuid>,
1159 }
1160
1161 let token = node
1162 .create_jwt(
1163 &state.database,
1164 &state.jwt,
1165 &BackupDownloadJwt {
1166 base: BasePayload {
1167 scope: "backup-download".into(),
1168 issuer: "panel".into(),
1169 subject: Some(user.uuid.to_compact_string()),
1170 audience: Vec::new(),
1171 expiration_time: Some(chrono::Utc::now().timestamp() + 900),
1172 not_before: None,
1173 issued_at: Some(chrono::Utc::now().timestamp()),
1174 jwt_id: uuid::Uuid::new_v4().to_compact_string(),
1175 },
1176 backup_uuid: self.uuid,
1177 unique_id: uuid::Uuid::new_v4(),
1178 database: self.kind == ServerBackupKind::DatabaseInstance,
1179 credential,
1180 },
1181 )
1182 .await?;
1183
1184 let mut url = node
1185 .public_url(state, request_host, "/download/backup")
1186 .await?;
1187 url.set_query(Some(&format!(
1188 "token={}&archive_format={}",
1189 urlencoding::encode(&token),
1190 archive_format
1191 )));
1192
1193 Ok(url.to_string())
1194 }
1195
1196 pub async fn restore(
1197 self,
1198 state: &crate::State,
1199 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1200 mut server: super::server::Server,
1201 options: ServerBackupRestoreOptions,
1202 ) -> Result<(), anyhow::Error> {
1203 if self.kind != ServerBackupKind::Server {
1204 return Err(crate::response::DisplayError::new(
1205 "database instance backups cannot be restored to server files",
1206 )
1207 .with_status(StatusCode::EXPECTATION_FAILED)
1208 .into());
1209 }
1210
1211 let backup_configuration = self
1212 .backup_configuration
1213 .as_ref()
1214 .ok_or_else(|| {
1215 crate::response::DisplayError::new(
1216 "no backup configuration available, unable to restore backup",
1217 )
1218 .with_status(StatusCode::EXPECTATION_FAILED)
1219 })?
1220 .fetch_cached(&state.database)
1221 .await?;
1222
1223 if backup_configuration.maintenance_enabled {
1224 return Err(crate::response::DisplayError::new(
1225 "cannot restore backup while backup configuration is in maintenance mode",
1226 )
1227 .with_status(StatusCode::EXPECTATION_FAILED)
1228 .into());
1229 }
1230
1231 if options.restore_startup {
1232 self.restore_startup(state, transaction, &mut server)
1233 .await?;
1234 }
1235
1236 server
1237 .node
1238 .fetch_cached(&state.database)
1239 .await?
1240 .api_client(&state.database)
1241 .await?
1242 .post_servers_server_backup_backup_restore(
1243 server.uuid,
1244 self.uuid,
1245 &wings_api::servers_server_backup_backup_restore::post::RequestBody {
1246 adapter: self.disk.to_wings_adapter(),
1247 download_url: self.wings_restore_download_url(state, server.uuid).await?,
1248 truncate_directory: options.truncate_directory,
1249 },
1250 )
1251 .await?;
1252
1253 Self::get_event_emitter().emit(
1254 state.clone(),
1255 ServerBackupEvent::RestoreStarted {
1256 backup: Box::new(self),
1257 server: Box::new(server),
1258 },
1259 );
1260
1261 Ok(())
1262 }
1263
1264 pub async fn restore_database(
1265 self,
1266 state: &crate::State,
1267 server: super::server::Server,
1268 database_instance: &super::server_database_instance::ServerDatabaseInstance,
1269 request_uuid: Option<uuid::Uuid>,
1270 ) -> Result<(), anyhow::Error> {
1271 if self.kind != ServerBackupKind::DatabaseInstance {
1272 return Err(crate::response::DisplayError::new(
1273 "only database instance backups can be restored into a database instance",
1274 )
1275 .with_status(StatusCode::EXPECTATION_FAILED)
1276 .into());
1277 }
1278
1279 if self.server.as_ref().map(|server| server.uuid) != Some(database_instance.server.uuid) {
1280 return Err(crate::response::DisplayError::new(
1281 "backup does not belong to this database instance's server",
1282 )
1283 .with_status(StatusCode::EXPECTATION_FAILED)
1284 .into());
1285 }
1286
1287 if self.database_type != Some(database_instance.r#type) {
1288 return Err(crate::response::DisplayError::new(
1289 "backup was taken from a different database engine",
1290 )
1291 .with_status(StatusCode::EXPECTATION_FAILED)
1292 .into());
1293 }
1294
1295 let backup_configuration = self
1296 .backup_configuration
1297 .as_ref()
1298 .ok_or_else(|| {
1299 crate::response::DisplayError::new(
1300 "no backup configuration available, unable to restore backup",
1301 )
1302 .with_status(StatusCode::EXPECTATION_FAILED)
1303 })?
1304 .fetch_cached(&state.database)
1305 .await?;
1306
1307 if backup_configuration.maintenance_enabled {
1308 return Err(crate::response::DisplayError::new(
1309 "cannot restore backup while backup configuration is in maintenance mode",
1310 )
1311 .with_status(StatusCode::EXPECTATION_FAILED)
1312 .into());
1313 }
1314
1315 server
1316 .node
1317 .fetch_cached(&state.database)
1318 .await?
1319 .api_client(&state.database)
1320 .await?
1321 .post_servers_server_database_backup_backup_restore(
1322 server.uuid,
1323 self.uuid,
1324 &wings_api::servers_server_database_backup_backup_restore::post::RequestBody {
1325 adapter: self.disk.to_wings_adapter(),
1326 database_instance: database_instance.uuid,
1327 download_url: self.wings_restore_download_url(state, server.uuid).await?,
1328 request_uuid,
1329 },
1330 )
1331 .await?;
1332
1333 Self::get_event_emitter().emit(
1334 state.clone(),
1335 ServerBackupEvent::RestoreStarted {
1336 backup: Box::new(self),
1337 server: Box::new(server),
1338 },
1339 );
1340
1341 Ok(())
1342 }
1343
1344 pub async fn restore_startup(
1345 &self,
1346 state: &crate::State,
1347 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1348 server: &mut super::server::Server,
1349 ) -> Result<(), anyhow::Error> {
1350 let startup_cmd = self
1351 .metadata
1352 .get("startup")
1353 .and_then(|v| v.as_str())
1354 .map(|s| s.to_compact_string());
1355 let image_str = self
1356 .metadata
1357 .get("image")
1358 .and_then(|v| v.as_str())
1359 .map(|s| s.to_compact_string());
1360 let variables = self
1361 .metadata
1362 .get("variables")
1363 .and_then(|v| v.as_object())
1364 .cloned()
1365 .unwrap_or_default();
1366
1367 if let Some(startup) = startup_cmd
1368 && let Ok(egg_config) = server.egg_configuration(&state.database).await
1369 {
1370 let is_predefined = server
1371 .egg
1372 .startup_commands
1373 .values()
1374 .any(|cmd| cmd == startup.as_str());
1375 let custom_allowed = egg_config
1376 .config_startup
1377 .as_ref()
1378 .is_some_and(|c| c.allow_custom_startup_command);
1379 if is_predefined || custom_allowed {
1380 server
1381 .update_with_transaction(
1382 state,
1383 super::server::UpdateServerOptions {
1384 startup: Some(startup),
1385 ..Default::default()
1386 },
1387 transaction,
1388 )
1389 .await?;
1390 }
1391 }
1392
1393 if let Some(image) = image_str {
1394 let is_valid_image = server
1395 .egg
1396 .docker_images
1397 .values()
1398 .any(|img| img == image.as_str());
1399 if is_valid_image {
1400 let current_is_custom = !server
1401 .egg
1402 .docker_images
1403 .values()
1404 .any(|img| img == server.image.as_str());
1405 let allow_overwrite = state
1406 .settings
1407 .get()
1408 .await
1409 .map(|s| s.server.allow_overwriting_custom_docker_image)
1410 .unwrap_or(false);
1411 if !current_is_custom || allow_overwrite {
1412 server
1413 .update_with_transaction(
1414 state,
1415 super::server::UpdateServerOptions {
1416 image: Some(image),
1417 ..Default::default()
1418 },
1419 transaction,
1420 )
1421 .await?;
1422 }
1423 }
1424 }
1425
1426 if !variables.is_empty() {
1427 let existing_variables = ServerVariable::all_by_server_uuid_egg_uuid(
1428 &state.database,
1429 server.uuid,
1430 server.egg.uuid,
1431 )
1432 .await?;
1433
1434 let mut validator_variables = HashMap::new();
1435 for variable in existing_variables.iter() {
1436 validator_variables.insert(
1437 variable.variable.env_variable.as_str(),
1438 (
1439 variable.variable.rules.as_slice(),
1440 if let Some(value) = variables
1441 .iter()
1442 .find(|v| v.0 == variable.variable.env_variable)
1443 && variable.variable.user_editable
1444 && let Some(value) = value.1.as_str()
1445 {
1446 value
1447 } else {
1448 variable.value.as_str()
1449 },
1450 ),
1451 );
1452 }
1453
1454 let validator = match rule_validator::Validator::new(validator_variables) {
1455 Ok(validator) => validator,
1456 Err(error) => {
1457 return Err(crate::response::DisplayError::new(error)
1458 .with_status(StatusCode::EXPECTATION_FAILED)
1459 .into());
1460 }
1461 };
1462 if let Err(error) = validator.validate() {
1463 return Err(crate::response::DisplayError::new(error)
1464 .with_status(StatusCode::EXPECTATION_FAILED)
1465 .into());
1466 }
1467
1468 for (env_var, value) in &variables {
1469 let Some(value) = value.as_str() else {
1470 continue;
1471 };
1472 let variable_uuid = match existing_variables
1473 .iter()
1474 .find(|v| v.variable.env_variable == env_var)
1475 {
1476 Some(variable) if variable.variable.user_editable => variable.variable.uuid,
1477 _ => continue,
1478 };
1479
1480 ServerVariable::create_with_transaction(
1481 transaction,
1482 server.uuid,
1483 variable_uuid,
1484 value,
1485 )
1486 .await?;
1487 }
1488 }
1489
1490 Ok(())
1491 }
1492
1493 pub async fn wings_restore_download_url(
1494 &self,
1495 state: &crate::State,
1496 server_uuid: uuid::Uuid,
1497 ) -> Result<Option<compact_str::CompactString>, anyhow::Error> {
1498 if !matches!(self.disk, BackupDisk::S3) {
1499 return Ok(None);
1500 }
1501
1502 let backup_configuration = self
1503 .backup_configuration
1504 .as_ref()
1505 .ok_or_else(|| {
1506 crate::response::DisplayError::new(
1507 "no backup configuration available, unable to restore backup",
1508 )
1509 .with_status(StatusCode::EXPECTATION_FAILED)
1510 })?
1511 .fetch_cached(&state.database)
1512 .await?;
1513
1514 let Some(mut s3_configuration) = backup_configuration.backup_configs.s3 else {
1515 return Ok(None);
1516 };
1517
1518 s3_configuration.decrypt(&state.database).await?;
1519
1520 let compression_type = s3_configuration.compression_type;
1521 let (client, bucket) = s3_configuration.into_client();
1522
1523 let file_path = match &self.upload_path {
1524 Some(path) => path.as_str(),
1525 None => &self.s3_path(server_uuid, compression_type),
1526 };
1527
1528 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1529 std::time::Duration::from_mins(60),
1530 )?;
1531 let presigned = client
1532 .get_object()
1533 .bucket(bucket)
1534 .key(file_path)
1535 .presigned(presigning_config)
1536 .await?;
1537
1538 Ok(Some(presigned.uri().to_compact_string()))
1539 }
1540
1541 pub async fn export(
1542 &self,
1543 state: &crate::State,
1544 server: &super::server::Server,
1545 path: compact_str::CompactString,
1546 archive_format: wings_api::StreamableArchiveFormat,
1547 foreground: bool,
1548 ) -> Result<wings_api::backups_backup_export::post::Response, anyhow::Error> {
1549 let backup_configuration = self
1550 .backup_configuration
1551 .as_ref()
1552 .ok_or_else(|| {
1553 crate::response::DisplayError::new(
1554 "no backup configuration available, unable to export backup",
1555 )
1556 .with_status(StatusCode::EXPECTATION_FAILED)
1557 })?
1558 .fetch_cached(&state.database)
1559 .await?;
1560
1561 if backup_configuration.maintenance_enabled {
1562 return Err(crate::response::DisplayError::new(
1563 "cannot export backup while backup configuration is in maintenance mode",
1564 )
1565 .with_status(StatusCode::EXPECTATION_FAILED)
1566 .into());
1567 }
1568
1569 let client = server
1570 .node
1571 .fetch_cached(&state.database)
1572 .await?
1573 .api_client(&state.database)
1574 .await?
1575 .ignoring(server.subuser_ignored_files.clone().unwrap_or_default());
1576
1577 match client
1578 .post_backups_backup_export(
1579 self.uuid,
1580 &wings_api::backups_backup_export::post::RequestBody {
1581 adapter: self.disk.to_wings_adapter(),
1582 server: server.uuid,
1583 path,
1584 archive_format,
1585 foreground,
1586 },
1587 )
1588 .await
1589 {
1590 Ok(response) => Ok(response),
1591 Err(wings_api::client::ApiHttpError::Http(
1592 status @ (StatusCode::NOT_FOUND | StatusCode::EXPECTATION_FAILED),
1593 err,
1594 )) => Err(crate::response::DisplayError::new(err.error.to_string())
1595 .with_status(status)
1596 .into()),
1597 Err(err) => Err(err.into()),
1598 }
1599 }
1600
1601 pub async fn query(
1602 &self,
1603 state: &crate::State,
1604 node: &super::node::Node,
1605 ) -> Result<wings_api::backups_backup_query::get::Response, anyhow::Error> {
1606 let client = node.api_client(&state.database).await?;
1607
1608 match client
1609 .get_backups_backup_query(
1610 self.uuid,
1611 &wings_api::backups_backup_query::get::Query {
1612 adapter: Some(self.disk.to_wings_adapter()),
1613 __priv: (),
1614 },
1615 )
1616 .await
1617 {
1618 Ok(response) => Ok(response),
1619 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, err)) => {
1620 Err(crate::response::DisplayError::new(err.error.to_string())
1621 .with_status(StatusCode::NOT_FOUND)
1622 .into())
1623 }
1624 Err(err) => Err(err.into()),
1625 }
1626 }
1627
1628 async fn log_eviction_activity(
1629 state: &crate::State,
1630 server_uuid: uuid::Uuid,
1631 backup: &Self,
1632 rule: &str,
1633 scope: EvictionScope<'_>,
1634 ) {
1635 if let Err(err) = super::server_activity::ServerActivity::create(
1636 state,
1637 super::server_activity::CreateServerActivityOptions {
1638 server_uuid,
1639 user_uuid: None,
1640 impersonator_uuid: None,
1641 api_key_uuid: None,
1642 schedule_uuid: None,
1643 event: match backup.kind {
1644 ServerBackupKind::Server => "server:backup.delete".into(),
1645 ServerBackupKind::DatabaseInstance => "server:database-backup.delete".into(),
1646 },
1647 ip: None,
1648 data: serde_json::json!({
1649 "source": "eviction",
1650 "uuid": backup.uuid,
1651 "name": backup.name,
1652 "database_instance_uuid": backup.database_instance_uuid,
1653 "rule": rule,
1654 "group": scope.group_name(),
1655 "policy": scope.policy_name(),
1656 }),
1657 created: None,
1658 },
1659 )
1660 .await
1661 {
1662 tracing::warn!(
1663 server = %server_uuid,
1664 "failed to log backup eviction activity: {:#?}",
1665 err
1666 );
1667 }
1668 }
1669
1670 #[inline]
1671 pub fn default_name() -> compact_str::CompactString {
1672 let now = chrono::Local::now();
1673
1674 now.format("%Y-%m-%d %H:%M:%S %z").to_compact_string()
1675 }
1676
1677 #[inline]
1678 pub fn s3_path(
1679 &self,
1680 server_uuid: uuid::Uuid,
1681 compression_type: wings_api::CompressionType,
1682 ) -> compact_str::CompactString {
1683 let base_name = match self.kind {
1684 ServerBackupKind::Server => compact_str::format_compact!("{}.tar", self.uuid),
1685 ServerBackupKind::DatabaseInstance => self.database_dump_name(),
1686 };
1687
1688 compact_str::format_compact!(
1689 "{server_uuid}/{base_name}{}",
1690 match compression_type {
1691 wings_api::CompressionType::None => "",
1692 wings_api::CompressionType::Gz => ".gz",
1693 wings_api::CompressionType::Xz => ".xz",
1694 wings_api::CompressionType::Lzip => ".lz",
1695 wings_api::CompressionType::Bz2 => ".bz2",
1696 wings_api::CompressionType::Lz4 => ".lz4",
1697 wings_api::CompressionType::Zstd => ".zst",
1698 }
1699 )
1700 }
1701
1702 #[inline]
1703 pub fn s3_content_type(name: &str) -> &'static str {
1704 if name.ends_with(".tar") {
1705 "application/x-tar"
1706 } else if name.ends_with(".gz") {
1707 "application/x-gzip"
1708 } else if name.ends_with(".xz") {
1709 "application/x-xz"
1710 } else if name.ends_with(".lz") {
1711 "application/x-lzip"
1712 } else if name.ends_with(".bz2") {
1713 "application/x-bzip2"
1714 } else if name.ends_with(".lz4") {
1715 "application/x-lz4"
1716 } else if name.ends_with(".zst") {
1717 "application/zstd"
1718 } else {
1719 "application/octet-stream"
1720 }
1721 }
1722
1723 pub async fn into_admin_node_api_object(
1724 self,
1725 state: &crate::State,
1726 storage_url_retriever: &StorageUrlRetriever<'_>,
1727 retention_status: Option<retention::ServerBackupRetentionStatus>,
1728 ) -> Result<AdminApiNodeServerBackup, crate::database::DatabaseError> {
1729 let deletion_status = self.deletion_status();
1730
1731 Ok(AdminApiNodeServerBackup {
1732 uuid: self.uuid,
1733 server: match self.server {
1734 Some(server) => Some(
1735 server
1736 .fetch_cached(&state.database)
1737 .await?
1738 .into_admin_api_object(state, storage_url_retriever)
1739 .await?,
1740 ),
1741 None => None,
1742 },
1743 node: self
1744 .node
1745 .fetch_cached(&state.database)
1746 .await?
1747 .into_admin_api_object(state, ())
1748 .await?,
1749 backup_group_uuid: self.backup_group_uuid,
1750 system_backup_policy_uuid: self.system_backup_policy_uuid,
1751 database_instance_uuid: self.database_instance_uuid,
1752 kind: self.kind,
1753 database_type: self.database_type,
1754 name: self.name,
1755 ignored_files: self.ignored_files,
1756 is_successful: self.successful,
1757 is_locked: self.locked,
1758 is_browsable: self.browsable,
1759 is_streaming: self.streaming,
1760 is_shared: self.shared,
1761 checksum: self.checksum,
1762 bytes: self.bytes,
1763 files: self.files,
1764 deletion_status,
1765 retention_status,
1766 metadata: self.metadata,
1767 completed: self.completed.map(|dt| dt.and_utc()),
1768 created: self.created.and_utc(),
1769 })
1770 }
1771}
1772
1773#[async_trait::async_trait]
1774impl IntoAdminApiObject for ServerBackup {
1775 type AdminApiObject = AdminApiServerBackup;
1776 type ExtraArgs<'a> = (
1777 &'a crate::storage::StorageUrlRetriever<'a>,
1778 Option<retention::ServerBackupRetentionStatus>,
1779 );
1780
1781 async fn into_admin_api_object<'a>(
1782 self,
1783 state: &crate::State,
1784 (storage_url_retriever, retention_status): Self::ExtraArgs<'a>,
1785 ) -> Result<Self::AdminApiObject, crate::database::DatabaseError> {
1786 let deletion_status = self.deletion_status();
1787 let api_object = AdminApiServerBackup::init_hooks(&self, state).await?;
1788
1789 let api_object = finish_extendible!(
1790 AdminApiServerBackup {
1791 uuid: self.uuid,
1792 server: match self.server {
1793 Some(server) => Some(
1794 server
1795 .fetch_cached(&state.database)
1796 .await?
1797 .into_admin_api_object(state, storage_url_retriever)
1798 .await?,
1799 ),
1800 None => None,
1801 },
1802 backup_group_uuid: self.backup_group_uuid,
1803 system_backup_policy_uuid: self.system_backup_policy_uuid,
1804 database_instance_uuid: self.database_instance_uuid,
1805 kind: self.kind,
1806 database_type: self.database_type,
1807 name: self.name,
1808 ignored_files: self.ignored_files,
1809 is_successful: self.successful,
1810 is_locked: self.locked,
1811 is_browsable: self.browsable,
1812 is_streaming: self.streaming,
1813 is_shared: self.shared,
1814 checksum: self.checksum,
1815 bytes: self.bytes,
1816 files: self.files,
1817 deletion_status,
1818 retention_status,
1819 metadata: self.metadata,
1820 completed: self.completed.map(|dt| dt.and_utc()),
1821 created: self.created.and_utc(),
1822 },
1823 api_object,
1824 state
1825 )?;
1826
1827 Ok(api_object)
1828 }
1829}
1830
1831#[async_trait::async_trait]
1832impl IntoApiObject for ServerBackup {
1833 type ApiObject = ApiServerBackup;
1834 type ExtraArgs<'a> = Option<retention::ServerBackupRetentionStatus>;
1835
1836 async fn into_api_object<'a>(
1837 self,
1838 state: &crate::State,
1839 retention_status: Self::ExtraArgs<'a>,
1840 ) -> Result<Self::ApiObject, crate::database::DatabaseError> {
1841 let deletion_status = self.deletion_status();
1842 let api_object = ApiServerBackup::init_hooks(&self, state).await?;
1843
1844 let api_object = finish_extendible!(
1845 ApiServerBackup {
1846 uuid: self.uuid,
1847 backup_group_uuid: self.backup_group_uuid,
1848 database_instance_uuid: self.database_instance_uuid,
1849 kind: self.kind,
1850 database_type: self.database_type,
1851 name: self.name,
1852 ignored_files: self.ignored_files,
1853 is_successful: self.successful,
1854 is_locked: self.locked,
1855 is_browsable: self.browsable,
1856 is_streaming: self.streaming,
1857 checksum: self.checksum,
1858 bytes: self.bytes,
1859 files: self.files,
1860 deletion_status,
1861 retention_status,
1862 metadata: self.metadata,
1863 completed: self.completed.map(|dt| dt.and_utc()),
1864 created: self.created.and_utc(),
1865 },
1866 api_object,
1867 state
1868 )?;
1869
1870 Ok(api_object)
1871 }
1872}
1873
1874#[derive(Validate)]
1875pub struct CreateServerBackupOptions<'a> {
1876 #[garde(skip)]
1877 pub server: &'a super::server::Server,
1878 #[garde(length(chars, min = 1, max = 255))]
1879 pub name: compact_str::CompactString,
1880 #[garde(skip)]
1881 pub backup_group_uuid: Option<uuid::Uuid>,
1882 #[garde(skip)]
1883 pub system_backup_policy_uuid: Option<uuid::Uuid>,
1884 #[garde(skip)]
1885 pub database_instance: Option<&'a super::server_database_instance::ServerDatabaseInstance>,
1886 #[garde(skip)]
1887 pub backup_configuration: Option<super::backup_configuration::BackupConfiguration>,
1888 #[garde(skip)]
1889 pub ignored_files: Vec<compact_str::CompactString>,
1890 #[garde(skip)]
1891 pub metadata: serde_json::Value,
1892}
1893
1894impl CreateServerBackupOptions<'_> {
1895 #[inline]
1896 pub fn kind(&self) -> ServerBackupKind {
1897 if self.database_instance.is_some() {
1898 ServerBackupKind::DatabaseInstance
1899 } else {
1900 ServerBackupKind::Server
1901 }
1902 }
1903}
1904
1905#[async_trait::async_trait]
1906impl CreatableModel for ServerBackup {
1907 type CreateOptions<'a> = CreateServerBackupOptions<'a>;
1908 type CreateResult = Self;
1909
1910 fn get_create_handlers() -> &'static LazyLock<CreateListenerList<Self>> {
1911 static CREATE_LISTENERS: LazyLock<CreateListenerList<ServerBackup>> =
1912 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
1913
1914 &CREATE_LISTENERS
1915 }
1916
1917 async fn create_with_transaction(
1918 _state: &crate::State,
1919 _options: Self::CreateOptions<'_>,
1920 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1921 ) -> Result<Self, crate::database::DatabaseError> {
1922 Err(anyhow::anyhow!("create_with_transaction is not supported for ServerBackup").into())
1923 }
1924
1925 async fn create(
1926 state: &crate::State,
1927 mut options: Self::CreateOptions<'_>,
1928 ) -> Result<Self, crate::database::DatabaseError> {
1929 options.validate()?;
1930
1931 let backup_configuration = match options.backup_configuration.take() {
1932 Some(backup_configuration) => backup_configuration,
1933 None => options
1934 .server
1935 .backup_configuration(&state.database)
1936 .await
1937 .ok_or_else(|| {
1938 anyhow::Error::new(
1939 crate::response::DisplayError::new(
1940 "no backup configuration available, unable to create backup",
1941 )
1942 .with_status(StatusCode::EXPECTATION_FAILED),
1943 )
1944 })?,
1945 };
1946
1947 if backup_configuration.maintenance_enabled {
1948 return Err(anyhow::Error::new(
1949 crate::response::DisplayError::new(
1950 "cannot create backup while backup configuration is in maintenance mode",
1951 )
1952 .with_status(StatusCode::EXPECTATION_FAILED),
1953 )
1954 .into());
1955 }
1956
1957 if options.database_instance.is_some()
1958 && matches!(
1959 backup_configuration.backup_disk,
1960 BackupDisk::Btrfs | BackupDisk::Zfs
1961 )
1962 {
1963 return Err(anyhow::Error::new(
1964 crate::response::DisplayError::new(
1965 "database backups cannot be created on a btrfs or zfs backup configuration",
1966 )
1967 .with_status(StatusCode::EXPECTATION_FAILED),
1968 )
1969 .into());
1970 }
1971
1972 let mut transaction = state.database.write().begin().await?;
1973
1974 let mut query_builder = InsertQueryBuilder::new("server_backups");
1975
1976 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
1977 .await?;
1978
1979 query_builder
1980 .set("server_uuid", options.server.uuid)
1981 .set("node_uuid", options.server.node.uuid)
1982 .set("backup_configuration_uuid", backup_configuration.uuid)
1983 .set("backup_group_uuid", options.backup_group_uuid)
1984 .set(
1985 "system_backup_policy_uuid",
1986 options.system_backup_policy_uuid,
1987 )
1988 .set(
1989 "database_instance_uuid",
1990 options.database_instance.map(|instance| instance.uuid),
1991 )
1992 .set("kind", options.kind())
1993 .set(
1994 "database_type",
1995 options.database_instance.map(|instance| instance.r#type),
1996 )
1997 .set("name", &options.name)
1998 .set("ignored_files", &options.ignored_files)
1999 .set("bytes", 0i64)
2000 .set("disk", backup_configuration.backup_disk)
2001 .set("shared", backup_configuration.shared)
2002 .set("metadata", &options.metadata);
2003
2004 let row = query_builder
2005 .returning(&Self::columns_sql(None))
2006 .fetch_one(&mut *transaction)
2007 .await?;
2008 let mut backup = Self::map(None, &row)?;
2009
2010 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
2011
2012 transaction.commit().await?;
2013
2014 let server = options.server.clone();
2015 let database = Arc::clone(&state.database);
2016 let backup_uuid = backup.uuid;
2017 let backup_disk = backup.disk;
2018 let database_dump = options
2019 .database_instance
2020 .map(|instance| (instance.uuid, instance.r#type.export_extension()));
2021 let ignored_files_str = options
2022 .ignored_files
2023 .iter()
2024 .map(|s| s.as_str())
2025 .collect::<Vec<_>>()
2026 .join("\n");
2027
2028 tokio::spawn(async move {
2029 tracing::debug!(backup = %backup_uuid, "creating server backup");
2030
2031 let node = match server.node.fetch_cached(&database).await {
2032 Ok(node) => node,
2033 Err(err) => {
2034 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2035
2036 if let Err(err) = sqlx::query!(
2037 "UPDATE server_backups
2038 SET successful = false, completed = NOW()
2039 WHERE server_backups.uuid = $1",
2040 backup_uuid
2041 )
2042 .execute(database.write())
2043 .await
2044 {
2045 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2046 }
2047
2048 return;
2049 }
2050 };
2051
2052 let api_client = match node.api_client(&database).await {
2053 Ok(api_client) => api_client,
2054 Err(err) => {
2055 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2056
2057 if let Err(err) = sqlx::query!(
2058 "UPDATE server_backups
2059 SET successful = false, completed = NOW()
2060 WHERE server_backups.uuid = $1",
2061 backup_uuid
2062 )
2063 .execute(database.write())
2064 .await
2065 {
2066 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2067 }
2068
2069 return;
2070 }
2071 };
2072
2073 let result = match database_dump {
2074 Some((database_instance, extension)) => api_client
2075 .post_servers_server_database_backup(
2076 server.uuid,
2077 &wings_api::servers_server_database_backup::post::RequestBody {
2078 adapter: backup_disk.to_wings_adapter(),
2079 uuid: backup_uuid,
2080 database_instance,
2081 extension: extension.into(),
2082 },
2083 )
2084 .await
2085 .map(|_| ()),
2086 None => api_client
2087 .post_servers_server_backup(
2088 server.uuid,
2089 &wings_api::servers_server_backup::post::RequestBody {
2090 adapter: backup_disk.to_wings_adapter(),
2091 uuid: backup_uuid,
2092 ignore: ignored_files_str.into(),
2093 },
2094 )
2095 .await
2096 .map(|_| ()),
2097 };
2098
2099 if let Err(err) = result {
2100 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2101
2102 if let Err(err) = sqlx::query!(
2103 "UPDATE server_backups
2104 SET successful = false, completed = NOW()
2105 WHERE server_backups.uuid = $1",
2106 backup_uuid
2107 )
2108 .execute(database.write())
2109 .await
2110 {
2111 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2112 }
2113 }
2114 });
2115
2116 Ok(backup)
2117 }
2118}
2119
2120#[derive(ToSchema, Validate, Deserialize, Serialize)]
2121#[serde(rename_all = "snake_case", tag = "type")]
2122#[non_exhaustive]
2123pub enum ServerBackupSelector {
2124 Uuids {
2125 #[garde(length(min = 1, max = 100))]
2126 #[schema(min_items = 1, max_items = 100)]
2127 uuids: Vec<uuid::Uuid>,
2128 },
2129}
2130
2131impl ServerBackupSelector {
2132 #[inline]
2133 pub fn uuids(&self) -> &[uuid::Uuid] {
2134 match self {
2135 Self::Uuids { uuids } => uuids,
2136 }
2137 }
2138}
2139
2140#[derive(ToSchema, Serialize, Deserialize, Validate, Default, Clone)]
2141pub struct UpdateServerBackupOptions {
2142 #[garde(length(chars, min = 1, max = 255))]
2143 #[schema(min_length = 1, max_length = 255)]
2144 pub name: Option<compact_str::CompactString>,
2145 #[garde(skip)]
2146 #[serde(default, with = "::serde_with::rust::double_option")]
2147 pub backup_group_uuid: Option<Option<uuid::Uuid>>,
2148 #[garde(skip)]
2149 pub locked: Option<bool>,
2150}
2151
2152#[async_trait::async_trait]
2153impl UpdatableModel for ServerBackup {
2154 type UpdateOptions = UpdateServerBackupOptions;
2155
2156 fn get_update_handlers() -> &'static LazyLock<UpdateHandlerList<Self>> {
2157 static UPDATE_LISTENERS: LazyLock<UpdateHandlerList<ServerBackup>> =
2158 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2159
2160 &UPDATE_LISTENERS
2161 }
2162
2163 async fn update_with_transaction(
2164 &mut self,
2165 state: &crate::State,
2166 mut options: Self::UpdateOptions,
2167 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2168 ) -> Result<(), crate::database::DatabaseError> {
2169 options.validate()?;
2170
2171 if let Some(Some(backup_group_uuid)) = options.backup_group_uuid {
2172 sqlx::query(
2173 r#"
2174 SELECT server_backup_groups.uuid
2175 FROM server_backup_groups
2176 WHERE server_backup_groups.uuid = $1
2177 FOR KEY SHARE
2178 "#,
2179 )
2180 .bind(backup_group_uuid)
2181 .fetch_optional(&mut **transaction)
2182 .await?;
2183
2184 let group = super::server_backup_group::ServerBackupGroup::by_uuid_with_transaction(
2185 transaction,
2186 backup_group_uuid,
2187 )
2188 .await?;
2189
2190 if Some(group.server_uuid) != self.server.as_ref().map(|server| server.uuid) {
2191 return Err(anyhow::Error::new(
2192 crate::response::DisplayError::new(
2193 "backup group does not belong to this backup's server",
2194 )
2195 .with_status(StatusCode::EXPECTATION_FAILED),
2196 )
2197 .into());
2198 }
2199 }
2200
2201 if sqlx::query(
2202 r#"
2203 SELECT server_backups.uuid
2204 FROM server_backups
2205 WHERE server_backups.uuid = $1
2206 AND server_backups.deleted IS NULL
2207 AND server_backups.deleting IS NULL
2208 FOR UPDATE
2209 "#,
2210 )
2211 .bind(self.uuid)
2212 .fetch_optional(&mut **transaction)
2213 .await?
2214 .is_none()
2215 {
2216 return Err(anyhow::Error::new(
2217 crate::response::DisplayError::new("backup is being deleted")
2218 .with_status(StatusCode::EXPECTATION_FAILED),
2219 )
2220 .into());
2221 }
2222
2223 let mut query_builder = UpdateQueryBuilder::new("server_backups");
2224
2225 self.run_update_handlers(&mut options, &mut query_builder, state, transaction)
2226 .await?;
2227
2228 query_builder
2229 .set("name", options.name.as_ref())
2230 .set("backup_group_uuid", options.backup_group_uuid)
2231 .set("locked", options.locked)
2232 .where_eq("uuid", self.uuid);
2233
2234 query_builder.execute(&mut **transaction).await?;
2235
2236 if let Some(name) = options.name {
2237 self.name = name;
2238 }
2239 if let Some(backup_group_uuid) = options.backup_group_uuid {
2240 self.backup_group_uuid = backup_group_uuid;
2241 }
2242 if let Some(locked) = options.locked {
2243 self.locked = locked;
2244 }
2245
2246 self.run_after_update_handlers(state, transaction).await?;
2247
2248 Ok(())
2249 }
2250}
2251
2252#[async_trait::async_trait]
2253impl ByUuid for ServerBackup {
2254 async fn by_uuid(
2255 database: &crate::database::Database,
2256 uuid: uuid::Uuid,
2257 ) -> Result<Self, crate::database::DatabaseError> {
2258 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2259 r#"
2260 SELECT {}
2261 FROM server_backups
2262 WHERE server_backups.uuid = $1
2263 "#,
2264 Self::columns_sql(None)
2265 )))
2266 .bind(uuid)
2267 .fetch_one(database.read())
2268 .await?;
2269
2270 Self::map(None, &row)
2271 }
2272
2273 async fn by_uuid_with_transaction(
2274 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2275 uuid: uuid::Uuid,
2276 ) -> Result<Self, crate::database::DatabaseError> {
2277 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2278 r#"
2279 SELECT {}
2280 FROM server_backups
2281 WHERE server_backups.uuid = $1
2282 "#,
2283 Self::columns_sql(None)
2284 )))
2285 .bind(uuid)
2286 .fetch_one(&mut **transaction)
2287 .await?;
2288
2289 Self::map(None, &row)
2290 }
2291}
2292
2293#[derive(Clone, Default)]
2294pub struct DeleteServerBackupOptions {
2295 pub force: bool,
2296 pub retention: Option<retention::RetentionDeletionGuard>,
2297}
2298
2299#[derive(Clone, Copy)]
2302pub enum FailedServerBackupScope {
2303 Node(uuid::Uuid),
2304 DetachedNode(uuid::Uuid),
2305 Server(uuid::Uuid),
2306 PartiallyDetachedServer {
2307 server_uuid: uuid::Uuid,
2308 node_uuid: uuid::Uuid,
2309 },
2310 BackupConfiguration(uuid::Uuid),
2311 SystemBackupPolicy(uuid::Uuid),
2312 DatabaseAgentHost(uuid::Uuid),
2313}
2314
2315impl FailedServerBackupScope {
2316 #[inline]
2317 fn condition(self) -> &'static str {
2318 match self {
2319 Self::Node(_) => "server_backups.node_uuid = $1",
2320 Self::DetachedNode(_) => {
2321 "server_backups.node_uuid = $1 AND server_backups.server_uuid IS NULL"
2322 }
2323 Self::Server(_) | Self::PartiallyDetachedServer { .. } => {
2324 "server_backups.server_uuid = $1"
2325 }
2326 Self::BackupConfiguration(_) => "server_backups.backup_configuration_uuid = $1",
2327 Self::SystemBackupPolicy(_) => "server_backups.system_backup_policy_uuid = $1",
2328 Self::DatabaseAgentHost(_) => {
2329 "server_backups.database_instance_uuid IN (
2330 SELECT server_database_instances.uuid
2331 FROM server_database_instances
2332 WHERE server_database_instances.database_agent_host_uuid = $1
2333 )"
2334 }
2335 }
2336 }
2337
2338 #[inline]
2340 fn bindings(self) -> (uuid::Uuid, Option<uuid::Uuid>) {
2341 match self {
2342 Self::Node(uuid)
2343 | Self::DetachedNode(uuid)
2344 | Self::Server(uuid)
2345 | Self::BackupConfiguration(uuid)
2346 | Self::SystemBackupPolicy(uuid)
2347 | Self::DatabaseAgentHost(uuid) => (uuid, None),
2348 Self::PartiallyDetachedServer {
2349 server_uuid,
2350 node_uuid,
2351 } => (server_uuid, Some(node_uuid)),
2352 }
2353 }
2354}
2355
2356impl ServerBackup {
2357 pub const MAX_DELETION_RETRIES: i32 = 8;
2358
2359 #[inline]
2360 pub fn deletion_status(&self) -> Option<ServerBackupDeletionStatus> {
2361 if self.deleted.is_some() || self.deleting.is_none() {
2362 return None;
2363 }
2364
2365 if self.deletion_retries >= Self::MAX_DELETION_RETRIES {
2366 Some(ServerBackupDeletionStatus::Failed)
2367 } else {
2368 Some(ServerBackupDeletionStatus::Deleting)
2369 }
2370 }
2371
2372 pub async fn dispatch_deletion(
2373 &self,
2374 state: &crate::State,
2375 options: &DeleteServerBackupOptions,
2376 ) -> Result<bool, anyhow::Error> {
2377 let node = self.node.fetch_cached(&state.database).await?;
2378
2379 let backup_configuration = match &self.backup_configuration {
2380 Some(backup_configuration) => {
2381 Some(backup_configuration.fetch_cached(&state.database).await?)
2382 }
2383 None if options.force => None,
2384 None => {
2385 return Err(crate::response::DisplayError::new(
2386 "no backup configuration available, unable to delete backup",
2387 )
2388 .with_status(StatusCode::EXPECTATION_FAILED)
2389 .into());
2390 }
2391 };
2392
2393 if let Some(backup_configuration) = &backup_configuration
2394 && backup_configuration.maintenance_enabled
2395 {
2396 return Err(crate::response::DisplayError::new(
2397 "cannot delete backup while backup configuration is in maintenance mode",
2398 )
2399 .with_status(StatusCode::EXPECTATION_FAILED)
2400 .into());
2401 }
2402
2403 if self.disk == BackupDisk::S3 {
2404 let Some(mut s3_configuration) =
2405 backup_configuration.and_then(|c| c.backup_configs.s3.clone())
2406 else {
2407 if options.force {
2408 tracing::warn!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "S3 backup deletion attempted but no S3 configuration found, ignoring");
2409
2410 return Ok(true);
2411 }
2412
2413 return Err(anyhow::anyhow!(
2414 "s3 backup deletion attempted but no S3 configuration found"
2415 ));
2416 };
2417
2418 s3_configuration.decrypt(&state.database).await?;
2419
2420 let compression_type = s3_configuration.compression_type;
2421 let (client, bucket) = s3_configuration.into_client();
2422
2423 let file_path = match &self.upload_path {
2424 Some(path) => path,
2425 None => {
2426 if let Some(server) = &self.server {
2427 &self.s3_path(server.uuid, compression_type)
2428 } else {
2429 return Err(anyhow::anyhow!("backup upload path not found"));
2430 }
2431 }
2432 };
2433
2434 if let Err(err) = client
2435 .delete_object()
2436 .bucket(bucket)
2437 .key(&**file_path)
2438 .send()
2439 .await
2440 {
2441 if options.force {
2442 tracing::error!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "failed to delete S3 backup, ignoring: {:?}", err);
2443 } else {
2444 return Err(err.into());
2445 }
2446 }
2447
2448 return Ok(true);
2449 }
2450
2451 match node
2452 .api_client(&state.database)
2453 .await?
2454 .delete_backups_backup(
2455 self.uuid,
2456 &wings_api::backups_backup::delete::RequestBody {
2457 adapter: self.disk.to_wings_adapter(),
2458 foreground: false,
2459 server: self.server.as_ref().map(|s| s.uuid),
2460 },
2461 )
2462 .await
2463 {
2464 Ok(_) => Ok(false),
2465 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, _)) => Ok(true),
2466 Err(err) if options.force => {
2467 tracing::error!(node = %node.uuid, backup = %self.uuid, "unable to delete backup on node, finalizing anyway: {:?}", err);
2468
2469 Ok(true)
2470 }
2471 Err(err) => Err(err.into()),
2472 }
2473 }
2474
2475 pub async fn finish_deletion(
2476 &self,
2477 state: &crate::State,
2478 options: &DeleteServerBackupOptions,
2479 ) -> Result<(), anyhow::Error> {
2480 let mut transaction = state.database.write().begin().await?;
2481
2482 let finalized = sqlx::query(
2483 r#"
2484 UPDATE server_backups
2485 SET deleted = NOW(), deleting = NULL
2486 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2487 "#,
2488 )
2489 .bind(self.uuid)
2490 .execute(&mut *transaction)
2491 .await?
2492 .rows_affected();
2493
2494 if finalized == 0 {
2495 return Ok(());
2496 }
2497
2498 self.run_after_delete_handlers(options, state, &mut transaction)
2499 .await?;
2500
2501 transaction.commit().await?;
2502
2503 Self::get_event_emitter().emit(
2504 state.clone(),
2505 ServerBackupEvent::DeletionCompleted {
2506 backup: Box::new(self.clone()),
2507 successful: true,
2508 },
2509 );
2510
2511 Ok(())
2512 }
2513
2514 pub async fn fail_deletion_attempt(&self, state: &crate::State) -> Result<(), anyhow::Error> {
2515 let finalized = sqlx::query(
2516 r#"
2517 UPDATE server_backups
2518 SET deletion_retries = $2
2519 WHERE
2520 server_backups.uuid = $1
2521 AND server_backups.deleted IS NULL
2522 AND server_backups.deleting IS NOT NULL
2523 AND server_backups.deletion_retries = $2 - 1
2524 "#,
2525 )
2526 .bind(self.uuid)
2527 .bind(Self::MAX_DELETION_RETRIES)
2528 .execute(state.database.write())
2529 .await?
2530 .rows_affected();
2531
2532 if finalized > 0 {
2533 self.finalize_failed_deletion(state).await;
2534 }
2535
2536 Ok(())
2537 }
2538
2539 async fn finalize_failed_deletion(&self, state: &crate::State) {
2540 if let Some(server) = &self.server
2541 && let Err(err) = super::server_activity::ServerActivity::create(
2542 state,
2543 super::server_activity::CreateServerActivityOptions {
2544 server_uuid: server.uuid,
2545 user_uuid: None,
2546 impersonator_uuid: None,
2547 api_key_uuid: None,
2548 schedule_uuid: None,
2549 event: match self.kind {
2550 ServerBackupKind::Server => "server:backup.delete-failed",
2551 ServerBackupKind::DatabaseInstance => {
2552 "server:database-backup.delete-failed"
2553 }
2554 }
2555 .into(),
2556 ip: None,
2557 data: serde_json::json!({
2558 "uuid": self.uuid,
2559 "name": self.name,
2560 "database_instance_uuid": self.database_instance_uuid,
2561 }),
2562 created: None,
2563 },
2564 )
2565 .await
2566 {
2567 tracing::warn!(
2568 backup = %self.uuid,
2569 "failed to log backup deletion failure activity: {:#?}",
2570 err
2571 );
2572 }
2573
2574 Self::get_event_emitter().emit(
2575 state.clone(),
2576 ServerBackupEvent::DeletionCompleted {
2577 backup: Box::new(self.clone()),
2578 successful: false,
2579 },
2580 );
2581 }
2582
2583 pub async fn redispatch_stale_deletions(state: &crate::State) -> Result<u64, anyhow::Error> {
2584 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2585 r#"
2586 SELECT {}
2587 FROM server_backups
2588 WHERE
2589 server_backups.deleted IS NULL
2590 AND server_backups.deleting IS NOT NULL
2591 AND server_backups.deletion_retries < $1
2592 AND server_backups.deleting < NOW() - make_interval(mins => LEAST(60.0, 5.0 * POWER(2.0, server_backups.deletion_retries))::int)
2593 ORDER BY server_backups.deleting
2594 LIMIT 32
2595 "#,
2596 Self::columns_sql(None)
2597 )))
2598 .bind(Self::MAX_DELETION_RETRIES)
2599 .fetch_all(state.database.read())
2600 .await?;
2601
2602 let mut redispatched = 0;
2603 for row in rows {
2604 let backup = Self::map(None, &row)?;
2605
2606 if let Some(backup_configuration) = &backup.backup_configuration
2607 && let Ok(backup_configuration) =
2608 backup_configuration.fetch_cached(&state.database).await
2609 && backup_configuration.maintenance_enabled
2610 {
2611 sqlx::query(
2612 "UPDATE server_backups
2613 SET deleting = NOW()
2614 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL",
2615 )
2616 .bind(backup.uuid)
2617 .execute(state.database.write())
2618 .await?;
2619
2620 continue;
2621 }
2622
2623 let deletion_retries: Option<i32> = sqlx::query_scalar(
2624 r#"
2625 UPDATE server_backups
2626 SET deleting = NOW(), deletion_retries = deletion_retries + 1
2627 WHERE
2628 server_backups.uuid = $1
2629 AND server_backups.deleted IS NULL
2630 AND server_backups.deleting = $2
2631 RETURNING server_backups.deletion_retries
2632 "#,
2633 )
2634 .bind(backup.uuid)
2635 .bind(backup.deleting)
2636 .fetch_optional(state.database.write())
2637 .await?;
2638
2639 let Some(deletion_retries) = deletion_retries else {
2640 continue;
2641 };
2642
2643 if deletion_retries >= Self::MAX_DELETION_RETRIES {
2644 backup.finalize_failed_deletion(state).await;
2645 continue;
2646 }
2647
2648 match backup
2649 .dispatch_deletion(state, &DeleteServerBackupOptions::default())
2650 .await
2651 {
2652 Ok(true) => {
2653 backup
2654 .finish_deletion(state, &DeleteServerBackupOptions::default())
2655 .await?;
2656 }
2657 Ok(false) => {}
2658 Err(err) => {
2659 tracing::error!(
2660 backup = %backup.uuid,
2661 deletion_retries,
2662 "failed to redispatch backup deletion: {:#?}",
2663 err
2664 );
2665
2666 backup.fail_deletion_attempt(state).await?;
2667
2668 continue;
2669 }
2670 }
2671
2672 redispatched += 1;
2673 }
2674
2675 Ok(redispatched)
2676 }
2677}
2678
2679#[async_trait::async_trait]
2680impl DeletableModel for ServerBackup {
2681 type DeleteOptions = DeleteServerBackupOptions;
2682
2683 fn get_delete_handlers() -> &'static LazyLock<DeleteHandlerList<Self>> {
2684 static DELETE_LISTENERS: LazyLock<DeleteHandlerList<ServerBackup>> =
2685 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2686
2687 &DELETE_LISTENERS
2688 }
2689
2690 async fn delete_with_transaction(
2691 &self,
2692 _state: &crate::State,
2693 _options: Self::DeleteOptions,
2694 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2695 ) -> Result<(), anyhow::Error> {
2696 Err(anyhow::anyhow!(
2697 "delete_with_transaction is not supported for ServerBackup"
2698 ))
2699 }
2700
2701 async fn delete(
2702 &self,
2703 state: &crate::State,
2704 options: Self::DeleteOptions,
2705 ) -> Result<(), anyhow::Error> {
2706 let mut transaction = state.database.write().begin().await?;
2707
2708 self.claim_deletion(state, &options, &mut transaction)
2709 .await?;
2710
2711 transaction.commit().await?;
2712
2713 self.dispatch_claimed_deletion(state, &options).await
2714 }
2715}
2716
2717impl ServerBackup {
2718 async fn backup_configuration_in_maintenance(
2719 &self,
2720 state: &crate::State,
2721 ) -> Result<bool, anyhow::Error> {
2722 let Some(configuration) = &self.backup_configuration else {
2723 return Ok(false);
2724 };
2725
2726 Ok(configuration
2727 .fetch_cached(&state.database)
2728 .await?
2729 .maintenance_enabled)
2730 }
2731
2732 async fn claim_deletion(
2733 &self,
2734 state: &crate::State,
2735 options: &DeleteServerBackupOptions,
2736 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2737 ) -> Result<(), anyhow::Error> {
2738 if self.backup_configuration_in_maintenance(state).await? {
2739 return Err(crate::response::DisplayError::new(
2740 "cannot delete backup while backup configuration is in maintenance mode",
2741 )
2742 .with_status(StatusCode::EXPECTATION_FAILED)
2743 .into());
2744 }
2745
2746 self.run_delete_handlers(options, state, transaction)
2747 .await?;
2748
2749 let retention_guard = options.retention.as_ref();
2750 let retention = retention_guard
2751 .map(|guard| serde_json::to_value(&guard.retention))
2752 .transpose()?;
2753
2754 let claimed = sqlx::query(
2755 r#"
2756 UPDATE server_backups
2757 SET deleting = NOW(), deletion_retries = 0
2758 WHERE
2759 server_backups.uuid = $1
2760 AND server_backups.deleted IS NULL
2761 AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $2)
2762 AND (NOT $3 OR (
2763 NOT server_backups.locked
2764 AND server_backups.completed = $4
2765 AND server_backups.successful = $5
2766 AND server_backups.backup_group_uuid IS NOT DISTINCT FROM $6
2767 AND server_backups.system_backup_policy_uuid IS NOT DISTINCT FROM $7
2768 AND server_backups.server_uuid IS NOT DISTINCT FROM $8
2769 AND server_backups.database_instance_uuid IS NOT DISTINCT FROM $9
2770 AND (
2771 EXISTS (
2772 SELECT 1 FROM server_backup_groups g
2773 WHERE g.uuid = $6
2774 AND g.retention IS NOT DISTINCT FROM $10
2775 ) OR EXISTS (
2776 SELECT 1 FROM system_backup_policies p
2777 WHERE p.uuid = $7
2778 AND p.retention IS NOT DISTINCT FROM $10
2779 )
2780 )
2781 ))
2782 "#,
2783 )
2784 .bind(self.uuid)
2785 .bind(Self::MAX_DELETION_RETRIES)
2786 .bind(retention_guard.is_some())
2787 .bind(retention_guard.map(|guard| guard.completed))
2788 .bind(retention_guard.map(|guard| guard.successful))
2789 .bind(retention_guard.and_then(|guard| guard.backup_group_uuid))
2790 .bind(retention_guard.and_then(|guard| guard.system_backup_policy_uuid))
2791 .bind(self.server.as_ref().map(|server| server.uuid))
2792 .bind(self.database_instance_uuid)
2793 .bind(retention)
2794 .execute(&mut **transaction)
2795 .await?
2796 .rows_affected();
2797
2798 if claimed == 0 {
2799 return Err(
2800 crate::response::DisplayError::new(if retention_guard.is_some() {
2801 "backup is no longer eligible for retention cleanup"
2802 } else {
2803 "backup is already being deleted"
2804 })
2805 .with_status(StatusCode::EXPECTATION_FAILED)
2806 .into(),
2807 );
2808 }
2809
2810 Ok(())
2811 }
2812
2813 async fn dispatch_claimed_deletion(
2814 &self,
2815 state: &crate::State,
2816 options: &DeleteServerBackupOptions,
2817 ) -> Result<(), anyhow::Error> {
2818 match self.dispatch_deletion(state, options).await {
2819 Ok(true) => self.finish_deletion(state, options).await,
2820 Ok(false) => Ok(()),
2821 Err(err) => {
2822 sqlx::query(
2823 r#"
2824 UPDATE server_backups
2825 SET deleting = NULL, deletion_retries = 0
2826 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2827 "#,
2828 )
2829 .bind(self.uuid)
2830 .execute(state.database.write())
2831 .await?;
2832
2833 Err(err)
2834 }
2835 }
2836 }
2837
2838 const FAILED_SWEEP_CONDITION: &'static str = r#"
2842 server_backups.deleted IS NULL
2843 AND server_backups.completed IS NOT NULL
2844 AND NOT server_backups.successful
2845 AND NOT server_backups.locked
2846 AND ($2::uuid IS NULL OR server_backups.node_uuid != $2)
2847 AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $3)
2848 AND NOT EXISTS (
2849 SELECT 1
2850 FROM backup_configurations
2851 WHERE backup_configurations.uuid = server_backups.backup_configuration_uuid
2852 AND backup_configurations.maintenance_enabled
2853 )
2854 "#;
2855
2856 const FAILED_SWEEP_CHUNK: i64 = 250;
2857 const FAILED_SWEEP_CONCURRENCY: usize = 5;
2858 const BULK_CONCURRENCY: usize = 5;
2859
2860 pub async fn count_failed(
2861 database: &crate::database::Database,
2862 scope: FailedServerBackupScope,
2863 ) -> Result<i64, sqlx::Error> {
2864 let (scope_uuid, excluded_node_uuid) = scope.bindings();
2865
2866 sqlx::query_scalar(sqlx::AssertSqlSafe(format!(
2867 r#"
2868 SELECT COUNT(*)
2869 FROM server_backups
2870 WHERE {} AND {}
2871 "#,
2872 scope.condition(),
2873 Self::FAILED_SWEEP_CONDITION
2874 )))
2875 .bind(scope_uuid)
2876 .bind(excluded_node_uuid)
2877 .bind(Self::MAX_DELETION_RETRIES)
2878 .fetch_one(database.read())
2879 .await
2880 }
2881
2882 async fn claim_failed_deletions(
2886 state: &crate::State,
2887 scope: FailedServerBackupScope,
2888 options: &DeleteServerBackupOptions,
2889 after: Option<uuid::Uuid>,
2890 ) -> Result<(Vec<Self>, Option<uuid::Uuid>), anyhow::Error> {
2891 let (scope_uuid, excluded_node_uuid) = scope.bindings();
2892
2893 let mut transaction = state.database.write().begin().await?;
2894
2895 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2896 r#"
2897 SELECT {}
2898 FROM server_backups
2899 WHERE {}
2900 AND {}
2901 AND ($4::uuid IS NULL OR server_backups.uuid > $4)
2902 ORDER BY server_backups.uuid
2903 LIMIT $5
2904 FOR UPDATE
2905 "#,
2906 Self::columns_sql(None),
2907 scope.condition(),
2908 Self::FAILED_SWEEP_CONDITION
2909 )))
2910 .bind(scope_uuid)
2911 .bind(excluded_node_uuid)
2912 .bind(Self::MAX_DELETION_RETRIES)
2913 .bind(after)
2914 .bind(Self::FAILED_SWEEP_CHUNK)
2915 .fetch_all(&mut *transaction)
2916 .await?;
2917
2918 let backups = rows
2919 .iter()
2920 .map(|row| Self::map(None, row))
2921 .try_collect_vec()?;
2922 let cursor = backups.last().map(|backup| backup.uuid);
2923
2924 let mut claimed = Vec::new();
2925 for backup in backups {
2926 if backup.backup_configuration_in_maintenance(state).await? {
2927 continue;
2928 }
2929
2930 backup
2931 .claim_deletion(state, options, &mut transaction)
2932 .await?;
2933 claimed.push(backup);
2934 }
2935
2936 transaction.commit().await?;
2937
2938 Ok((claimed, cursor))
2939 }
2940
2941 async fn dispatch_swept_deletion(
2942 &self,
2943 state: &crate::State,
2944 options: &DeleteServerBackupOptions,
2945 ) -> bool {
2946 if let Err(err) = self.dispatch_claimed_deletion(state, options).await {
2947 tracing::error!(backup = %self.uuid, "failed to delete failed backup: {err:#?}");
2948
2949 return false;
2950 }
2951
2952 true
2953 }
2954
2955 pub async fn update_by_selector(
2959 state: &crate::State,
2960 server_uuid: uuid::Uuid,
2961 selector: &ServerBackupSelector,
2962 options: UpdateServerBackupOptions,
2963 ) -> Result<(Vec<Self>, usize), anyhow::Error> {
2964 let mut uuids = selector.uuids().to_vec();
2965 uuids.sort_unstable();
2966 uuids.dedup();
2967 let requested = uuids.len();
2968
2969 let mut transaction = state.database.write().begin().await?;
2970
2971 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2972 r#"
2973 SELECT {}
2974 FROM server_backups
2975 WHERE server_backups.server_uuid = $1
2976 AND server_backups.uuid = ANY($2)
2977 AND server_backups.deleted IS NULL
2978 AND server_backups.deleting IS NULL
2979 AND server_backups.system_backup_policy_uuid IS NULL
2980 ORDER BY server_backups.uuid
2981 FOR UPDATE
2982 "#,
2983 Self::columns_sql(None)
2984 )))
2985 .bind(server_uuid)
2986 .bind(&uuids)
2987 .fetch_all(&mut *transaction)
2988 .await?;
2989
2990 let mut backups = rows
2991 .iter()
2992 .map(|row| Self::map(None, row))
2993 .try_collect_vec()?;
2994
2995 for backup in backups.iter_mut() {
2996 backup
2997 .update_with_transaction(state, options.clone(), &mut transaction)
2998 .await?;
2999 }
3000
3001 transaction.commit().await?;
3002
3003 let mut backup_group_uuids: Vec<uuid::Uuid> = backups
3004 .iter()
3005 .filter_map(|backup| backup.backup_group_uuid)
3006 .collect();
3007 backup_group_uuids.sort_unstable();
3008 backup_group_uuids.dedup();
3009
3010 if !backup_group_uuids.is_empty() {
3011 Self::prune_retention_scopes(state, Some(server_uuid), &backup_group_uuids).await;
3012 }
3013
3014 let skipped = requested - backups.len();
3015
3016 Ok((backups, skipped))
3017 }
3018
3019 pub async fn claim_deletions_by_selector(
3023 state: &crate::State,
3024 server_uuid: uuid::Uuid,
3025 selector: &ServerBackupSelector,
3026 options: &DeleteServerBackupOptions,
3027 ) -> Result<(Vec<Self>, usize), anyhow::Error> {
3028 let mut uuids = selector.uuids().to_vec();
3029 uuids.sort_unstable();
3030 uuids.dedup();
3031 let requested = uuids.len();
3032
3033 let mut transaction = state.database.write().begin().await?;
3034
3035 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
3036 r#"
3037 SELECT {}
3038 FROM server_backups
3039 WHERE server_backups.server_uuid = $1
3040 AND server_backups.uuid = ANY($2)
3041 AND server_backups.deleted IS NULL
3042 AND server_backups.deleting IS NULL
3043 AND server_backups.system_backup_policy_uuid IS NULL
3044 AND server_backups.completed IS NOT NULL
3045 AND NOT server_backups.locked
3046 ORDER BY server_backups.uuid
3047 FOR UPDATE
3048 "#,
3049 Self::columns_sql(None)
3050 )))
3051 .bind(server_uuid)
3052 .bind(&uuids)
3053 .fetch_all(&mut *transaction)
3054 .await?;
3055
3056 let backups = rows
3057 .iter()
3058 .map(|row| Self::map(None, row))
3059 .try_collect_vec()?;
3060
3061 let mut claimed = Vec::with_capacity(backups.len());
3062 for backup in backups {
3063 if backup.backup_configuration_in_maintenance(state).await? {
3064 continue;
3065 }
3066
3067 backup
3068 .claim_deletion(state, options, &mut transaction)
3069 .await?;
3070 claimed.push(backup);
3071 }
3072
3073 transaction.commit().await?;
3074
3075 let skipped = requested - claimed.len();
3076
3077 Ok((claimed, skipped))
3078 }
3079
3080 pub async fn dispatch_claimed_deletions(
3084 state: &crate::State,
3085 backups: Vec<Self>,
3086 options: &DeleteServerBackupOptions,
3087 ) -> Vec<Self> {
3088 let mut futures = Vec::with_capacity(backups.len());
3089 for backup in backups {
3090 futures.push(async move {
3091 match backup.dispatch_claimed_deletion(state, options).await {
3092 Ok(()) => Some(backup),
3093 Err(err) => {
3094 tracing::error!(backup = %backup.uuid, "failed to delete backup: {err:#?}");
3095 None
3096 }
3097 }
3098 });
3099 }
3100
3101 let mut results_stream =
3102 futures_util::stream::iter(futures).buffer_unordered(Self::BULK_CONCURRENCY);
3103
3104 let mut deleted = Vec::new();
3105 while let Some(backup) = results_stream.next().await {
3106 if let Some(backup) = backup {
3107 deleted.push(backup);
3108 }
3109 }
3110
3111 deleted
3112 }
3113
3114 pub async fn delete_failed(
3118 state: &crate::State,
3119 scope: FailedServerBackupScope,
3120 force: bool,
3121 ) -> Result<u64, anyhow::Error> {
3122 let options = DeleteServerBackupOptions {
3123 force,
3124 ..Default::default()
3125 };
3126
3127 let mut deleted = 0;
3128 let mut after = None;
3129
3130 loop {
3131 let (claimed, cursor) =
3132 Self::claim_failed_deletions(state, scope, &options, after).await?;
3133
3134 let mut futures = Vec::with_capacity(claimed.len());
3135 for backup in &claimed {
3136 futures.push(backup.dispatch_swept_deletion(state, &options));
3137 }
3138
3139 let mut results_stream = futures_util::stream::iter(futures)
3140 .buffer_unordered(Self::FAILED_SWEEP_CONCURRENCY);
3141
3142 while let Some(dispatched) = results_stream.next().await {
3143 if dispatched {
3144 deleted += 1;
3145 }
3146 }
3147
3148 match cursor {
3149 Some(cursor) => after = Some(cursor),
3150 None => break,
3151 }
3152 }
3153
3154 Ok(deleted)
3155 }
3156}
3157
3158#[derive(ToSchema, Serialize)]
3159#[schema(title = "AdminNodeServerBackup")]
3160pub struct AdminApiNodeServerBackup {
3161 pub uuid: uuid::Uuid,
3162 pub server: Option<super::server::AdminApiServer>,
3163 pub node: super::node::AdminApiNode,
3164 pub backup_group_uuid: Option<uuid::Uuid>,
3165 pub system_backup_policy_uuid: Option<uuid::Uuid>,
3166 pub database_instance_uuid: Option<uuid::Uuid>,
3167
3168 pub kind: ServerBackupKind,
3169 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3170 pub name: compact_str::CompactString,
3171 pub ignored_files: Vec<compact_str::CompactString>,
3172
3173 pub is_successful: bool,
3174 pub is_locked: bool,
3175 pub is_browsable: bool,
3176 pub is_streaming: bool,
3177 pub is_shared: bool,
3178
3179 pub checksum: Option<compact_str::CompactString>,
3180 pub bytes: i64,
3181 pub files: i64,
3182
3183 pub metadata: serde_json::Value,
3184 pub deletion_status: Option<ServerBackupDeletionStatus>,
3185 pub retention_status: Option<retention::ServerBackupRetentionStatus>,
3186
3187 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3188 pub created: chrono::DateTime<chrono::Utc>,
3189}
3190
3191#[schema_extension_derive::extendible]
3192#[init_args(ServerBackup, crate::State)]
3193#[hook_args(crate::State)]
3194#[derive(ToSchema, Serialize)]
3195#[schema(title = "AdminServerBackup")]
3196pub struct AdminApiServerBackup {
3197 pub uuid: uuid::Uuid,
3198 pub server: Option<super::server::AdminApiServer>,
3199 pub backup_group_uuid: Option<uuid::Uuid>,
3200 pub system_backup_policy_uuid: Option<uuid::Uuid>,
3201 pub database_instance_uuid: Option<uuid::Uuid>,
3202
3203 pub kind: ServerBackupKind,
3204 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3205 pub name: compact_str::CompactString,
3206 pub ignored_files: Vec<compact_str::CompactString>,
3207
3208 pub is_successful: bool,
3209 pub is_locked: bool,
3210 pub is_browsable: bool,
3211 pub is_streaming: bool,
3212 pub is_shared: bool,
3213
3214 pub checksum: Option<compact_str::CompactString>,
3215 pub bytes: i64,
3216 pub files: i64,
3217
3218 pub metadata: serde_json::Value,
3219 pub deletion_status: Option<ServerBackupDeletionStatus>,
3220 pub retention_status: Option<retention::ServerBackupRetentionStatus>,
3221
3222 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3223 pub created: chrono::DateTime<chrono::Utc>,
3224}
3225
3226#[derive(ToSchema, Serialize)]
3227#[schema(title = "ServerBackupUsage")]
3228pub struct ServerBackupUsage {
3229 pub server: i64,
3230 pub database_instance: i64,
3231}
3232
3233#[schema_extension_derive::extendible]
3234#[init_args(ServerBackup, crate::State)]
3235#[hook_args(crate::State)]
3236#[derive(ToSchema, Serialize)]
3237#[schema(title = "ServerBackup")]
3238pub struct ApiServerBackup {
3239 pub uuid: uuid::Uuid,
3240 pub backup_group_uuid: Option<uuid::Uuid>,
3241 pub database_instance_uuid: Option<uuid::Uuid>,
3242
3243 pub kind: ServerBackupKind,
3244 pub database_type: Option<db_agent_api::DatabaseAgentType>,
3245 pub name: compact_str::CompactString,
3246 pub ignored_files: Vec<compact_str::CompactString>,
3247
3248 pub is_successful: bool,
3249 pub is_locked: bool,
3250 pub is_browsable: bool,
3251 pub is_streaming: bool,
3252
3253 pub checksum: Option<compact_str::CompactString>,
3254 pub bytes: i64,
3255 pub files: i64,
3256
3257 pub metadata: serde_json::Value,
3258 pub deletion_status: Option<ServerBackupDeletionStatus>,
3259 pub retention_status: Option<retention::ServerBackupRetentionStatus>,
3260
3261 pub completed: Option<chrono::DateTime<chrono::Utc>>,
3262 pub created: chrono::DateTime<chrono::Utc>,
3263}