1mod events;
2pub use events::ServerBackupEvent;
3
4use crate::{
5 jwt::BasePayload,
6 models::{InsertQueryBuilder, UpdateQueryBuilder, server_variable::ServerVariable},
7 prelude::*,
8 storage::StorageUrlRetriever,
9};
10use compact_str::ToCompactString;
11use garde::Validate;
12use reqwest::StatusCode;
13use serde::{Deserialize, Serialize};
14use sqlx::{Row, postgres::PgRow, prelude::Type};
15use std::{
16 collections::{BTreeMap, HashMap},
17 sync::{Arc, LazyLock},
18};
19use utoipa::ToSchema;
20
21#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
22#[serde(rename_all = "kebab-case")]
23#[sqlx(type_name = "backup_disk", rename_all = "SCREAMING_SNAKE_CASE")]
24pub enum BackupDisk {
25 Local,
26 S3,
27 DdupBak,
28 Btrfs,
29 Zfs,
30 Restic,
31 ProxmoxBackupServer,
32 Kopia,
33}
34
35impl BackupDisk {
36 #[inline]
37 pub fn from_wings_adapter(adapter: wings_api::BackupAdapter) -> Self {
38 match adapter {
39 wings_api::BackupAdapter::Wings => BackupDisk::Local,
40 wings_api::BackupAdapter::S3 => BackupDisk::S3,
41 wings_api::BackupAdapter::DdupBak => BackupDisk::DdupBak,
42 wings_api::BackupAdapter::Btrfs => BackupDisk::Btrfs,
43 wings_api::BackupAdapter::Zfs => BackupDisk::Zfs,
44 wings_api::BackupAdapter::Restic => BackupDisk::Restic,
45 wings_api::BackupAdapter::ProxmoxBackupServer => BackupDisk::ProxmoxBackupServer,
46 wings_api::BackupAdapter::Kopia => BackupDisk::Kopia,
47 }
48 }
49
50 #[inline]
51 pub fn to_wings_adapter(self) -> wings_api::BackupAdapter {
52 match self {
53 BackupDisk::Local => wings_api::BackupAdapter::Wings,
54 BackupDisk::S3 => wings_api::BackupAdapter::S3,
55 BackupDisk::DdupBak => wings_api::BackupAdapter::DdupBak,
56 BackupDisk::Btrfs => wings_api::BackupAdapter::Btrfs,
57 BackupDisk::Zfs => wings_api::BackupAdapter::Zfs,
58 BackupDisk::Restic => wings_api::BackupAdapter::Restic,
59 BackupDisk::ProxmoxBackupServer => wings_api::BackupAdapter::ProxmoxBackupServer,
60 BackupDisk::Kopia => wings_api::BackupAdapter::Kopia,
61 }
62 }
63}
64
65pub struct ServerBackupRestoreOptions {
66 pub truncate_directory: bool,
67 pub restore_startup: bool,
68}
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum GroupRotationOutcome {
72 NotConfigured,
74 WithinRetention,
76 Evicted,
78 BlockedAllLocked,
80}
81
82#[derive(Debug, Clone, Copy)]
84enum EvictionScope<'a> {
85 Server,
86 Group(&'a str),
87 Policy(&'a str),
88}
89
90impl<'a> EvictionScope<'a> {
91 #[inline]
92 fn group_name(self) -> Option<&'a str> {
93 match self {
94 Self::Group(name) => Some(name),
95 _ => None,
96 }
97 }
98
99 #[inline]
100 fn policy_name(self) -> Option<&'a str> {
101 match self {
102 Self::Policy(name) => Some(name),
103 _ => None,
104 }
105 }
106}
107
108#[derive(Serialize, Deserialize, Clone)]
109pub struct ServerBackup {
110 pub uuid: uuid::Uuid,
111 pub server: Option<Fetchable<super::server::Server>>,
112 pub node: Fetchable<super::node::Node>,
113 pub backup_configuration: Option<Fetchable<super::backup_configuration::BackupConfiguration>>,
114 pub backup_group_uuid: Option<uuid::Uuid>,
115 pub system_backup_policy_uuid: Option<uuid::Uuid>,
116
117 pub name: compact_str::CompactString,
118 pub successful: bool,
119 pub browsable: bool,
120 pub streaming: bool,
121 pub locked: bool,
122 pub shared: bool,
123
124 pub ignored_files: Vec<compact_str::CompactString>,
125 pub checksum: Option<compact_str::CompactString>,
126 pub bytes: i64,
127 pub files: i64,
128
129 pub disk: BackupDisk,
130 pub upload_id: Option<compact_str::CompactString>,
131 pub upload_path: Option<compact_str::CompactString>,
132 pub metadata: serde_json::Value,
133
134 pub completed: Option<chrono::NaiveDateTime>,
135 pub deleting: Option<chrono::NaiveDateTime>,
136 pub deletion_retries: i32,
137 pub deleted: Option<chrono::NaiveDateTime>,
138 pub created: chrono::NaiveDateTime,
139
140 extension_data: super::ModelExtensionData,
141}
142
143#[derive(Debug, ToSchema, Serialize, Clone, Copy, PartialEq, Eq)]
144#[serde(rename_all = "snake_case")]
145pub enum ServerBackupDeletionStatus {
146 Deleting,
147 Failed,
148}
149
150impl BaseModel for ServerBackup {
151 const NAME: &'static str = "server_backup";
152
153 fn get_extension_list() -> &'static super::ModelExtensionList {
154 static EXTENSIONS: LazyLock<super::ModelExtensionList> =
155 LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
156
157 &EXTENSIONS
158 }
159
160 fn get_extension_data(&self) -> &super::ModelExtensionData {
161 &self.extension_data
162 }
163
164 #[inline]
165 fn base_columns(prefix: Option<&str>) -> BTreeMap<&'static str, compact_str::CompactString> {
166 let prefix = prefix.unwrap_or_default();
167
168 BTreeMap::from([
169 (
170 "server_backups.uuid",
171 compact_str::format_compact!("{prefix}uuid"),
172 ),
173 (
174 "server_backups.server_uuid",
175 compact_str::format_compact!("{prefix}server_uuid"),
176 ),
177 (
178 "server_backups.node_uuid",
179 compact_str::format_compact!("{prefix}node_uuid"),
180 ),
181 (
182 "server_backups.backup_configuration_uuid",
183 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
184 ),
185 (
186 "server_backups.backup_group_uuid",
187 compact_str::format_compact!("{prefix}backup_group_uuid"),
188 ),
189 (
190 "server_backups.system_backup_policy_uuid",
191 compact_str::format_compact!("{prefix}system_backup_policy_uuid"),
192 ),
193 (
194 "server_backups.name",
195 compact_str::format_compact!("{prefix}name"),
196 ),
197 (
198 "server_backups.successful",
199 compact_str::format_compact!("{prefix}successful"),
200 ),
201 (
202 "server_backups.browsable",
203 compact_str::format_compact!("{prefix}browsable"),
204 ),
205 (
206 "server_backups.streaming",
207 compact_str::format_compact!("{prefix}streaming"),
208 ),
209 (
210 "server_backups.locked",
211 compact_str::format_compact!("{prefix}locked"),
212 ),
213 (
214 "server_backups.shared",
215 compact_str::format_compact!("{prefix}shared"),
216 ),
217 (
218 "server_backups.ignored_files",
219 compact_str::format_compact!("{prefix}ignored_files"),
220 ),
221 (
222 "server_backups.checksum",
223 compact_str::format_compact!("{prefix}checksum"),
224 ),
225 (
226 "server_backups.bytes",
227 compact_str::format_compact!("{prefix}bytes"),
228 ),
229 (
230 "server_backups.files",
231 compact_str::format_compact!("{prefix}files"),
232 ),
233 (
234 "server_backups.disk",
235 compact_str::format_compact!("{prefix}disk"),
236 ),
237 (
238 "server_backups.upload_id",
239 compact_str::format_compact!("{prefix}upload_id"),
240 ),
241 (
242 "server_backups.upload_path",
243 compact_str::format_compact!("{prefix}upload_path"),
244 ),
245 (
246 "server_backups.metadata",
247 compact_str::format_compact!("{prefix}metadata"),
248 ),
249 (
250 "server_backups.completed",
251 compact_str::format_compact!("{prefix}completed"),
252 ),
253 (
254 "server_backups.deleting",
255 compact_str::format_compact!("{prefix}deleting"),
256 ),
257 (
258 "server_backups.deletion_retries",
259 compact_str::format_compact!("{prefix}deletion_retries"),
260 ),
261 (
262 "server_backups.deleted",
263 compact_str::format_compact!("{prefix}deleted"),
264 ),
265 (
266 "server_backups.created",
267 compact_str::format_compact!("{prefix}created"),
268 ),
269 ])
270 }
271
272 #[inline]
273 fn map(prefix: Option<&str>, row: &PgRow) -> Result<Self, crate::database::DatabaseError> {
274 let prefix = prefix.unwrap_or_default();
275
276 Ok(Self {
277 uuid: row.try_get(compact_str::format_compact!("{prefix}uuid").as_str())?,
278 server: super::server::Server::get_fetchable_from_row(
279 row,
280 compact_str::format_compact!("{prefix}server_uuid"),
281 ),
282 backup_configuration:
283 super::backup_configuration::BackupConfiguration::get_fetchable_from_row(
284 row,
285 compact_str::format_compact!("{prefix}backup_configuration_uuid"),
286 ),
287 node: super::node::Node::get_fetchable(
288 row.try_get(compact_str::format_compact!("{prefix}node_uuid").as_str())?,
289 ),
290 backup_group_uuid: row
291 .try_get(compact_str::format_compact!("{prefix}backup_group_uuid").as_str())?,
292 system_backup_policy_uuid: row.try_get(
293 compact_str::format_compact!("{prefix}system_backup_policy_uuid").as_str(),
294 )?,
295 name: row.try_get(compact_str::format_compact!("{prefix}name").as_str())?,
296 successful: row.try_get(compact_str::format_compact!("{prefix}successful").as_str())?,
297 browsable: row.try_get(compact_str::format_compact!("{prefix}browsable").as_str())?,
298 streaming: row.try_get(compact_str::format_compact!("{prefix}streaming").as_str())?,
299 locked: row.try_get(compact_str::format_compact!("{prefix}locked").as_str())?,
300 shared: row.try_get(compact_str::format_compact!("{prefix}shared").as_str())?,
301 ignored_files: row
302 .try_get(compact_str::format_compact!("{prefix}ignored_files").as_str())?,
303 checksum: row.try_get(compact_str::format_compact!("{prefix}checksum").as_str())?,
304 bytes: row.try_get(compact_str::format_compact!("{prefix}bytes").as_str())?,
305 files: row.try_get(compact_str::format_compact!("{prefix}files").as_str())?,
306 disk: row.try_get(compact_str::format_compact!("{prefix}disk").as_str())?,
307 upload_id: row.try_get(compact_str::format_compact!("{prefix}upload_id").as_str())?,
308 upload_path: row
309 .try_get(compact_str::format_compact!("{prefix}upload_path").as_str())?,
310 metadata: row.try_get(compact_str::format_compact!("{prefix}metadata").as_str())?,
311 completed: row.try_get(compact_str::format_compact!("{prefix}completed").as_str())?,
312 deleting: row.try_get(compact_str::format_compact!("{prefix}deleting").as_str())?,
313 deletion_retries: row
314 .try_get(compact_str::format_compact!("{prefix}deletion_retries").as_str())?,
315 deleted: row.try_get(compact_str::format_compact!("{prefix}deleted").as_str())?,
316 created: row.try_get(compact_str::format_compact!("{prefix}created").as_str())?,
317 extension_data: Self::map_extensions(prefix, row)?,
318 })
319 }
320}
321
322impl ServerBackup {
323 pub async fn create_raw(
324 state: &crate::State,
325 mut options: CreateServerBackupOptions<'_>,
326 ) -> Result<Self, anyhow::Error> {
327 let backup_configuration = match options.backup_configuration.take() {
328 Some(backup_configuration) => backup_configuration,
329 None => options
330 .server
331 .backup_configuration(&state.database)
332 .await
333 .ok_or_else(|| {
334 crate::response::DisplayError::new(
335 "no backup configuration available, unable to create backup",
336 )
337 .with_status(StatusCode::EXPECTATION_FAILED)
338 })?,
339 };
340
341 if backup_configuration.maintenance_enabled {
342 return Err(crate::response::DisplayError::new(
343 "cannot create backup while backup configuration is in maintenance mode",
344 )
345 .with_status(StatusCode::EXPECTATION_FAILED)
346 .into());
347 }
348
349 let mut transaction = state.database.write().begin().await?;
350
351 let mut query_builder = InsertQueryBuilder::new("server_backups");
352
353 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
354 .await?;
355
356 query_builder
357 .set("server_uuid", options.server.uuid)
358 .set("node_uuid", options.server.node.uuid)
359 .set("backup_configuration_uuid", backup_configuration.uuid)
360 .set("backup_group_uuid", options.backup_group_uuid)
361 .set(
362 "system_backup_policy_uuid",
363 options.system_backup_policy_uuid,
364 )
365 .set("name", &options.name)
366 .set("ignored_files", &options.ignored_files)
367 .set("bytes", 0i64)
368 .set("disk", backup_configuration.backup_disk)
369 .set("shared", backup_configuration.shared)
370 .set("metadata", &options.metadata);
371
372 let row = query_builder
373 .returning(&Self::columns_sql(None))
374 .fetch_one(&mut *transaction)
375 .await?;
376 let mut backup = Self::map(None, &row)?;
377
378 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
379
380 transaction.commit().await?;
381
382 Ok(backup)
383 }
384
385 pub async fn by_server_uuid_uuid(
386 database: &crate::database::Database,
387 server_uuid: uuid::Uuid,
388 uuid: uuid::Uuid,
389 ) -> Result<Option<Self>, crate::database::DatabaseError> {
390 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
391 r#"
392 SELECT {}
393 FROM server_backups
394 WHERE server_backups.server_uuid = $1 AND server_backups.uuid = $2
395 "#,
396 Self::columns_sql(None)
397 )))
398 .bind(server_uuid)
399 .bind(uuid)
400 .fetch_optional(database.read())
401 .await?;
402
403 row.try_map(|row| Self::map(None, &row))
404 }
405
406 pub async fn select_completed_by_server_uuid(
407 database: &crate::database::Database,
408 server_uuid: uuid::Uuid,
409 name: Option<&str>,
410 backup_group_uuid: Option<uuid::Uuid>,
411 oldest: bool,
412 ) -> Result<Option<Self>, crate::database::DatabaseError> {
413 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
414 r#"
415 SELECT {}
416 FROM server_backups
417 WHERE
418 server_backups.server_uuid = $1
419 AND server_backups.deleted IS NULL
420 AND server_backups.deleting IS NULL
421 AND server_backups.completed IS NOT NULL
422 AND server_backups.successful
423 AND server_backups.system_backup_policy_uuid IS NULL
424 AND ($2 IS NULL OR server_backups.name = $2)
425 AND ($3::uuid IS NULL OR server_backups.backup_group_uuid = $3)
426 ORDER BY server_backups.created {}
427 LIMIT 1
428 "#,
429 Self::columns_sql(None),
430 if oldest { "ASC" } else { "DESC" }
431 )))
432 .bind(server_uuid)
433 .bind(name)
434 .bind(backup_group_uuid)
435 .fetch_optional(database.read())
436 .await?;
437
438 row.try_map(|row| Self::map(None, &row))
439 }
440
441 pub async fn by_node_uuid_uuid(
442 database: &crate::database::Database,
443 node_uuid: uuid::Uuid,
444 uuid: uuid::Uuid,
445 ) -> Result<Option<Self>, crate::database::DatabaseError> {
446 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
447 r#"
448 SELECT {}
449 FROM server_backups
450 WHERE server_backups.node_uuid = $1 AND server_backups.uuid = $2
451 "#,
452 Self::columns_sql(None)
453 )))
454 .bind(node_uuid)
455 .bind(uuid)
456 .fetch_optional(database.read())
457 .await?;
458
459 row.try_map(|row| Self::map(None, &row))
460 }
461
462 pub async fn by_server_uuid_with_pagination(
463 database: &crate::database::Database,
464 server_uuid: uuid::Uuid,
465 page: i64,
466 per_page: i64,
467 search: Option<&str>,
468 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
469 let offset = (page - 1) * per_page;
470
471 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
472 r#"
473 SELECT {}, COUNT(*) OVER() AS total_count
474 FROM server_backups
475 WHERE
476 server_backups.server_uuid = $1
477 AND server_backups.deleted IS NULL
478 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
479 ORDER BY server_backups.created
480 LIMIT $3 OFFSET $4
481 "#,
482 Self::columns_sql(None)
483 )))
484 .bind(server_uuid)
485 .bind(search)
486 .bind(per_page)
487 .bind(offset)
488 .fetch_all(database.read())
489 .await?;
490
491 Ok(super::Pagination {
492 total: rows
493 .first()
494 .map_or(Ok(0), |row| row.try_get("total_count"))?,
495 per_page,
496 page,
497 data: rows
498 .into_iter()
499 .map(|row| Self::map(None, &row))
500 .try_collect_vec()?,
501 })
502 }
503
504 pub async fn by_server_uuid_node_uuid_with_pagination(
505 database: &crate::database::Database,
506 server_uuid: uuid::Uuid,
507 node_uuid: uuid::Uuid,
508 page: i64,
509 per_page: i64,
510 search: Option<&str>,
511 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
512 let offset = (page - 1) * per_page;
513
514 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
515 r#"
516 SELECT {}, COUNT(*) OVER() AS total_count
517 FROM server_backups
518 WHERE
519 server_backups.server_uuid = $1
520 AND server_backups.node_uuid = $2
521 AND server_backups.system_backup_policy_uuid IS NULL
522 AND server_backups.deleted IS NULL
523 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
524 ORDER BY server_backups.created
525 LIMIT $4 OFFSET $5
526 "#,
527 Self::columns_sql(None)
528 )))
529 .bind(server_uuid)
530 .bind(node_uuid)
531 .bind(search)
532 .bind(per_page)
533 .bind(offset)
534 .fetch_all(database.read())
535 .await?;
536
537 Ok(super::Pagination {
538 total: rows
539 .first()
540 .map_or(Ok(0), |row| row.try_get("total_count"))?,
541 per_page,
542 page,
543 data: rows
544 .into_iter()
545 .map(|row| Self::map(None, &row))
546 .try_collect_vec()?,
547 })
548 }
549
550 pub async fn by_server_uuid_node_uuid_backup_group_uuid_with_pagination(
551 database: &crate::database::Database,
552 server_uuid: uuid::Uuid,
553 node_uuid: uuid::Uuid,
554 backup_group_uuid: uuid::Uuid,
555 page: i64,
556 per_page: i64,
557 search: Option<&str>,
558 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
559 let offset = (page - 1) * per_page;
560
561 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
562 r#"
563 SELECT {}, COUNT(*) OVER() AS total_count
564 FROM server_backups
565 WHERE
566 server_backups.server_uuid = $1
567 AND server_backups.node_uuid = $2
568 AND server_backups.backup_group_uuid = $3
569 AND server_backups.deleted IS NULL
570 AND ($4 IS NULL OR server_backups.name ILIKE '%' || $4 || '%')
571 ORDER BY server_backups.created
572 LIMIT $5 OFFSET $6
573 "#,
574 Self::columns_sql(None)
575 )))
576 .bind(server_uuid)
577 .bind(node_uuid)
578 .bind(backup_group_uuid)
579 .bind(search)
580 .bind(per_page)
581 .bind(offset)
582 .fetch_all(database.read())
583 .await?;
584
585 Ok(super::Pagination {
586 total: rows
587 .first()
588 .map_or(Ok(0), |row| row.try_get("total_count"))?,
589 per_page,
590 page,
591 data: rows
592 .into_iter()
593 .map(|row| Self::map(None, &row))
594 .try_collect_vec()?,
595 })
596 }
597
598 pub async fn by_ungrouped_server_uuid_node_uuid_with_pagination(
599 database: &crate::database::Database,
600 server_uuid: uuid::Uuid,
601 node_uuid: uuid::Uuid,
602 page: i64,
603 per_page: i64,
604 search: Option<&str>,
605 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
606 let offset = (page - 1) * per_page;
607
608 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
609 r#"
610 SELECT {}, COUNT(*) OVER() AS total_count
611 FROM server_backups
612 WHERE
613 server_backups.server_uuid = $1
614 AND server_backups.node_uuid = $2
615 AND server_backups.backup_group_uuid IS NULL
616 AND server_backups.system_backup_policy_uuid IS NULL
617 AND server_backups.deleted IS NULL
618 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
619 ORDER BY server_backups.created
620 LIMIT $4 OFFSET $5
621 "#,
622 Self::columns_sql(None)
623 )))
624 .bind(server_uuid)
625 .bind(node_uuid)
626 .bind(search)
627 .bind(per_page)
628 .bind(offset)
629 .fetch_all(database.read())
630 .await?;
631
632 Ok(super::Pagination {
633 total: rows
634 .first()
635 .map_or(Ok(0), |row| row.try_get("total_count"))?,
636 per_page,
637 page,
638 data: rows
639 .into_iter()
640 .map(|row| Self::map(None, &row))
641 .try_collect_vec()?,
642 })
643 }
644
645 pub async fn by_system_server_uuid_node_uuid_with_pagination(
646 database: &crate::database::Database,
647 server_uuid: uuid::Uuid,
648 node_uuid: uuid::Uuid,
649 page: i64,
650 per_page: i64,
651 search: Option<&str>,
652 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
653 let offset = (page - 1) * per_page;
654
655 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
656 r#"
657 SELECT {}, COUNT(*) OVER() AS total_count
658 FROM server_backups
659 WHERE
660 server_backups.server_uuid = $1
661 AND server_backups.node_uuid = $2
662 AND server_backups.system_backup_policy_uuid IS NOT NULL
663 AND server_backups.deleted IS NULL
664 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
665 ORDER BY server_backups.created
666 LIMIT $4 OFFSET $5
667 "#,
668 Self::columns_sql(None)
669 )))
670 .bind(server_uuid)
671 .bind(node_uuid)
672 .bind(search)
673 .bind(per_page)
674 .bind(offset)
675 .fetch_all(database.read())
676 .await?;
677
678 Ok(super::Pagination {
679 total: rows
680 .first()
681 .map_or(Ok(0), |row| row.try_get("total_count"))?,
682 per_page,
683 page,
684 data: rows
685 .into_iter()
686 .map(|row| Self::map(None, &row))
687 .try_collect_vec()?,
688 })
689 }
690
691 pub async fn by_system_backup_policy_uuid_with_pagination(
692 database: &crate::database::Database,
693 system_backup_policy_uuid: uuid::Uuid,
694 page: i64,
695 per_page: i64,
696 search: Option<&str>,
697 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
698 let offset = (page - 1) * per_page;
699
700 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
701 r#"
702 SELECT {}, COUNT(*) OVER() AS total_count
703 FROM server_backups
704 WHERE
705 server_backups.system_backup_policy_uuid = $1
706 AND server_backups.deleted IS NULL
707 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
708 ORDER BY server_backups.created
709 LIMIT $3 OFFSET $4
710 "#,
711 Self::columns_sql(None)
712 )))
713 .bind(system_backup_policy_uuid)
714 .bind(search)
715 .bind(per_page)
716 .bind(offset)
717 .fetch_all(database.read())
718 .await?;
719
720 Ok(super::Pagination {
721 total: rows
722 .first()
723 .map_or(Ok(0), |row| row.try_get("total_count"))?,
724 per_page,
725 page,
726 data: rows
727 .into_iter()
728 .map(|row| Self::map(None, &row))
729 .try_collect_vec()?,
730 })
731 }
732
733 pub async fn all_by_system_backup_policy_uuid(
734 database: &crate::database::Database,
735 system_backup_policy_uuid: uuid::Uuid,
736 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
737 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
738 r#"
739 SELECT {}
740 FROM server_backups
741 WHERE
742 server_backups.system_backup_policy_uuid = $1
743 AND server_backups.deleted IS NULL
744 "#,
745 Self::columns_sql(None)
746 )))
747 .bind(system_backup_policy_uuid)
748 .fetch_all(database.read())
749 .await?;
750
751 rows.into_iter()
752 .map(|row| Self::map(None, &row))
753 .try_collect_vec()
754 }
755
756 pub async fn by_partially_detached_server_uuid_node_uuid_with_pagination(
757 database: &crate::database::Database,
758 server_uuid: uuid::Uuid,
759 node_uuid: uuid::Uuid,
760 page: i64,
761 per_page: i64,
762 search: Option<&str>,
763 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
764 let offset = (page - 1) * per_page;
765
766 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
767 r#"
768 SELECT {}, COUNT(*) OVER() AS total_count
769 FROM server_backups
770 WHERE
771 server_backups.server_uuid = $1
772 AND server_backups.node_uuid != $2
773 AND server_backups.deleted IS NULL
774 AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
775 ORDER BY server_backups.created
776 LIMIT $4 OFFSET $5
777 "#,
778 Self::columns_sql(None)
779 )))
780 .bind(server_uuid)
781 .bind(node_uuid)
782 .bind(search)
783 .bind(per_page)
784 .bind(offset)
785 .fetch_all(database.read())
786 .await?;
787
788 Ok(super::Pagination {
789 total: rows
790 .first()
791 .map_or(Ok(0), |row| row.try_get("total_count"))?,
792 per_page,
793 page,
794 data: rows
795 .into_iter()
796 .map(|row| Self::map(None, &row))
797 .try_collect_vec()?,
798 })
799 }
800
801 pub async fn by_node_uuid_with_pagination(
802 database: &crate::database::Database,
803 node_uuid: uuid::Uuid,
804 page: i64,
805 per_page: i64,
806 search: Option<&str>,
807 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
808 let offset = (page - 1) * per_page;
809
810 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
811 r#"
812 SELECT {}, COUNT(*) OVER() AS total_count
813 FROM server_backups
814 WHERE
815 server_backups.node_uuid = $1
816 AND server_backups.deleted IS NULL
817 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
818 ORDER BY server_backups.created
819 LIMIT $3 OFFSET $4
820 "#,
821 Self::columns_sql(None)
822 )))
823 .bind(node_uuid)
824 .bind(search)
825 .bind(per_page)
826 .bind(offset)
827 .fetch_all(database.read())
828 .await?;
829
830 Ok(super::Pagination {
831 total: rows
832 .first()
833 .map_or(Ok(0), |row| row.try_get("total_count"))?,
834 per_page,
835 page,
836 data: rows
837 .into_iter()
838 .map(|row| Self::map(None, &row))
839 .try_collect_vec()?,
840 })
841 }
842
843 pub async fn by_backup_configuration_uuid_with_pagination(
844 database: &crate::database::Database,
845 backup_configuration_uuid: uuid::Uuid,
846 page: i64,
847 per_page: i64,
848 search: Option<&str>,
849 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
850 let offset = (page - 1) * per_page;
851
852 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
853 r#"
854 SELECT {}, COUNT(*) OVER() AS total_count
855 FROM server_backups
856 WHERE
857 server_backups.backup_configuration_uuid = $1
858 AND server_backups.deleted IS NULL
859 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
860 ORDER BY server_backups.created
861 LIMIT $3 OFFSET $4
862 "#,
863 Self::columns_sql(None)
864 )))
865 .bind(backup_configuration_uuid)
866 .bind(search)
867 .bind(per_page)
868 .bind(offset)
869 .fetch_all(database.read())
870 .await?;
871
872 Ok(super::Pagination {
873 total: rows
874 .first()
875 .map_or(Ok(0), |row| row.try_get("total_count"))?,
876 per_page,
877 page,
878 data: rows
879 .into_iter()
880 .map(|row| Self::map(None, &row))
881 .try_collect_vec()?,
882 })
883 }
884
885 pub async fn by_detached_node_uuid_with_pagination(
886 database: &crate::database::Database,
887 node_uuid: uuid::Uuid,
888 page: i64,
889 per_page: i64,
890 search: Option<&str>,
891 ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
892 let offset = (page - 1) * per_page;
893
894 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
895 r#"
896 SELECT {}, COUNT(*) OVER() AS total_count
897 FROM server_backups
898 WHERE
899 server_backups.node_uuid = $1
900 AND server_backups.server_uuid IS NULL
901 AND server_backups.deleted IS NULL
902 AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
903 ORDER BY server_backups.created
904 LIMIT $3 OFFSET $4
905 "#,
906 Self::columns_sql(None)
907 )))
908 .bind(node_uuid)
909 .bind(search)
910 .bind(per_page)
911 .bind(offset)
912 .fetch_all(database.read())
913 .await?;
914
915 Ok(super::Pagination {
916 total: rows
917 .first()
918 .map_or(Ok(0), |row| row.try_get("total_count"))?,
919 per_page,
920 page,
921 data: rows
922 .into_iter()
923 .map(|row| Self::map(None, &row))
924 .try_collect_vec()?,
925 })
926 }
927
928 pub async fn all_uuids_by_server_uuid(
929 database: &crate::database::Database,
930 server_uuid: uuid::Uuid,
931 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
932 let rows = sqlx::query(
933 r#"
934 SELECT server_backups.uuid
935 FROM server_backups
936 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
937 "#,
938 )
939 .bind(server_uuid)
940 .fetch_all(database.read())
941 .await?;
942
943 Ok(rows
944 .into_iter()
945 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
946 .collect())
947 }
948
949 pub async fn all_uuids_by_server_uuid_not_shared(
950 database: &crate::database::Database,
951 server_uuid: uuid::Uuid,
952 ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
953 let rows = sqlx::query(
954 r#"
955 SELECT server_backups.uuid
956 FROM server_backups
957 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL AND server_backups.shared = false
958 "#,
959 )
960 .bind(server_uuid)
961 .fetch_all(database.read())
962 .await?;
963
964 Ok(rows
965 .into_iter()
966 .map(|row| row.get::<uuid::Uuid, _>("uuid"))
967 .collect())
968 }
969
970 pub async fn all_by_server_uuid(
971 database: &crate::database::Database,
972 server_uuid: uuid::Uuid,
973 ) -> Result<Vec<Self>, crate::database::DatabaseError> {
974 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
975 r#"
976 SELECT {}
977 FROM server_backups
978 WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
979 "#,
980 Self::columns_sql(None)
981 )))
982 .bind(server_uuid)
983 .fetch_all(database.read())
984 .await?;
985
986 rows.into_iter()
987 .map(|row| Self::map(None, &row))
988 .try_collect_vec()
989 }
990
991 pub async fn count_by_server_uuid(
992 database: &crate::database::Database,
993 server_uuid: uuid::Uuid,
994 ) -> Result<i64, sqlx::Error> {
995 sqlx::query_scalar(
996 r#"
997 SELECT COUNT(*)
998 FROM server_backups
999 WHERE
1000 server_backups.server_uuid = $1
1001 AND server_backups.system_backup_policy_uuid IS NULL
1002 AND server_backups.deleted IS NULL
1003 "#,
1004 )
1005 .bind(server_uuid)
1006 .fetch_one(database.read())
1007 .await
1008 }
1009
1010 pub async fn count_system_inflight_by_system_backup_policy_uuid_node_uuid(
1014 database: &crate::database::Database,
1015 system_backup_policy_uuid: uuid::Uuid,
1016 node_uuid: uuid::Uuid,
1017 ) -> Result<i64, sqlx::Error> {
1018 sqlx::query_scalar(
1019 r#"
1020 SELECT COUNT(*)
1021 FROM server_backups
1022 WHERE
1023 server_backups.system_backup_policy_uuid = $1
1024 AND server_backups.node_uuid = $2
1025 AND server_backups.completed IS NULL
1026 AND server_backups.deleted IS NULL
1027 AND server_backups.created >= NOW() - INTERVAL '1 day'
1028 "#,
1029 )
1030 .bind(system_backup_policy_uuid)
1031 .bind(node_uuid)
1032 .fetch_one(database.read())
1033 .await
1034 }
1035
1036 pub async fn generate_metadata(
1037 state: &crate::State,
1038 server: &super::server::Server,
1039 ) -> Result<serde_json::Value, anyhow::Error> {
1040 let mut variables = serde_json::Map::new();
1041
1042 for variable in ServerVariable::all_by_server_uuid_egg_uuid(
1043 &state.database,
1044 server.uuid,
1045 server.egg.uuid,
1046 )
1047 .await?
1048 {
1049 variables.insert(variable.variable.env_variable.into(), variable.value.into());
1050 }
1051
1052 Ok(serde_json::json!({
1053 "startup": server.startup,
1054 "image": server.image,
1055 "variables": variables,
1056 }))
1057 }
1058
1059 pub async fn download_url(
1060 &self,
1061 state: &crate::State,
1062 user: &super::user::User,
1063 node: &super::node::Node,
1064 archive_format: wings_api::StreamableArchiveFormat,
1065 ) -> Result<String, anyhow::Error> {
1066 let backup_configuration = self
1067 .backup_configuration
1068 .as_ref()
1069 .ok_or_else(|| {
1070 crate::response::DisplayError::new(
1071 "no backup configuration available, unable to restore backup",
1072 )
1073 .with_status(StatusCode::EXPECTATION_FAILED)
1074 })?
1075 .fetch_cached(&state.database)
1076 .await?;
1077
1078 if backup_configuration.maintenance_enabled {
1079 return Err(crate::response::DisplayError::new(
1080 "cannot restore backup while backup configuration is in maintenance mode",
1081 )
1082 .with_status(StatusCode::EXPECTATION_FAILED)
1083 .into());
1084 }
1085
1086 if matches!(self.disk, BackupDisk::S3)
1087 && let Some(mut s3_configuration) = backup_configuration.backup_configs.s3
1088 {
1089 s3_configuration.decrypt(&state.database).await?;
1090
1091 let (client, bucket) = s3_configuration.into_client();
1092
1093 let file_path = match &self.upload_path {
1094 Some(path) => path,
1095 None => {
1096 return Err(crate::response::DisplayError::new(
1097 "backup does not have an upload path",
1098 )
1099 .with_status(StatusCode::EXPECTATION_FAILED)
1100 .into());
1101 }
1102 };
1103
1104 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1105 std::time::Duration::from_mins(15),
1106 )?;
1107 let presigned = client
1108 .get_object()
1109 .bucket(bucket)
1110 .key(&**file_path)
1111 .presigned(presigning_config)
1112 .await?;
1113
1114 return Ok(presigned.uri().to_string());
1115 }
1116
1117 #[derive(Serialize)]
1118 struct BackupDownloadJwt {
1119 #[serde(flatten)]
1120 base: BasePayload,
1121
1122 backup_uuid: uuid::Uuid,
1123 unique_id: uuid::Uuid,
1124 }
1125
1126 let token = node.create_jwt(
1127 &state.database,
1128 &state.jwt,
1129 &BackupDownloadJwt {
1130 base: BasePayload {
1131 scope: "backup-download".into(),
1132 issuer: "panel".into(),
1133 subject: None,
1134 audience: Vec::new(),
1135 expiration_time: Some(chrono::Utc::now().timestamp() + 900),
1136 not_before: None,
1137 issued_at: Some(chrono::Utc::now().timestamp()),
1138 jwt_id: user.uuid.to_compact_string(),
1139 },
1140 backup_uuid: self.uuid,
1141 unique_id: uuid::Uuid::new_v4(),
1142 },
1143 )?;
1144
1145 let mut url = node.public_url(state, "/download/backup").await?;
1146 url.set_query(Some(&format!(
1147 "token={}&archive_format={}",
1148 urlencoding::encode(&token),
1149 archive_format
1150 )));
1151
1152 Ok(url.to_string())
1153 }
1154
1155 pub async fn restore(
1156 self,
1157 state: &crate::State,
1158 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1159 mut server: super::server::Server,
1160 options: ServerBackupRestoreOptions,
1161 ) -> Result<(), anyhow::Error> {
1162 let backup_configuration = self
1163 .backup_configuration
1164 .as_ref()
1165 .ok_or_else(|| {
1166 crate::response::DisplayError::new(
1167 "no backup configuration available, unable to restore backup",
1168 )
1169 .with_status(StatusCode::EXPECTATION_FAILED)
1170 })?
1171 .fetch_cached(&state.database)
1172 .await?;
1173
1174 if backup_configuration.maintenance_enabled {
1175 return Err(crate::response::DisplayError::new(
1176 "cannot restore backup while backup configuration is in maintenance mode",
1177 )
1178 .with_status(StatusCode::EXPECTATION_FAILED)
1179 .into());
1180 }
1181
1182 if options.restore_startup {
1183 self.restore_startup(state, transaction, &mut server)
1184 .await?;
1185 }
1186
1187 server
1188 .node
1189 .fetch_cached(&state.database)
1190 .await?
1191 .api_client(&state.database)
1192 .await?
1193 .post_servers_server_backup_backup_restore(
1194 server.uuid,
1195 self.uuid,
1196 &wings_api::servers_server_backup_backup_restore::post::RequestBody {
1197 adapter: self.disk.to_wings_adapter(),
1198 download_url: self.wings_restore_download_url(state, server.uuid).await?,
1199 truncate_directory: options.truncate_directory,
1200 },
1201 )
1202 .await?;
1203
1204 Self::get_event_emitter().emit(
1205 state.clone(),
1206 ServerBackupEvent::RestoreStarted {
1207 backup: Box::new(self),
1208 server: Box::new(server),
1209 },
1210 );
1211
1212 Ok(())
1213 }
1214
1215 pub async fn restore_startup(
1216 &self,
1217 state: &crate::State,
1218 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1219 server: &mut super::server::Server,
1220 ) -> Result<(), anyhow::Error> {
1221 let startup_cmd = self
1222 .metadata
1223 .get("startup")
1224 .and_then(|v| v.as_str())
1225 .map(|s| s.to_compact_string());
1226 let image_str = self
1227 .metadata
1228 .get("image")
1229 .and_then(|v| v.as_str())
1230 .map(|s| s.to_compact_string());
1231 let variables = self
1232 .metadata
1233 .get("variables")
1234 .and_then(|v| v.as_object())
1235 .cloned()
1236 .unwrap_or_default();
1237
1238 if let Some(startup) = startup_cmd
1239 && let Ok(egg_config) = server.egg.configuration(&state.database).await
1240 {
1241 let is_predefined = server
1242 .egg
1243 .startup_commands
1244 .values()
1245 .any(|cmd| cmd == startup.as_str());
1246 let custom_allowed = egg_config
1247 .config_startup
1248 .as_ref()
1249 .is_some_and(|c| c.allow_custom_startup_command);
1250 if is_predefined || custom_allowed {
1251 server
1252 .update_with_transaction(
1253 state,
1254 super::server::UpdateServerOptions {
1255 startup: Some(startup),
1256 ..Default::default()
1257 },
1258 transaction,
1259 )
1260 .await?;
1261 }
1262 }
1263
1264 if let Some(image) = image_str {
1265 let is_valid_image = server
1266 .egg
1267 .docker_images
1268 .values()
1269 .any(|img| img == image.as_str());
1270 if is_valid_image {
1271 let current_is_custom = !server
1272 .egg
1273 .docker_images
1274 .values()
1275 .any(|img| img == server.image.as_str());
1276 let allow_overwrite = state
1277 .settings
1278 .get()
1279 .await
1280 .map(|s| s.server.allow_overwriting_custom_docker_image)
1281 .unwrap_or(false);
1282 if !current_is_custom || allow_overwrite {
1283 server
1284 .update_with_transaction(
1285 state,
1286 super::server::UpdateServerOptions {
1287 image: Some(image),
1288 ..Default::default()
1289 },
1290 transaction,
1291 )
1292 .await?;
1293 }
1294 }
1295 }
1296
1297 if !variables.is_empty() {
1298 let existing_variables = ServerVariable::all_by_server_uuid_egg_uuid(
1299 &state.database,
1300 server.uuid,
1301 server.egg.uuid,
1302 )
1303 .await?;
1304
1305 let mut validator_variables = HashMap::new();
1306 for variable in existing_variables.iter() {
1307 validator_variables.insert(
1308 variable.variable.env_variable.as_str(),
1309 (
1310 variable.variable.rules.as_slice(),
1311 if let Some(value) = variables
1312 .iter()
1313 .find(|v| v.0 == variable.variable.env_variable)
1314 && variable.variable.user_editable
1315 && let Some(value) = value.1.as_str()
1316 {
1317 value
1318 } else {
1319 variable.value.as_str()
1320 },
1321 ),
1322 );
1323 }
1324
1325 let validator = match rule_validator::Validator::new(validator_variables) {
1326 Ok(validator) => validator,
1327 Err(error) => {
1328 return Err(crate::response::DisplayError::new(error)
1329 .with_status(StatusCode::EXPECTATION_FAILED)
1330 .into());
1331 }
1332 };
1333 if let Err(error) = validator.validate() {
1334 return Err(crate::response::DisplayError::new(error)
1335 .with_status(StatusCode::EXPECTATION_FAILED)
1336 .into());
1337 }
1338
1339 for (env_var, value) in &variables {
1340 let Some(value) = value.as_str() else {
1341 continue;
1342 };
1343 let variable_uuid = match existing_variables
1344 .iter()
1345 .find(|v| v.variable.env_variable == env_var)
1346 {
1347 Some(variable) if variable.variable.user_editable => variable.variable.uuid,
1348 _ => continue,
1349 };
1350
1351 ServerVariable::create_with_transaction(
1352 transaction,
1353 server.uuid,
1354 variable_uuid,
1355 value,
1356 )
1357 .await?;
1358 }
1359 }
1360
1361 Ok(())
1362 }
1363
1364 pub async fn wings_restore_download_url(
1365 &self,
1366 state: &crate::State,
1367 server_uuid: uuid::Uuid,
1368 ) -> Result<Option<compact_str::CompactString>, anyhow::Error> {
1369 if !matches!(self.disk, BackupDisk::S3) {
1370 return Ok(None);
1371 }
1372
1373 let backup_configuration = self
1374 .backup_configuration
1375 .as_ref()
1376 .ok_or_else(|| {
1377 crate::response::DisplayError::new(
1378 "no backup configuration available, unable to restore backup",
1379 )
1380 .with_status(StatusCode::EXPECTATION_FAILED)
1381 })?
1382 .fetch_cached(&state.database)
1383 .await?;
1384
1385 let Some(mut s3_configuration) = backup_configuration.backup_configs.s3 else {
1386 return Ok(None);
1387 };
1388
1389 s3_configuration.decrypt(&state.database).await?;
1390
1391 let compression_type = s3_configuration.compression_type;
1392 let (client, bucket) = s3_configuration.into_client();
1393
1394 let file_path = match &self.upload_path {
1395 Some(path) => path.as_str(),
1396 None => &Self::s3_path(server_uuid, self.uuid, compression_type),
1397 };
1398
1399 let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1400 std::time::Duration::from_mins(60),
1401 )?;
1402 let presigned = client
1403 .get_object()
1404 .bucket(bucket)
1405 .key(file_path)
1406 .presigned(presigning_config)
1407 .await?;
1408
1409 Ok(Some(presigned.uri().to_compact_string()))
1410 }
1411
1412 pub async fn export(
1413 &self,
1414 state: &crate::State,
1415 server: &super::server::Server,
1416 path: compact_str::CompactString,
1417 archive_format: wings_api::StreamableArchiveFormat,
1418 foreground: bool,
1419 ) -> Result<wings_api::backups_backup_export::post::Response, anyhow::Error> {
1420 let backup_configuration = self
1421 .backup_configuration
1422 .as_ref()
1423 .ok_or_else(|| {
1424 crate::response::DisplayError::new(
1425 "no backup configuration available, unable to export backup",
1426 )
1427 .with_status(StatusCode::EXPECTATION_FAILED)
1428 })?
1429 .fetch_cached(&state.database)
1430 .await?;
1431
1432 if backup_configuration.maintenance_enabled {
1433 return Err(crate::response::DisplayError::new(
1434 "cannot export backup while backup configuration is in maintenance mode",
1435 )
1436 .with_status(StatusCode::EXPECTATION_FAILED)
1437 .into());
1438 }
1439
1440 let client = server
1441 .node
1442 .fetch_cached(&state.database)
1443 .await?
1444 .api_client(&state.database)
1445 .await?;
1446
1447 match client
1448 .post_backups_backup_export(
1449 self.uuid,
1450 &wings_api::backups_backup_export::post::RequestBody {
1451 adapter: self.disk.to_wings_adapter(),
1452 server: server.uuid,
1453 path,
1454 archive_format,
1455 foreground,
1456 },
1457 )
1458 .await
1459 {
1460 Ok(response) => Ok(response),
1461 Err(wings_api::client::ApiHttpError::Http(
1462 status @ (StatusCode::NOT_FOUND | StatusCode::EXPECTATION_FAILED),
1463 err,
1464 )) => Err(crate::response::DisplayError::new(
1465 crate::ApiError::new_wings_value(err).to_string(),
1466 )
1467 .with_status(status)
1468 .into()),
1469 Err(err) => Err(err.into()),
1470 }
1471 }
1472
1473 pub async fn query(
1474 &self,
1475 state: &crate::State,
1476 node: &super::node::Node,
1477 ) -> Result<wings_api::backups_backup_query::get::Response, anyhow::Error> {
1478 let client = node.api_client(&state.database).await?;
1479
1480 match client
1481 .get_backups_backup_query(
1482 self.uuid,
1483 &wings_api::backups_backup_query::get::Query {
1484 adapter: Some(self.disk.to_wings_adapter()),
1485 __priv: (),
1486 },
1487 )
1488 .await
1489 {
1490 Ok(response) => Ok(response),
1491 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, err)) => {
1492 Err(crate::response::DisplayError::new(
1493 crate::ApiError::new_wings_value(err).to_string(),
1494 )
1495 .with_status(StatusCode::NOT_FOUND)
1496 .into())
1497 }
1498 Err(err) => Err(err.into()),
1499 }
1500 }
1501
1502 pub async fn evict_one_by_server_uuid(
1503 state: &crate::State,
1504 server: &super::server::Server,
1505 ) -> Result<(), anyhow::Error> {
1506 let row = sqlx::query(
1507 r#"
1508 SELECT candidates.uuid, candidates.tier, candidates.group_name
1509 FROM (
1510 SELECT
1511 server_backups.uuid,
1512 server_backups.created,
1513 (CASE
1514 WHEN NOT server_backups.successful THEN 0
1515 WHEN g.retention_count IS NOT NULL AND (
1516 SELECT COUNT(*)
1517 FROM server_backups b2
1518 WHERE b2.backup_group_uuid = server_backups.backup_group_uuid
1519 AND b2.deleted IS NULL
1520 AND b2.deleting IS NULL
1521 AND b2.successful
1522 AND b2.completed IS NOT NULL
1523 AND b2.created >= server_backups.created
1524 ) > g.retention_count THEN 1
1525 WHEN server_backups.backup_group_uuid IS NULL THEN 2
1526 ELSE 3
1527 END) AS tier,
1528 g.name AS group_name
1529 FROM server_backups
1530 LEFT JOIN server_backup_groups g ON g.uuid = server_backups.backup_group_uuid
1531 WHERE server_backups.server_uuid = $1
1532 AND server_backups.system_backup_policy_uuid IS NULL
1533 AND server_backups.locked = false
1534 AND server_backups.completed IS NOT NULL
1535 AND server_backups.deleted IS NULL
1536 AND server_backups.deleting IS NULL
1537 ) candidates
1538 ORDER BY candidates.tier ASC, candidates.created ASC
1539 LIMIT 1
1540 "#,
1541 )
1542 .bind(server.uuid)
1543 .fetch_optional(state.database.read())
1544 .await?;
1545
1546 let Some(row) = row else {
1547 return Err(sqlx::Error::RowNotFound.into());
1548 };
1549
1550 let row_uuid: uuid::Uuid = row.try_get("uuid")?;
1551 let row_tier: i32 = row.try_get("tier")?;
1552 let row_group_name: Option<String> = row.try_get("group_name")?;
1553
1554 let rule = match row_tier {
1555 0 => "failed",
1556 1 => "over-retention",
1557 2 => "ungrouped",
1558 _ => "in-retention",
1559 };
1560
1561 if row_tier == 3 {
1562 tracing::warn!(
1563 server = %server.uuid,
1564 backup = %row_uuid,
1565 group = ?row_group_name,
1566 "evicting an in-retention grouped backup to satisfy backup_limit; retention quota exceeds backup_limit"
1567 );
1568 }
1569
1570 let Some(backup) =
1571 Self::by_server_uuid_uuid(&state.database, server.uuid, row_uuid).await?
1572 else {
1573 return Err(sqlx::Error::RowNotFound.into());
1574 };
1575
1576 backup.delete(state, Default::default()).await?;
1577
1578 Self::log_eviction_activity(
1579 state,
1580 server.uuid,
1581 backup.uuid,
1582 &backup.name,
1583 rule,
1584 match row_group_name.as_deref() {
1585 Some(group_name) => EvictionScope::Group(group_name),
1586 None => EvictionScope::Server,
1587 },
1588 )
1589 .await;
1590
1591 Ok(())
1592 }
1593
1594 pub async fn rotate_group_for_create(
1595 state: &crate::State,
1596 group: &super::server_backup_group::ServerBackupGroup,
1597 ) -> Result<GroupRotationOutcome, anyhow::Error> {
1598 let Some(retention_count) = group.retention_count else {
1599 return Ok(GroupRotationOutcome::NotConfigured);
1600 };
1601
1602 let row = sqlx::query(
1603 r#"
1604 SELECT
1605 (SELECT COUNT(*)
1606 FROM server_backups
1607 WHERE server_backups.backup_group_uuid = $1
1608 AND server_backups.deleted IS NULL
1609 AND server_backups.deleting IS NULL
1610 AND server_backups.successful
1611 AND server_backups.completed IS NOT NULL) AS usable,
1612 (SELECT server_backups.uuid
1613 FROM server_backups
1614 WHERE server_backups.backup_group_uuid = $1
1615 AND server_backups.deleted IS NULL
1616 AND server_backups.deleting IS NULL
1617 AND server_backups.successful
1618 AND server_backups.completed IS NOT NULL
1619 AND server_backups.locked = false
1620 ORDER BY server_backups.created ASC
1621 LIMIT 1) AS oldest_unlocked
1622 "#,
1623 )
1624 .bind(group.uuid)
1625 .fetch_one(state.database.read())
1626 .await?;
1627
1628 let usable: i64 = row.try_get("usable")?;
1629 let oldest_unlocked: Option<uuid::Uuid> = row.try_get("oldest_unlocked")?;
1630
1631 if usable < retention_count as i64 {
1632 return Ok(GroupRotationOutcome::WithinRetention);
1633 }
1634
1635 let Some(oldest_unlocked) = oldest_unlocked else {
1636 return Ok(GroupRotationOutcome::BlockedAllLocked);
1637 };
1638
1639 let Some(backup) =
1640 Self::by_server_uuid_uuid(&state.database, group.server_uuid, oldest_unlocked).await?
1641 else {
1642 return Ok(GroupRotationOutcome::WithinRetention);
1643 };
1644
1645 backup.delete(state, Default::default()).await?;
1646
1647 Self::log_eviction_activity(
1648 state,
1649 group.server_uuid,
1650 backup.uuid,
1651 &backup.name,
1652 "group-rotation",
1653 EvictionScope::Group(group.name.as_str()),
1654 )
1655 .await;
1656
1657 Ok(GroupRotationOutcome::Evicted)
1658 }
1659
1660 pub async fn prune_expired_group_backups(state: &crate::State) -> Result<u64, anyhow::Error> {
1661 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1662 r#"
1663 SELECT {}, g.name AS group_name
1664 FROM server_backups
1665 JOIN server_backup_groups g ON g.uuid = server_backups.backup_group_uuid
1666 WHERE g.retention_days IS NOT NULL
1667 AND server_backups.deleted IS NULL
1668 AND server_backups.deleting IS NULL
1669 AND server_backups.locked = false
1670 AND server_backups.completed IS NOT NULL
1671 AND server_backups.created < NOW() - make_interval(days => g.retention_days)
1672 "#,
1673 Self::columns_sql(None)
1674 )))
1675 .fetch_all(state.database.read())
1676 .await?;
1677
1678 let mut pruned = 0;
1679 for row in rows {
1680 let group_name: compact_str::CompactString = row.try_get("group_name")?;
1681 let server_uuid: Option<uuid::Uuid> = row.try_get("server_uuid")?;
1682 let backup = Self::map(None, &row)?;
1683
1684 if let Err(err) = backup.delete(state, Default::default()).await {
1685 tracing::error!(
1686 backup = %backup.uuid,
1687 "failed to prune expired group backup: {:#?}",
1688 err
1689 );
1690 continue;
1691 }
1692
1693 if let Some(server_uuid) = server_uuid {
1694 Self::log_eviction_activity(
1695 state,
1696 server_uuid,
1697 backup.uuid,
1698 &backup.name,
1699 "retention-days",
1700 EvictionScope::Group(group_name.as_str()),
1701 )
1702 .await;
1703 }
1704
1705 pruned += 1;
1706 }
1707
1708 Ok(pruned)
1709 }
1710
1711 pub async fn rotate_system_for_create(
1712 state: &crate::State,
1713 system_backup_policy: &super::system_backup_policy::SystemBackupPolicy,
1714 server_uuid: uuid::Uuid,
1715 ) -> Result<(), anyhow::Error> {
1716 let failed_rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1717 r#"
1718 SELECT {}
1719 FROM server_backups
1720 WHERE server_backups.server_uuid = $1
1721 AND server_backups.system_backup_policy_uuid = $2
1722 AND server_backups.deleted IS NULL
1723 AND server_backups.deleting IS NULL
1724 AND server_backups.locked = false
1725 AND NOT server_backups.successful
1726 AND server_backups.completed IS NOT NULL
1727 "#,
1728 Self::columns_sql(None)
1729 )))
1730 .bind(server_uuid)
1731 .bind(system_backup_policy.uuid)
1732 .fetch_all(state.database.read())
1733 .await?;
1734
1735 for row in failed_rows {
1736 let backup = Self::map(None, &row)?;
1737
1738 if let Err(err) = backup.delete(state, Default::default()).await {
1739 tracing::error!(
1740 backup = %backup.uuid,
1741 "failed to delete failed system backup: {:#?}",
1742 err
1743 );
1744 continue;
1745 }
1746
1747 Self::log_eviction_activity(
1748 state,
1749 server_uuid,
1750 backup.uuid,
1751 &backup.name,
1752 "system-failed",
1753 EvictionScope::Policy(system_backup_policy.name.as_str()),
1754 )
1755 .await;
1756 }
1757
1758 let Some(retention_count) = system_backup_policy.retention_count else {
1759 return Ok(());
1760 };
1761
1762 let row = sqlx::query(
1763 r#"
1764 SELECT
1765 (SELECT COUNT(*)
1766 FROM server_backups
1767 WHERE server_backups.server_uuid = $1
1768 AND server_backups.system_backup_policy_uuid = $2
1769 AND server_backups.deleted IS NULL
1770 AND server_backups.deleting IS NULL
1771 AND server_backups.successful
1772 AND server_backups.completed IS NOT NULL) AS usable,
1773 (SELECT server_backups.uuid
1774 FROM server_backups
1775 WHERE server_backups.server_uuid = $1
1776 AND server_backups.system_backup_policy_uuid = $2
1777 AND server_backups.deleted IS NULL
1778 AND server_backups.deleting IS NULL
1779 AND server_backups.successful
1780 AND server_backups.completed IS NOT NULL
1781 AND server_backups.locked = false
1782 ORDER BY server_backups.created ASC
1783 LIMIT 1) AS oldest_unlocked
1784 "#,
1785 )
1786 .bind(server_uuid)
1787 .bind(system_backup_policy.uuid)
1788 .fetch_one(state.database.read())
1789 .await?;
1790
1791 let usable: i64 = row.try_get("usable")?;
1792 let oldest_unlocked: Option<uuid::Uuid> = row.try_get("oldest_unlocked")?;
1793
1794 if usable < retention_count as i64 {
1795 return Ok(());
1796 }
1797
1798 let Some(oldest_unlocked) = oldest_unlocked else {
1799 return Ok(());
1800 };
1801
1802 let Some(backup) =
1803 Self::by_server_uuid_uuid(&state.database, server_uuid, oldest_unlocked).await?
1804 else {
1805 return Ok(());
1806 };
1807
1808 backup.delete(state, Default::default()).await?;
1809
1810 Self::log_eviction_activity(
1811 state,
1812 server_uuid,
1813 backup.uuid,
1814 &backup.name,
1815 "system-rotation",
1816 EvictionScope::Policy(system_backup_policy.name.as_str()),
1817 )
1818 .await;
1819
1820 Ok(())
1821 }
1822
1823 pub async fn prune_system_backups(state: &crate::State) -> Result<u64, anyhow::Error> {
1824 let expired_rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1825 r#"
1826 SELECT {}, p.name AS policy_name, 'system-retention-days' AS rule
1827 FROM server_backups
1828 JOIN system_backup_policies p ON p.uuid = server_backups.system_backup_policy_uuid
1829 WHERE p.retention_days IS NOT NULL
1830 AND server_backups.deleted IS NULL
1831 AND server_backups.deleting IS NULL
1832 AND server_backups.locked = false
1833 AND server_backups.completed IS NOT NULL
1834 AND server_backups.created < NOW() - make_interval(days => p.retention_days)
1835 "#,
1836 Self::columns_sql(None)
1837 )))
1838 .fetch_all(state.database.read())
1839 .await?;
1840
1841 let over_retention_rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1842 r#"
1843 SELECT {}, p.name AS policy_name, 'system-over-retention' AS rule
1844 FROM (
1845 SELECT b.*, ROW_NUMBER() OVER (
1846 PARTITION BY b.server_uuid, b.system_backup_policy_uuid
1847 ORDER BY b.created DESC
1848 ) AS usable_position
1849 FROM server_backups b
1850 WHERE b.system_backup_policy_uuid IS NOT NULL
1851 AND b.server_uuid IS NOT NULL
1852 AND b.deleted IS NULL
1853 AND b.deleting IS NULL
1854 AND b.locked = false
1855 AND b.successful
1856 AND b.completed IS NOT NULL
1857 ) server_backups
1858 JOIN system_backup_policies p ON p.uuid = server_backups.system_backup_policy_uuid
1859 WHERE p.retention_count IS NOT NULL
1860 AND server_backups.usable_position > p.retention_count
1861 "#,
1862 Self::columns_sql(None)
1863 )))
1864 .fetch_all(state.database.read())
1865 .await?;
1866
1867 let failed_rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1868 r#"
1869 SELECT {}, p.name AS policy_name, 'system-failed' AS rule
1870 FROM server_backups
1871 JOIN system_backup_policies p ON p.uuid = server_backups.system_backup_policy_uuid
1872 WHERE server_backups.deleted IS NULL
1873 AND server_backups.deleting IS NULL
1874 AND server_backups.locked = false
1875 AND NOT server_backups.successful
1876 AND server_backups.completed IS NOT NULL
1877 AND server_backups.created < NOW() - INTERVAL '1 day'
1878 "#,
1879 Self::columns_sql(None)
1880 )))
1881 .fetch_all(state.database.read())
1882 .await?;
1883
1884 let mut seen = std::collections::HashSet::new();
1885 let mut pruned = 0;
1886 for row in expired_rows
1887 .into_iter()
1888 .chain(over_retention_rows)
1889 .chain(failed_rows)
1890 {
1891 let policy_name: compact_str::CompactString = row.try_get("policy_name")?;
1892 let rule: String = row.try_get("rule")?;
1893 let server_uuid: Option<uuid::Uuid> = row.try_get("server_uuid")?;
1894 let backup = Self::map(None, &row)?;
1895
1896 if !seen.insert(backup.uuid) {
1897 continue;
1898 }
1899
1900 if let Err(err) = backup.delete(state, Default::default()).await {
1901 tracing::error!(
1902 backup = %backup.uuid,
1903 "failed to prune system backup: {:#?}",
1904 err
1905 );
1906 continue;
1907 }
1908
1909 if let Some(server_uuid) = server_uuid {
1910 Self::log_eviction_activity(
1911 state,
1912 server_uuid,
1913 backup.uuid,
1914 &backup.name,
1915 &rule,
1916 EvictionScope::Policy(policy_name.as_str()),
1917 )
1918 .await;
1919 }
1920
1921 pruned += 1;
1922 }
1923
1924 Ok(pruned)
1925 }
1926
1927 async fn log_eviction_activity(
1928 state: &crate::State,
1929 server_uuid: uuid::Uuid,
1930 backup_uuid: uuid::Uuid,
1931 backup_name: &str,
1932 rule: &str,
1933 scope: EvictionScope<'_>,
1934 ) {
1935 if let Err(err) = super::server_activity::ServerActivity::create(
1936 state,
1937 super::server_activity::CreateServerActivityOptions {
1938 server_uuid,
1939 user_uuid: None,
1940 impersonator_uuid: None,
1941 api_key_uuid: None,
1942 schedule_uuid: None,
1943 event: "server:backup.delete".into(),
1944 ip: None,
1945 data: serde_json::json!({
1946 "source": "eviction",
1947 "uuid": backup_uuid,
1948 "name": backup_name,
1949 "rule": rule,
1950 "group": scope.group_name(),
1951 "policy": scope.policy_name(),
1952 }),
1953 created: None,
1954 },
1955 )
1956 .await
1957 {
1958 tracing::warn!(
1959 server = %server_uuid,
1960 "failed to log backup eviction activity: {:#?}",
1961 err
1962 );
1963 }
1964 }
1965
1966 #[inline]
1967 pub fn default_name() -> compact_str::CompactString {
1968 let now = chrono::Local::now();
1969
1970 now.format("%Y-%m-%d %H:%M:%S %z").to_compact_string()
1971 }
1972
1973 #[inline]
1974 pub fn s3_path(
1975 server_uuid: uuid::Uuid,
1976 backup_uuid: uuid::Uuid,
1977 compression_type: wings_api::CompressionType,
1978 ) -> compact_str::CompactString {
1979 compact_str::format_compact!(
1980 "{server_uuid}/{backup_uuid}.tar{}",
1981 match compression_type {
1982 wings_api::CompressionType::None => "",
1983 wings_api::CompressionType::Gz => ".gz",
1984 wings_api::CompressionType::Xz => ".xz",
1985 wings_api::CompressionType::Lzip => ".lz",
1986 wings_api::CompressionType::Bz2 => ".bz2",
1987 wings_api::CompressionType::Lz4 => ".lz4",
1988 wings_api::CompressionType::Zstd => ".zst",
1989 }
1990 )
1991 }
1992
1993 #[inline]
1994 pub fn s3_content_type(name: &str) -> &'static str {
1995 if name.ends_with("tar") {
1996 "application/x-tar"
1997 } else if name.ends_with(".tar.gz") {
1998 "application/x-gzip"
1999 } else if name.ends_with(".tar.xz") {
2000 "application/x-xz"
2001 } else if name.ends_with(".tar.lz") {
2002 "application/x-lzip"
2003 } else if name.ends_with(".tar.bz2") {
2004 "application/x-bzip2"
2005 } else if name.ends_with(".tar.lz4") {
2006 "application/x-lz4"
2007 } else if name.ends_with(".tar.zst") {
2008 "application/zstd"
2009 } else {
2010 "application/octet-stream"
2011 }
2012 }
2013
2014 pub async fn into_admin_node_api_object(
2015 self,
2016 state: &crate::State,
2017 storage_url_retriever: &StorageUrlRetriever<'_>,
2018 ) -> Result<AdminApiNodeServerBackup, crate::database::DatabaseError> {
2019 let deletion_status = self.deletion_status();
2020
2021 Ok(AdminApiNodeServerBackup {
2022 uuid: self.uuid,
2023 server: match self.server {
2024 Some(server) => Some(
2025 server
2026 .fetch_cached(&state.database)
2027 .await?
2028 .into_admin_api_object(state, storage_url_retriever)
2029 .await?,
2030 ),
2031 None => None,
2032 },
2033 node: self
2034 .node
2035 .fetch_cached(&state.database)
2036 .await?
2037 .into_admin_api_object(state, ())
2038 .await?,
2039 backup_group_uuid: self.backup_group_uuid,
2040 system_backup_policy_uuid: self.system_backup_policy_uuid,
2041 name: self.name,
2042 ignored_files: self.ignored_files,
2043 is_successful: self.successful,
2044 is_locked: self.locked,
2045 is_browsable: self.browsable,
2046 is_streaming: self.streaming,
2047 is_shared: self.shared,
2048 checksum: self.checksum,
2049 bytes: self.bytes,
2050 files: self.files,
2051 deletion_status,
2052 metadata: self.metadata,
2053 completed: self.completed.map(|dt| dt.and_utc()),
2054 created: self.created.and_utc(),
2055 })
2056 }
2057}
2058
2059#[async_trait::async_trait]
2060impl IntoAdminApiObject for ServerBackup {
2061 type AdminApiObject = AdminApiServerBackup;
2062 type ExtraArgs<'a> = &'a crate::storage::StorageUrlRetriever<'a>;
2063
2064 async fn into_admin_api_object<'a>(
2065 self,
2066 state: &crate::State,
2067 storage_url_retriever: Self::ExtraArgs<'a>,
2068 ) -> Result<Self::AdminApiObject, crate::database::DatabaseError> {
2069 let deletion_status = self.deletion_status();
2070 let api_object = AdminApiServerBackup::init_hooks(&self, state).await?;
2071
2072 let api_object = finish_extendible!(
2073 AdminApiServerBackup {
2074 uuid: self.uuid,
2075 server: match self.server {
2076 Some(server) => Some(
2077 server
2078 .fetch_cached(&state.database)
2079 .await?
2080 .into_admin_api_object(state, storage_url_retriever)
2081 .await?,
2082 ),
2083 None => None,
2084 },
2085 backup_group_uuid: self.backup_group_uuid,
2086 system_backup_policy_uuid: self.system_backup_policy_uuid,
2087 name: self.name,
2088 ignored_files: self.ignored_files,
2089 is_successful: self.successful,
2090 is_locked: self.locked,
2091 is_browsable: self.browsable,
2092 is_streaming: self.streaming,
2093 is_shared: self.shared,
2094 checksum: self.checksum,
2095 bytes: self.bytes,
2096 files: self.files,
2097 deletion_status,
2098 metadata: self.metadata,
2099 completed: self.completed.map(|dt| dt.and_utc()),
2100 created: self.created.and_utc(),
2101 },
2102 api_object,
2103 state
2104 )?;
2105
2106 Ok(api_object)
2107 }
2108}
2109
2110#[async_trait::async_trait]
2111impl IntoApiObject for ServerBackup {
2112 type ApiObject = ApiServerBackup;
2113 type ExtraArgs<'a> = ();
2114
2115 async fn into_api_object<'a>(
2116 self,
2117 state: &crate::State,
2118 _args: Self::ExtraArgs<'a>,
2119 ) -> Result<Self::ApiObject, crate::database::DatabaseError> {
2120 let deletion_status = self.deletion_status();
2121 let api_object = ApiServerBackup::init_hooks(&self, state).await?;
2122
2123 let api_object = finish_extendible!(
2124 ApiServerBackup {
2125 uuid: self.uuid,
2126 backup_group_uuid: self.backup_group_uuid,
2127 name: self.name,
2128 ignored_files: self.ignored_files,
2129 is_successful: self.successful,
2130 is_locked: self.locked,
2131 is_browsable: self.browsable,
2132 is_streaming: self.streaming,
2133 checksum: self.checksum,
2134 bytes: self.bytes,
2135 files: self.files,
2136 deletion_status,
2137 metadata: self.metadata,
2138 completed: self.completed.map(|dt| dt.and_utc()),
2139 created: self.created.and_utc(),
2140 },
2141 api_object,
2142 state
2143 )?;
2144
2145 Ok(api_object)
2146 }
2147}
2148
2149#[derive(Validate)]
2150pub struct CreateServerBackupOptions<'a> {
2151 #[garde(skip)]
2152 pub server: &'a super::server::Server,
2153 #[garde(length(chars, min = 1, max = 255))]
2154 pub name: compact_str::CompactString,
2155 #[garde(skip)]
2156 pub backup_group_uuid: Option<uuid::Uuid>,
2157 #[garde(skip)]
2158 pub system_backup_policy_uuid: Option<uuid::Uuid>,
2159 #[garde(skip)]
2160 pub backup_configuration: Option<super::backup_configuration::BackupConfiguration>,
2161 #[garde(skip)]
2162 pub ignored_files: Vec<compact_str::CompactString>,
2163 #[garde(skip)]
2164 pub metadata: serde_json::Value,
2165}
2166
2167#[async_trait::async_trait]
2168impl CreatableModel for ServerBackup {
2169 type CreateOptions<'a> = CreateServerBackupOptions<'a>;
2170 type CreateResult = Self;
2171
2172 fn get_create_handlers() -> &'static LazyLock<CreateListenerList<Self>> {
2173 static CREATE_LISTENERS: LazyLock<CreateListenerList<ServerBackup>> =
2174 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2175
2176 &CREATE_LISTENERS
2177 }
2178
2179 async fn create_with_transaction(
2180 _state: &crate::State,
2181 _options: Self::CreateOptions<'_>,
2182 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2183 ) -> Result<Self, crate::database::DatabaseError> {
2184 Err(anyhow::anyhow!("create_with_transaction is not supported for ServerBackup").into())
2185 }
2186
2187 async fn create(
2188 state: &crate::State,
2189 mut options: Self::CreateOptions<'_>,
2190 ) -> Result<Self, crate::database::DatabaseError> {
2191 options.validate()?;
2192
2193 let backup_configuration = match options.backup_configuration.take() {
2194 Some(backup_configuration) => backup_configuration,
2195 None => options
2196 .server
2197 .backup_configuration(&state.database)
2198 .await
2199 .ok_or_else(|| {
2200 anyhow::Error::new(
2201 crate::response::DisplayError::new(
2202 "no backup configuration available, unable to create backup",
2203 )
2204 .with_status(StatusCode::EXPECTATION_FAILED),
2205 )
2206 })?,
2207 };
2208
2209 if backup_configuration.maintenance_enabled {
2210 return Err(anyhow::Error::new(
2211 crate::response::DisplayError::new(
2212 "cannot create backup while backup configuration is in maintenance mode",
2213 )
2214 .with_status(StatusCode::EXPECTATION_FAILED),
2215 )
2216 .into());
2217 }
2218
2219 let mut transaction = state.database.write().begin().await?;
2220
2221 let mut query_builder = InsertQueryBuilder::new("server_backups");
2222
2223 Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
2224 .await?;
2225
2226 query_builder
2227 .set("server_uuid", options.server.uuid)
2228 .set("node_uuid", options.server.node.uuid)
2229 .set("backup_configuration_uuid", backup_configuration.uuid)
2230 .set("backup_group_uuid", options.backup_group_uuid)
2231 .set(
2232 "system_backup_policy_uuid",
2233 options.system_backup_policy_uuid,
2234 )
2235 .set("name", &options.name)
2236 .set("ignored_files", &options.ignored_files)
2237 .set("bytes", 0i64)
2238 .set("disk", backup_configuration.backup_disk)
2239 .set("shared", backup_configuration.shared)
2240 .set("metadata", &options.metadata);
2241
2242 let row = query_builder
2243 .returning(&Self::columns_sql(None))
2244 .fetch_one(&mut *transaction)
2245 .await?;
2246 let mut backup = Self::map(None, &row)?;
2247
2248 Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
2249
2250 transaction.commit().await?;
2251
2252 let server = options.server.clone();
2253 let database = Arc::clone(&state.database);
2254 let backup_uuid = backup.uuid;
2255 let backup_disk = backup_configuration.backup_disk;
2256 let ignored_files_str = options
2257 .ignored_files
2258 .iter()
2259 .map(|s| s.as_str())
2260 .collect::<Vec<_>>()
2261 .join("\n");
2262
2263 tokio::spawn(async move {
2264 tracing::debug!(backup = %backup_uuid, "creating server backup");
2265
2266 let node = match server.node.fetch_cached(&database).await {
2267 Ok(node) => node,
2268 Err(err) => {
2269 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2270
2271 if let Err(err) = sqlx::query!(
2272 "UPDATE server_backups
2273 SET successful = false, completed = NOW()
2274 WHERE server_backups.uuid = $1",
2275 backup_uuid
2276 )
2277 .execute(database.write())
2278 .await
2279 {
2280 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2281 }
2282
2283 return;
2284 }
2285 };
2286
2287 let api_client = match node.api_client(&database).await {
2288 Ok(api_client) => api_client,
2289 Err(err) => {
2290 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2291
2292 if let Err(err) = sqlx::query!(
2293 "UPDATE server_backups
2294 SET successful = false, completed = NOW()
2295 WHERE server_backups.uuid = $1",
2296 backup_uuid
2297 )
2298 .execute(database.write())
2299 .await
2300 {
2301 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2302 }
2303
2304 return;
2305 }
2306 };
2307
2308 if let Err(err) = api_client
2309 .post_servers_server_backup(
2310 server.uuid,
2311 &wings_api::servers_server_backup::post::RequestBody {
2312 adapter: backup_disk.to_wings_adapter(),
2313 uuid: backup_uuid,
2314 ignore: ignored_files_str.into(),
2315 },
2316 )
2317 .await
2318 {
2319 tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
2320
2321 if let Err(err) = sqlx::query!(
2322 "UPDATE server_backups
2323 SET successful = false, completed = NOW()
2324 WHERE server_backups.uuid = $1",
2325 backup_uuid
2326 )
2327 .execute(database.write())
2328 .await
2329 {
2330 tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
2331 }
2332 }
2333 });
2334
2335 Ok(backup)
2336 }
2337}
2338
2339#[derive(ToSchema, Serialize, Deserialize, Validate, Default)]
2340pub struct UpdateServerBackupOptions {
2341 #[garde(length(chars, min = 1, max = 255))]
2342 #[schema(min_length = 1, max_length = 255)]
2343 pub name: Option<compact_str::CompactString>,
2344 #[garde(skip)]
2345 #[serde(default, with = "::serde_with::rust::double_option")]
2346 pub backup_group_uuid: Option<Option<uuid::Uuid>>,
2347 #[garde(skip)]
2348 pub locked: Option<bool>,
2349}
2350
2351#[async_trait::async_trait]
2352impl UpdatableModel for ServerBackup {
2353 type UpdateOptions = UpdateServerBackupOptions;
2354
2355 fn get_update_handlers() -> &'static LazyLock<UpdateHandlerList<Self>> {
2356 static UPDATE_LISTENERS: LazyLock<UpdateHandlerList<ServerBackup>> =
2357 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2358
2359 &UPDATE_LISTENERS
2360 }
2361
2362 async fn update_with_transaction(
2363 &mut self,
2364 state: &crate::State,
2365 mut options: Self::UpdateOptions,
2366 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2367 ) -> Result<(), crate::database::DatabaseError> {
2368 options.validate()?;
2369
2370 let mut query_builder = UpdateQueryBuilder::new("server_backups");
2371
2372 self.run_update_handlers(&mut options, &mut query_builder, state, transaction)
2373 .await?;
2374
2375 query_builder
2376 .set("name", options.name.as_ref())
2377 .set("backup_group_uuid", options.backup_group_uuid)
2378 .set("locked", options.locked)
2379 .where_eq("uuid", self.uuid);
2380
2381 query_builder.execute(&mut **transaction).await?;
2382
2383 if let Some(name) = options.name {
2384 self.name = name;
2385 }
2386 if let Some(backup_group_uuid) = options.backup_group_uuid {
2387 self.backup_group_uuid = backup_group_uuid;
2388 }
2389 if let Some(locked) = options.locked {
2390 self.locked = locked;
2391 }
2392
2393 self.run_after_update_handlers(state, transaction).await?;
2394
2395 Ok(())
2396 }
2397}
2398
2399#[async_trait::async_trait]
2400impl ByUuid for ServerBackup {
2401 async fn by_uuid(
2402 database: &crate::database::Database,
2403 uuid: uuid::Uuid,
2404 ) -> Result<Self, crate::database::DatabaseError> {
2405 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2406 r#"
2407 SELECT {}
2408 FROM server_backups
2409 WHERE server_backups.uuid = $1
2410 "#,
2411 Self::columns_sql(None)
2412 )))
2413 .bind(uuid)
2414 .fetch_one(database.read())
2415 .await?;
2416
2417 Self::map(None, &row)
2418 }
2419
2420 async fn by_uuid_with_transaction(
2421 transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2422 uuid: uuid::Uuid,
2423 ) -> Result<Self, crate::database::DatabaseError> {
2424 let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2425 r#"
2426 SELECT {}
2427 FROM server_backups
2428 WHERE server_backups.uuid = $1
2429 "#,
2430 Self::columns_sql(None)
2431 )))
2432 .bind(uuid)
2433 .fetch_one(&mut **transaction)
2434 .await?;
2435
2436 Self::map(None, &row)
2437 }
2438}
2439
2440#[derive(Clone, Default)]
2441pub struct DeleteServerBackupOptions {
2442 pub force: bool,
2443}
2444
2445impl ServerBackup {
2446 pub const MAX_DELETION_RETRIES: i32 = 8;
2447
2448 #[inline]
2449 pub fn deletion_status(&self) -> Option<ServerBackupDeletionStatus> {
2450 if self.deleted.is_some() || self.deleting.is_none() {
2451 return None;
2452 }
2453
2454 if self.deletion_retries >= Self::MAX_DELETION_RETRIES {
2455 Some(ServerBackupDeletionStatus::Failed)
2456 } else {
2457 Some(ServerBackupDeletionStatus::Deleting)
2458 }
2459 }
2460
2461 pub async fn dispatch_deletion(
2462 &self,
2463 state: &crate::State,
2464 options: &DeleteServerBackupOptions,
2465 ) -> Result<bool, anyhow::Error> {
2466 let node = self.node.fetch_cached(&state.database).await?;
2467
2468 let backup_configuration = match &self.backup_configuration {
2469 Some(backup_configuration) => {
2470 Some(backup_configuration.fetch_cached(&state.database).await?)
2471 }
2472 None if options.force => None,
2473 None => {
2474 return Err(crate::response::DisplayError::new(
2475 "no backup configuration available, unable to delete backup",
2476 )
2477 .with_status(StatusCode::EXPECTATION_FAILED)
2478 .into());
2479 }
2480 };
2481
2482 if let Some(backup_configuration) = &backup_configuration
2483 && backup_configuration.maintenance_enabled
2484 {
2485 return Err(crate::response::DisplayError::new(
2486 "cannot delete backup while backup configuration is in maintenance mode",
2487 )
2488 .with_status(StatusCode::EXPECTATION_FAILED)
2489 .into());
2490 }
2491
2492 if self.disk == BackupDisk::S3 {
2493 let Some(mut s3_configuration) =
2494 backup_configuration.and_then(|c| c.backup_configs.s3.clone())
2495 else {
2496 if options.force {
2497 tracing::warn!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "S3 backup deletion attempted but no S3 configuration found, ignoring");
2498
2499 return Ok(true);
2500 }
2501
2502 return Err(anyhow::anyhow!(
2503 "s3 backup deletion attempted but no S3 configuration found"
2504 ));
2505 };
2506
2507 s3_configuration.decrypt(&state.database).await?;
2508
2509 let compression_type = s3_configuration.compression_type;
2510 let (client, bucket) = s3_configuration.into_client();
2511
2512 let file_path = match &self.upload_path {
2513 Some(path) => path,
2514 None => {
2515 if let Some(server) = &self.server {
2516 &Self::s3_path(server.uuid, self.uuid, compression_type)
2517 } else {
2518 return Err(anyhow::anyhow!("backup upload path not found"));
2519 }
2520 }
2521 };
2522
2523 if let Err(err) = client
2524 .delete_object()
2525 .bucket(bucket)
2526 .key(&**file_path)
2527 .send()
2528 .await
2529 {
2530 if options.force {
2531 tracing::error!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "failed to delete S3 backup, ignoring: {:?}", err);
2532 } else {
2533 return Err(err.into());
2534 }
2535 }
2536
2537 return Ok(true);
2538 }
2539
2540 match node
2541 .api_client(&state.database)
2542 .await?
2543 .delete_backups_backup(
2544 self.uuid,
2545 &wings_api::backups_backup::delete::RequestBody {
2546 adapter: self.disk.to_wings_adapter(),
2547 foreground: false,
2548 server: self.server.as_ref().map(|s| s.uuid),
2549 },
2550 )
2551 .await
2552 {
2553 Ok(_) => Ok(false),
2554 Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, _)) => Ok(true),
2555 Err(err) if options.force => {
2556 tracing::error!(node = %node.uuid, backup = %self.uuid, "unable to delete backup on node, finalizing anyway: {:?}", err);
2557
2558 Ok(true)
2559 }
2560 Err(err) => Err(err.into()),
2561 }
2562 }
2563
2564 pub async fn finish_deletion(
2565 &self,
2566 state: &crate::State,
2567 options: &DeleteServerBackupOptions,
2568 ) -> Result<(), anyhow::Error> {
2569 let mut transaction = state.database.write().begin().await?;
2570
2571 let finalized = sqlx::query(
2572 r#"
2573 UPDATE server_backups
2574 SET deleted = NOW(), deleting = NULL
2575 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2576 "#,
2577 )
2578 .bind(self.uuid)
2579 .execute(&mut *transaction)
2580 .await?
2581 .rows_affected();
2582
2583 if finalized == 0 {
2584 return Ok(());
2585 }
2586
2587 self.run_after_delete_handlers(options, state, &mut transaction)
2588 .await?;
2589
2590 transaction.commit().await?;
2591
2592 Self::get_event_emitter().emit(
2593 state.clone(),
2594 ServerBackupEvent::DeletionCompleted {
2595 backup: Box::new(self.clone()),
2596 successful: true,
2597 },
2598 );
2599
2600 Ok(())
2601 }
2602
2603 pub async fn fail_deletion_attempt(&self, state: &crate::State) -> Result<i32, anyhow::Error> {
2604 let deletion_retries: Option<i32> = sqlx::query_scalar(
2605 r#"
2606 UPDATE server_backups
2607 SET deletion_retries = deletion_retries + 1
2608 WHERE
2609 server_backups.uuid = $1
2610 AND server_backups.deleted IS NULL
2611 AND server_backups.deleting IS NOT NULL
2612 RETURNING server_backups.deletion_retries
2613 "#,
2614 )
2615 .bind(self.uuid)
2616 .fetch_optional(state.database.write())
2617 .await?;
2618
2619 let Some(deletion_retries) = deletion_retries else {
2620 return Ok(0);
2621 };
2622
2623 if deletion_retries >= Self::MAX_DELETION_RETRIES {
2624 if let Some(server) = &self.server
2625 && let Err(err) = super::server_activity::ServerActivity::create(
2626 state,
2627 super::server_activity::CreateServerActivityOptions {
2628 server_uuid: server.uuid,
2629 user_uuid: None,
2630 impersonator_uuid: None,
2631 api_key_uuid: None,
2632 schedule_uuid: None,
2633 event: "server:backup.delete-failed".into(),
2634 ip: None,
2635 data: serde_json::json!({
2636 "uuid": self.uuid,
2637 "name": self.name,
2638 }),
2639 created: None,
2640 },
2641 )
2642 .await
2643 {
2644 tracing::warn!(
2645 backup = %self.uuid,
2646 "failed to log backup deletion failure activity: {:#?}",
2647 err
2648 );
2649 }
2650
2651 Self::get_event_emitter().emit(
2652 state.clone(),
2653 ServerBackupEvent::DeletionCompleted {
2654 backup: Box::new(self.clone()),
2655 successful: false,
2656 },
2657 );
2658 }
2659
2660 Ok(deletion_retries)
2661 }
2662
2663 pub async fn redispatch_stale_deletions(state: &crate::State) -> Result<u64, anyhow::Error> {
2664 let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2665 r#"
2666 SELECT {}
2667 FROM server_backups
2668 WHERE
2669 server_backups.deleted IS NULL
2670 AND server_backups.deleting IS NOT NULL
2671 AND server_backups.deletion_retries < $1
2672 AND server_backups.deleting < NOW() - make_interval(mins => LEAST(60.0, 5.0 * POWER(2.0, server_backups.deletion_retries))::int)
2673 ORDER BY server_backups.deleting
2674 LIMIT 32
2675 "#,
2676 Self::columns_sql(None)
2677 )))
2678 .bind(Self::MAX_DELETION_RETRIES)
2679 .fetch_all(state.database.read())
2680 .await?;
2681
2682 let mut redispatched = 0;
2683 for row in rows {
2684 let backup = Self::map(None, &row)?;
2685
2686 if let Some(backup_configuration) = &backup.backup_configuration
2687 && let Ok(backup_configuration) =
2688 backup_configuration.fetch_cached(&state.database).await
2689 && backup_configuration.maintenance_enabled
2690 {
2691 sqlx::query(
2692 "UPDATE server_backups
2693 SET deleting = NOW()
2694 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL",
2695 )
2696 .bind(backup.uuid)
2697 .execute(state.database.write())
2698 .await?;
2699
2700 continue;
2701 }
2702
2703 let deletion_retries: i32 = sqlx::query_scalar(
2704 r#"
2705 UPDATE server_backups
2706 SET deleting = NOW(), deletion_retries = deletion_retries + 1
2707 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2708 RETURNING server_backups.deletion_retries
2709 "#,
2710 )
2711 .bind(backup.uuid)
2712 .fetch_one(state.database.write())
2713 .await?;
2714
2715 match backup
2716 .dispatch_deletion(state, &DeleteServerBackupOptions::default())
2717 .await
2718 {
2719 Ok(true) => {
2720 backup
2721 .finish_deletion(state, &DeleteServerBackupOptions::default())
2722 .await?;
2723 }
2724 Ok(false) => {}
2725 Err(err) => {
2726 tracing::error!(
2727 backup = %backup.uuid,
2728 deletion_retries,
2729 "failed to redispatch backup deletion: {:#?}",
2730 err
2731 );
2732
2733 if deletion_retries >= Self::MAX_DELETION_RETRIES {
2734 Self::get_event_emitter().emit(
2735 state.clone(),
2736 ServerBackupEvent::DeletionCompleted {
2737 backup: Box::new(backup.clone()),
2738 successful: false,
2739 },
2740 );
2741 }
2742
2743 continue;
2744 }
2745 }
2746
2747 redispatched += 1;
2748 }
2749
2750 Ok(redispatched)
2751 }
2752}
2753
2754#[async_trait::async_trait]
2755impl DeletableModel for ServerBackup {
2756 type DeleteOptions = DeleteServerBackupOptions;
2757
2758 fn get_delete_handlers() -> &'static LazyLock<DeleteHandlerList<Self>> {
2759 static DELETE_LISTENERS: LazyLock<DeleteHandlerList<ServerBackup>> =
2760 LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2761
2762 &DELETE_LISTENERS
2763 }
2764
2765 async fn delete_with_transaction(
2766 &self,
2767 _state: &crate::State,
2768 _options: Self::DeleteOptions,
2769 _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2770 ) -> Result<(), anyhow::Error> {
2771 Err(anyhow::anyhow!(
2772 "delete_with_transaction is not supported for ServerBackup"
2773 ))
2774 }
2775
2776 async fn delete(
2777 &self,
2778 state: &crate::State,
2779 options: Self::DeleteOptions,
2780 ) -> Result<(), anyhow::Error> {
2781 if let Some(backup_configuration) = &self.backup_configuration
2782 && backup_configuration
2783 .fetch_cached(&state.database)
2784 .await?
2785 .maintenance_enabled
2786 {
2787 return Err(crate::response::DisplayError::new(
2788 "cannot delete backup while backup configuration is in maintenance mode",
2789 )
2790 .with_status(StatusCode::EXPECTATION_FAILED)
2791 .into());
2792 }
2793
2794 let mut transaction = state.database.write().begin().await?;
2795
2796 self.run_delete_handlers(&options, state, &mut transaction)
2797 .await?;
2798
2799 let claimed = sqlx::query(
2800 r#"
2801 UPDATE server_backups
2802 SET deleting = NOW(), deletion_retries = 0
2803 WHERE
2804 server_backups.uuid = $1
2805 AND server_backups.deleted IS NULL
2806 AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $2)
2807 "#,
2808 )
2809 .bind(self.uuid)
2810 .bind(Self::MAX_DELETION_RETRIES)
2811 .execute(&mut *transaction)
2812 .await?
2813 .rows_affected();
2814
2815 if claimed == 0 {
2816 return Err(
2817 crate::response::DisplayError::new("backup is already being deleted")
2818 .with_status(StatusCode::EXPECTATION_FAILED)
2819 .into(),
2820 );
2821 }
2822
2823 transaction.commit().await?;
2824
2825 match self.dispatch_deletion(state, &options).await {
2826 Ok(true) => self.finish_deletion(state, &options).await,
2827 Ok(false) => Ok(()),
2828 Err(err) => {
2829 sqlx::query(
2830 r#"
2831 UPDATE server_backups
2832 SET deleting = NULL, deletion_retries = 0
2833 WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2834 "#,
2835 )
2836 .bind(self.uuid)
2837 .execute(state.database.write())
2838 .await?;
2839
2840 Err(err)
2841 }
2842 }
2843 }
2844}
2845
2846#[derive(ToSchema, Serialize)]
2847#[schema(title = "AdminNodeServerBackup")]
2848pub struct AdminApiNodeServerBackup {
2849 pub uuid: uuid::Uuid,
2850 pub server: Option<super::server::AdminApiServer>,
2851 pub node: super::node::AdminApiNode,
2852 pub backup_group_uuid: Option<uuid::Uuid>,
2853 pub system_backup_policy_uuid: Option<uuid::Uuid>,
2854
2855 pub name: compact_str::CompactString,
2856 pub ignored_files: Vec<compact_str::CompactString>,
2857
2858 pub is_successful: bool,
2859 pub is_locked: bool,
2860 pub is_browsable: bool,
2861 pub is_streaming: bool,
2862 pub is_shared: bool,
2863
2864 pub checksum: Option<compact_str::CompactString>,
2865 pub bytes: i64,
2866 pub files: i64,
2867
2868 pub metadata: serde_json::Value,
2869 pub deletion_status: Option<ServerBackupDeletionStatus>,
2870
2871 pub completed: Option<chrono::DateTime<chrono::Utc>>,
2872 pub created: chrono::DateTime<chrono::Utc>,
2873}
2874
2875#[schema_extension_derive::extendible]
2876#[init_args(ServerBackup, crate::State)]
2877#[hook_args(crate::State)]
2878#[derive(ToSchema, Serialize)]
2879#[schema(title = "AdminServerBackup")]
2880pub struct AdminApiServerBackup {
2881 pub uuid: uuid::Uuid,
2882 pub server: Option<super::server::AdminApiServer>,
2883 pub backup_group_uuid: Option<uuid::Uuid>,
2884 pub system_backup_policy_uuid: Option<uuid::Uuid>,
2885
2886 pub name: compact_str::CompactString,
2887 pub ignored_files: Vec<compact_str::CompactString>,
2888
2889 pub is_successful: bool,
2890 pub is_locked: bool,
2891 pub is_browsable: bool,
2892 pub is_streaming: bool,
2893 pub is_shared: bool,
2894
2895 pub checksum: Option<compact_str::CompactString>,
2896 pub bytes: i64,
2897 pub files: i64,
2898
2899 pub metadata: serde_json::Value,
2900 pub deletion_status: Option<ServerBackupDeletionStatus>,
2901
2902 pub completed: Option<chrono::DateTime<chrono::Utc>>,
2903 pub created: chrono::DateTime<chrono::Utc>,
2904}
2905
2906#[schema_extension_derive::extendible]
2907#[init_args(ServerBackup, crate::State)]
2908#[hook_args(crate::State)]
2909#[derive(ToSchema, Serialize)]
2910#[schema(title = "ServerBackup")]
2911pub struct ApiServerBackup {
2912 pub uuid: uuid::Uuid,
2913 pub backup_group_uuid: Option<uuid::Uuid>,
2914
2915 pub name: compact_str::CompactString,
2916 pub ignored_files: Vec<compact_str::CompactString>,
2917
2918 pub is_successful: bool,
2919 pub is_locked: bool,
2920 pub is_browsable: bool,
2921 pub is_streaming: bool,
2922
2923 pub checksum: Option<compact_str::CompactString>,
2924 pub bytes: i64,
2925 pub files: i64,
2926
2927 pub metadata: serde_json::Value,
2928 pub deletion_status: Option<ServerBackupDeletionStatus>,
2929
2930 pub completed: Option<chrono::DateTime<chrono::Utc>>,
2931 pub created: chrono::DateTime<chrono::Utc>,
2932}