Skip to main content

shared/models/server_backup/
mod.rs

1mod events;
2pub use events::ServerBackupEvent;
3
4use crate::{
5    jwt::BasePayload,
6    models::{InsertQueryBuilder, UpdateQueryBuilder, server_variable::ServerVariable},
7    prelude::*,
8    storage::StorageUrlRetriever,
9};
10use compact_str::ToCompactString;
11use garde::Validate;
12use reqwest::StatusCode;
13use serde::{Deserialize, Serialize};
14use sqlx::{Row, postgres::PgRow, prelude::Type};
15use std::{
16    collections::{BTreeMap, HashMap},
17    sync::{Arc, LazyLock},
18};
19use utoipa::ToSchema;
20
21#[derive(Debug, ToSchema, Serialize, Deserialize, Type, PartialEq, Eq, Hash, Clone, Copy)]
22#[serde(rename_all = "kebab-case")]
23#[sqlx(type_name = "backup_disk", rename_all = "SCREAMING_SNAKE_CASE")]
24pub enum BackupDisk {
25    Local,
26    S3,
27    DdupBak,
28    Btrfs,
29    Zfs,
30    Restic,
31    ProxmoxBackupServer,
32    Kopia,
33}
34
35impl BackupDisk {
36    #[inline]
37    pub fn from_wings_adapter(adapter: wings_api::BackupAdapter) -> Self {
38        match adapter {
39            wings_api::BackupAdapter::Wings => BackupDisk::Local,
40            wings_api::BackupAdapter::S3 => BackupDisk::S3,
41            wings_api::BackupAdapter::DdupBak => BackupDisk::DdupBak,
42            wings_api::BackupAdapter::Btrfs => BackupDisk::Btrfs,
43            wings_api::BackupAdapter::Zfs => BackupDisk::Zfs,
44            wings_api::BackupAdapter::Restic => BackupDisk::Restic,
45            wings_api::BackupAdapter::ProxmoxBackupServer => BackupDisk::ProxmoxBackupServer,
46            wings_api::BackupAdapter::Kopia => BackupDisk::Kopia,
47        }
48    }
49
50    #[inline]
51    pub fn to_wings_adapter(self) -> wings_api::BackupAdapter {
52        match self {
53            BackupDisk::Local => wings_api::BackupAdapter::Wings,
54            BackupDisk::S3 => wings_api::BackupAdapter::S3,
55            BackupDisk::DdupBak => wings_api::BackupAdapter::DdupBak,
56            BackupDisk::Btrfs => wings_api::BackupAdapter::Btrfs,
57            BackupDisk::Zfs => wings_api::BackupAdapter::Zfs,
58            BackupDisk::Restic => wings_api::BackupAdapter::Restic,
59            BackupDisk::ProxmoxBackupServer => wings_api::BackupAdapter::ProxmoxBackupServer,
60            BackupDisk::Kopia => wings_api::BackupAdapter::Kopia,
61        }
62    }
63}
64
65pub struct ServerBackupRestoreOptions {
66    pub truncate_directory: bool,
67    pub restore_startup: bool,
68}
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum GroupRotationOutcome {
72    /// The group has no `retention_count`, so count-based rotation does not apply.
73    NotConfigured,
74    /// The group is still under its `retention_count`; nothing was evicted.
75    WithinRetention,
76    /// The oldest unlocked usable backup in the group was evicted to make room.
77    Evicted,
78    /// The group is at/over `retention_count` but every usable backup is locked.
79    BlockedAllLocked,
80}
81
82#[derive(Serialize, Deserialize, Clone)]
83pub struct ServerBackup {
84    pub uuid: uuid::Uuid,
85    pub server: Option<Fetchable<super::server::Server>>,
86    pub node: Fetchable<super::node::Node>,
87    pub backup_configuration: Option<Fetchable<super::backup_configuration::BackupConfiguration>>,
88    pub backup_group_uuid: Option<uuid::Uuid>,
89
90    pub name: compact_str::CompactString,
91    pub successful: bool,
92    pub browsable: bool,
93    pub streaming: bool,
94    pub locked: bool,
95    pub shared: bool,
96
97    pub ignored_files: Vec<compact_str::CompactString>,
98    pub checksum: Option<compact_str::CompactString>,
99    pub bytes: i64,
100    pub files: i64,
101
102    pub disk: BackupDisk,
103    pub upload_id: Option<compact_str::CompactString>,
104    pub upload_path: Option<compact_str::CompactString>,
105    pub metadata: serde_json::Value,
106
107    pub completed: Option<chrono::NaiveDateTime>,
108    pub deleting: Option<chrono::NaiveDateTime>,
109    pub deletion_retries: i32,
110    pub deleted: Option<chrono::NaiveDateTime>,
111    pub created: chrono::NaiveDateTime,
112
113    extension_data: super::ModelExtensionData,
114}
115
116#[derive(Debug, ToSchema, Serialize, Clone, Copy, PartialEq, Eq)]
117#[serde(rename_all = "snake_case")]
118pub enum ServerBackupDeletionStatus {
119    Deleting,
120    Failed,
121}
122
123impl BaseModel for ServerBackup {
124    const NAME: &'static str = "server_backup";
125
126    fn get_extension_list() -> &'static super::ModelExtensionList {
127        static EXTENSIONS: LazyLock<super::ModelExtensionList> =
128            LazyLock::new(|| parking_lot::RwLock::new(Vec::new()));
129
130        &EXTENSIONS
131    }
132
133    fn get_extension_data(&self) -> &super::ModelExtensionData {
134        &self.extension_data
135    }
136
137    #[inline]
138    fn base_columns(prefix: Option<&str>) -> BTreeMap<&'static str, compact_str::CompactString> {
139        let prefix = prefix.unwrap_or_default();
140
141        BTreeMap::from([
142            (
143                "server_backups.uuid",
144                compact_str::format_compact!("{prefix}uuid"),
145            ),
146            (
147                "server_backups.server_uuid",
148                compact_str::format_compact!("{prefix}server_uuid"),
149            ),
150            (
151                "server_backups.node_uuid",
152                compact_str::format_compact!("{prefix}node_uuid"),
153            ),
154            (
155                "server_backups.backup_configuration_uuid",
156                compact_str::format_compact!("{prefix}backup_configuration_uuid"),
157            ),
158            (
159                "server_backups.backup_group_uuid",
160                compact_str::format_compact!("{prefix}backup_group_uuid"),
161            ),
162            (
163                "server_backups.name",
164                compact_str::format_compact!("{prefix}name"),
165            ),
166            (
167                "server_backups.successful",
168                compact_str::format_compact!("{prefix}successful"),
169            ),
170            (
171                "server_backups.browsable",
172                compact_str::format_compact!("{prefix}browsable"),
173            ),
174            (
175                "server_backups.streaming",
176                compact_str::format_compact!("{prefix}streaming"),
177            ),
178            (
179                "server_backups.locked",
180                compact_str::format_compact!("{prefix}locked"),
181            ),
182            (
183                "server_backups.shared",
184                compact_str::format_compact!("{prefix}shared"),
185            ),
186            (
187                "server_backups.ignored_files",
188                compact_str::format_compact!("{prefix}ignored_files"),
189            ),
190            (
191                "server_backups.checksum",
192                compact_str::format_compact!("{prefix}checksum"),
193            ),
194            (
195                "server_backups.bytes",
196                compact_str::format_compact!("{prefix}bytes"),
197            ),
198            (
199                "server_backups.files",
200                compact_str::format_compact!("{prefix}files"),
201            ),
202            (
203                "server_backups.disk",
204                compact_str::format_compact!("{prefix}disk"),
205            ),
206            (
207                "server_backups.upload_id",
208                compact_str::format_compact!("{prefix}upload_id"),
209            ),
210            (
211                "server_backups.upload_path",
212                compact_str::format_compact!("{prefix}upload_path"),
213            ),
214            (
215                "server_backups.metadata",
216                compact_str::format_compact!("{prefix}metadata"),
217            ),
218            (
219                "server_backups.completed",
220                compact_str::format_compact!("{prefix}completed"),
221            ),
222            (
223                "server_backups.deleting",
224                compact_str::format_compact!("{prefix}deleting"),
225            ),
226            (
227                "server_backups.deletion_retries",
228                compact_str::format_compact!("{prefix}deletion_retries"),
229            ),
230            (
231                "server_backups.deleted",
232                compact_str::format_compact!("{prefix}deleted"),
233            ),
234            (
235                "server_backups.created",
236                compact_str::format_compact!("{prefix}created"),
237            ),
238        ])
239    }
240
241    #[inline]
242    fn map(prefix: Option<&str>, row: &PgRow) -> Result<Self, crate::database::DatabaseError> {
243        let prefix = prefix.unwrap_or_default();
244
245        Ok(Self {
246            uuid: row.try_get(compact_str::format_compact!("{prefix}uuid").as_str())?,
247            server: super::server::Server::get_fetchable_from_row(
248                row,
249                compact_str::format_compact!("{prefix}server_uuid"),
250            ),
251            backup_configuration:
252                super::backup_configuration::BackupConfiguration::get_fetchable_from_row(
253                    row,
254                    compact_str::format_compact!("{prefix}backup_configuration_uuid"),
255                ),
256            node: super::node::Node::get_fetchable(
257                row.try_get(compact_str::format_compact!("{prefix}node_uuid").as_str())?,
258            ),
259            backup_group_uuid: row
260                .try_get(compact_str::format_compact!("{prefix}backup_group_uuid").as_str())?,
261            name: row.try_get(compact_str::format_compact!("{prefix}name").as_str())?,
262            successful: row.try_get(compact_str::format_compact!("{prefix}successful").as_str())?,
263            browsable: row.try_get(compact_str::format_compact!("{prefix}browsable").as_str())?,
264            streaming: row.try_get(compact_str::format_compact!("{prefix}streaming").as_str())?,
265            locked: row.try_get(compact_str::format_compact!("{prefix}locked").as_str())?,
266            shared: row.try_get(compact_str::format_compact!("{prefix}shared").as_str())?,
267            ignored_files: row
268                .try_get(compact_str::format_compact!("{prefix}ignored_files").as_str())?,
269            checksum: row.try_get(compact_str::format_compact!("{prefix}checksum").as_str())?,
270            bytes: row.try_get(compact_str::format_compact!("{prefix}bytes").as_str())?,
271            files: row.try_get(compact_str::format_compact!("{prefix}files").as_str())?,
272            disk: row.try_get(compact_str::format_compact!("{prefix}disk").as_str())?,
273            upload_id: row.try_get(compact_str::format_compact!("{prefix}upload_id").as_str())?,
274            upload_path: row
275                .try_get(compact_str::format_compact!("{prefix}upload_path").as_str())?,
276            metadata: row.try_get(compact_str::format_compact!("{prefix}metadata").as_str())?,
277            completed: row.try_get(compact_str::format_compact!("{prefix}completed").as_str())?,
278            deleting: row.try_get(compact_str::format_compact!("{prefix}deleting").as_str())?,
279            deletion_retries: row
280                .try_get(compact_str::format_compact!("{prefix}deletion_retries").as_str())?,
281            deleted: row.try_get(compact_str::format_compact!("{prefix}deleted").as_str())?,
282            created: row.try_get(compact_str::format_compact!("{prefix}created").as_str())?,
283            extension_data: Self::map_extensions(prefix, row)?,
284        })
285    }
286}
287
288impl ServerBackup {
289    pub async fn create_raw(
290        state: &crate::State,
291        mut options: CreateServerBackupOptions<'_>,
292    ) -> Result<Self, anyhow::Error> {
293        let backup_configuration = options
294            .server
295            .backup_configuration(&state.database)
296            .await
297            .ok_or_else(|| {
298                crate::response::DisplayError::new(
299                    "no backup configuration available, unable to create backup",
300                )
301                .with_status(StatusCode::EXPECTATION_FAILED)
302            })?;
303
304        if backup_configuration.maintenance_enabled {
305            return Err(crate::response::DisplayError::new(
306                "cannot create backup while backup configuration is in maintenance mode",
307            )
308            .with_status(StatusCode::EXPECTATION_FAILED)
309            .into());
310        }
311
312        let mut transaction = state.database.write().begin().await?;
313
314        let mut query_builder = InsertQueryBuilder::new("server_backups");
315
316        Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
317            .await?;
318
319        query_builder
320            .set("server_uuid", options.server.uuid)
321            .set("node_uuid", options.server.node.uuid)
322            .set("backup_configuration_uuid", backup_configuration.uuid)
323            .set("backup_group_uuid", options.backup_group_uuid)
324            .set("name", &options.name)
325            .set("ignored_files", &options.ignored_files)
326            .set("bytes", 0i64)
327            .set("disk", backup_configuration.backup_disk)
328            .set("shared", backup_configuration.shared)
329            .set("metadata", &options.metadata);
330
331        let row = query_builder
332            .returning(&Self::columns_sql(None))
333            .fetch_one(&mut *transaction)
334            .await?;
335        let mut backup = Self::map(None, &row)?;
336
337        Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
338
339        transaction.commit().await?;
340
341        Ok(backup)
342    }
343
344    pub async fn by_server_uuid_uuid(
345        database: &crate::database::Database,
346        server_uuid: uuid::Uuid,
347        uuid: uuid::Uuid,
348    ) -> Result<Option<Self>, crate::database::DatabaseError> {
349        let row = sqlx::query(sqlx::AssertSqlSafe(format!(
350            r#"
351            SELECT {}
352            FROM server_backups
353            WHERE server_backups.server_uuid = $1 AND server_backups.uuid = $2
354            "#,
355            Self::columns_sql(None)
356        )))
357        .bind(server_uuid)
358        .bind(uuid)
359        .fetch_optional(database.read())
360        .await?;
361
362        row.try_map(|row| Self::map(None, &row))
363    }
364
365    pub async fn select_completed_by_server_uuid(
366        database: &crate::database::Database,
367        server_uuid: uuid::Uuid,
368        name: Option<&str>,
369        backup_group_uuid: Option<uuid::Uuid>,
370        oldest: bool,
371    ) -> Result<Option<Self>, crate::database::DatabaseError> {
372        let row = sqlx::query(sqlx::AssertSqlSafe(format!(
373            r#"
374            SELECT {}
375            FROM server_backups
376            WHERE
377                server_backups.server_uuid = $1
378                AND server_backups.deleted IS NULL
379                AND server_backups.deleting IS NULL
380                AND server_backups.completed IS NOT NULL
381                AND server_backups.successful
382                AND ($2 IS NULL OR server_backups.name = $2)
383                AND ($3::uuid IS NULL OR server_backups.backup_group_uuid = $3)
384            ORDER BY server_backups.created {}
385            LIMIT 1
386            "#,
387            Self::columns_sql(None),
388            if oldest { "ASC" } else { "DESC" }
389        )))
390        .bind(server_uuid)
391        .bind(name)
392        .bind(backup_group_uuid)
393        .fetch_optional(database.read())
394        .await?;
395
396        row.try_map(|row| Self::map(None, &row))
397    }
398
399    pub async fn by_node_uuid_uuid(
400        database: &crate::database::Database,
401        node_uuid: uuid::Uuid,
402        uuid: uuid::Uuid,
403    ) -> Result<Option<Self>, crate::database::DatabaseError> {
404        let row = sqlx::query(sqlx::AssertSqlSafe(format!(
405            r#"
406            SELECT {}
407            FROM server_backups
408            WHERE server_backups.node_uuid = $1 AND server_backups.uuid = $2
409            "#,
410            Self::columns_sql(None)
411        )))
412        .bind(node_uuid)
413        .bind(uuid)
414        .fetch_optional(database.read())
415        .await?;
416
417        row.try_map(|row| Self::map(None, &row))
418    }
419
420    pub async fn by_server_uuid_with_pagination(
421        database: &crate::database::Database,
422        server_uuid: uuid::Uuid,
423        page: i64,
424        per_page: i64,
425        search: Option<&str>,
426    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
427        let offset = (page - 1) * per_page;
428
429        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
430            r#"
431            SELECT {}, COUNT(*) OVER() AS total_count
432            FROM server_backups
433            WHERE
434                server_backups.server_uuid = $1
435                AND server_backups.deleted IS NULL
436                AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
437            ORDER BY server_backups.created
438            LIMIT $3 OFFSET $4
439            "#,
440            Self::columns_sql(None)
441        )))
442        .bind(server_uuid)
443        .bind(search)
444        .bind(per_page)
445        .bind(offset)
446        .fetch_all(database.read())
447        .await?;
448
449        Ok(super::Pagination {
450            total: rows
451                .first()
452                .map_or(Ok(0), |row| row.try_get("total_count"))?,
453            per_page,
454            page,
455            data: rows
456                .into_iter()
457                .map(|row| Self::map(None, &row))
458                .try_collect_vec()?,
459        })
460    }
461
462    pub async fn by_server_uuid_node_uuid_with_pagination(
463        database: &crate::database::Database,
464        server_uuid: uuid::Uuid,
465        node_uuid: uuid::Uuid,
466        page: i64,
467        per_page: i64,
468        search: Option<&str>,
469    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
470        let offset = (page - 1) * per_page;
471
472        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
473            r#"
474            SELECT {}, COUNT(*) OVER() AS total_count
475            FROM server_backups
476            WHERE
477                server_backups.server_uuid = $1
478                AND server_backups.node_uuid = $2
479                AND server_backups.deleted IS NULL
480                AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
481            ORDER BY server_backups.created
482            LIMIT $4 OFFSET $5
483            "#,
484            Self::columns_sql(None)
485        )))
486        .bind(server_uuid)
487        .bind(node_uuid)
488        .bind(search)
489        .bind(per_page)
490        .bind(offset)
491        .fetch_all(database.read())
492        .await?;
493
494        Ok(super::Pagination {
495            total: rows
496                .first()
497                .map_or(Ok(0), |row| row.try_get("total_count"))?,
498            per_page,
499            page,
500            data: rows
501                .into_iter()
502                .map(|row| Self::map(None, &row))
503                .try_collect_vec()?,
504        })
505    }
506
507    pub async fn by_server_uuid_node_uuid_backup_group_uuid_with_pagination(
508        database: &crate::database::Database,
509        server_uuid: uuid::Uuid,
510        node_uuid: uuid::Uuid,
511        backup_group_uuid: uuid::Uuid,
512        page: i64,
513        per_page: i64,
514        search: Option<&str>,
515    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
516        let offset = (page - 1) * per_page;
517
518        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
519            r#"
520            SELECT {}, COUNT(*) OVER() AS total_count
521            FROM server_backups
522            WHERE
523                server_backups.server_uuid = $1
524                AND server_backups.node_uuid = $2
525                AND server_backups.backup_group_uuid = $3
526                AND server_backups.deleted IS NULL
527                AND ($4 IS NULL OR server_backups.name ILIKE '%' || $4 || '%')
528            ORDER BY server_backups.created
529            LIMIT $5 OFFSET $6
530            "#,
531            Self::columns_sql(None)
532        )))
533        .bind(server_uuid)
534        .bind(node_uuid)
535        .bind(backup_group_uuid)
536        .bind(search)
537        .bind(per_page)
538        .bind(offset)
539        .fetch_all(database.read())
540        .await?;
541
542        Ok(super::Pagination {
543            total: rows
544                .first()
545                .map_or(Ok(0), |row| row.try_get("total_count"))?,
546            per_page,
547            page,
548            data: rows
549                .into_iter()
550                .map(|row| Self::map(None, &row))
551                .try_collect_vec()?,
552        })
553    }
554
555    pub async fn by_ungrouped_server_uuid_node_uuid_with_pagination(
556        database: &crate::database::Database,
557        server_uuid: uuid::Uuid,
558        node_uuid: uuid::Uuid,
559        page: i64,
560        per_page: i64,
561        search: Option<&str>,
562    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
563        let offset = (page - 1) * per_page;
564
565        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
566            r#"
567            SELECT {}, COUNT(*) OVER() AS total_count
568            FROM server_backups
569            WHERE
570                server_backups.server_uuid = $1
571                AND server_backups.node_uuid = $2
572                AND server_backups.backup_group_uuid IS NULL
573                AND server_backups.deleted IS NULL
574                AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
575            ORDER BY server_backups.created
576            LIMIT $4 OFFSET $5
577            "#,
578            Self::columns_sql(None)
579        )))
580        .bind(server_uuid)
581        .bind(node_uuid)
582        .bind(search)
583        .bind(per_page)
584        .bind(offset)
585        .fetch_all(database.read())
586        .await?;
587
588        Ok(super::Pagination {
589            total: rows
590                .first()
591                .map_or(Ok(0), |row| row.try_get("total_count"))?,
592            per_page,
593            page,
594            data: rows
595                .into_iter()
596                .map(|row| Self::map(None, &row))
597                .try_collect_vec()?,
598        })
599    }
600
601    pub async fn by_partially_detached_server_uuid_node_uuid_with_pagination(
602        database: &crate::database::Database,
603        server_uuid: uuid::Uuid,
604        node_uuid: uuid::Uuid,
605        page: i64,
606        per_page: i64,
607        search: Option<&str>,
608    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
609        let offset = (page - 1) * per_page;
610
611        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
612            r#"
613            SELECT {}, COUNT(*) OVER() AS total_count
614            FROM server_backups
615            WHERE
616                server_backups.server_uuid = $1
617                AND server_backups.node_uuid != $2
618                AND server_backups.deleted IS NULL
619                AND ($3 IS NULL OR server_backups.name ILIKE '%' || $3 || '%')
620            ORDER BY server_backups.created
621            LIMIT $4 OFFSET $5
622            "#,
623            Self::columns_sql(None)
624        )))
625        .bind(server_uuid)
626        .bind(node_uuid)
627        .bind(search)
628        .bind(per_page)
629        .bind(offset)
630        .fetch_all(database.read())
631        .await?;
632
633        Ok(super::Pagination {
634            total: rows
635                .first()
636                .map_or(Ok(0), |row| row.try_get("total_count"))?,
637            per_page,
638            page,
639            data: rows
640                .into_iter()
641                .map(|row| Self::map(None, &row))
642                .try_collect_vec()?,
643        })
644    }
645
646    pub async fn by_node_uuid_with_pagination(
647        database: &crate::database::Database,
648        node_uuid: uuid::Uuid,
649        page: i64,
650        per_page: i64,
651        search: Option<&str>,
652    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
653        let offset = (page - 1) * per_page;
654
655        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
656            r#"
657            SELECT {}, COUNT(*) OVER() AS total_count
658            FROM server_backups
659            WHERE
660                server_backups.node_uuid = $1
661                AND server_backups.deleted IS NULL
662                AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
663            ORDER BY server_backups.created
664            LIMIT $3 OFFSET $4
665            "#,
666            Self::columns_sql(None)
667        )))
668        .bind(node_uuid)
669        .bind(search)
670        .bind(per_page)
671        .bind(offset)
672        .fetch_all(database.read())
673        .await?;
674
675        Ok(super::Pagination {
676            total: rows
677                .first()
678                .map_or(Ok(0), |row| row.try_get("total_count"))?,
679            per_page,
680            page,
681            data: rows
682                .into_iter()
683                .map(|row| Self::map(None, &row))
684                .try_collect_vec()?,
685        })
686    }
687
688    pub async fn by_backup_configuration_uuid_with_pagination(
689        database: &crate::database::Database,
690        backup_configuration_uuid: uuid::Uuid,
691        page: i64,
692        per_page: i64,
693        search: Option<&str>,
694    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
695        let offset = (page - 1) * per_page;
696
697        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
698            r#"
699            SELECT {}, COUNT(*) OVER() AS total_count
700            FROM server_backups
701            WHERE
702                server_backups.backup_configuration_uuid = $1
703                AND server_backups.deleted IS NULL
704                AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
705            ORDER BY server_backups.created
706            LIMIT $3 OFFSET $4
707            "#,
708            Self::columns_sql(None)
709        )))
710        .bind(backup_configuration_uuid)
711        .bind(search)
712        .bind(per_page)
713        .bind(offset)
714        .fetch_all(database.read())
715        .await?;
716
717        Ok(super::Pagination {
718            total: rows
719                .first()
720                .map_or(Ok(0), |row| row.try_get("total_count"))?,
721            per_page,
722            page,
723            data: rows
724                .into_iter()
725                .map(|row| Self::map(None, &row))
726                .try_collect_vec()?,
727        })
728    }
729
730    pub async fn by_detached_node_uuid_with_pagination(
731        database: &crate::database::Database,
732        node_uuid: uuid::Uuid,
733        page: i64,
734        per_page: i64,
735        search: Option<&str>,
736    ) -> Result<super::Pagination<Self>, crate::database::DatabaseError> {
737        let offset = (page - 1) * per_page;
738
739        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
740            r#"
741            SELECT {}, COUNT(*) OVER() AS total_count
742            FROM server_backups
743            WHERE
744                server_backups.node_uuid = $1
745                AND server_backups.server_uuid IS NULL
746                AND server_backups.deleted IS NULL
747                AND ($2 IS NULL OR server_backups.name ILIKE '%' || $2 || '%')
748            ORDER BY server_backups.created
749            LIMIT $3 OFFSET $4
750            "#,
751            Self::columns_sql(None)
752        )))
753        .bind(node_uuid)
754        .bind(search)
755        .bind(per_page)
756        .bind(offset)
757        .fetch_all(database.read())
758        .await?;
759
760        Ok(super::Pagination {
761            total: rows
762                .first()
763                .map_or(Ok(0), |row| row.try_get("total_count"))?,
764            per_page,
765            page,
766            data: rows
767                .into_iter()
768                .map(|row| Self::map(None, &row))
769                .try_collect_vec()?,
770        })
771    }
772
773    pub async fn all_uuids_by_server_uuid(
774        database: &crate::database::Database,
775        server_uuid: uuid::Uuid,
776    ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
777        let rows = sqlx::query(
778            r#"
779            SELECT server_backups.uuid
780            FROM server_backups
781            WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
782            "#,
783        )
784        .bind(server_uuid)
785        .fetch_all(database.read())
786        .await?;
787
788        Ok(rows
789            .into_iter()
790            .map(|row| row.get::<uuid::Uuid, _>("uuid"))
791            .collect())
792    }
793
794    pub async fn all_uuids_by_server_uuid_not_shared(
795        database: &crate::database::Database,
796        server_uuid: uuid::Uuid,
797    ) -> Result<Vec<uuid::Uuid>, crate::database::DatabaseError> {
798        let rows = sqlx::query(
799            r#"
800            SELECT server_backups.uuid
801            FROM server_backups
802            WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL AND server_backups.shared = false
803            "#,
804        )
805        .bind(server_uuid)
806        .fetch_all(database.read())
807        .await?;
808
809        Ok(rows
810            .into_iter()
811            .map(|row| row.get::<uuid::Uuid, _>("uuid"))
812            .collect())
813    }
814
815    pub async fn all_by_server_uuid(
816        database: &crate::database::Database,
817        server_uuid: uuid::Uuid,
818    ) -> Result<Vec<Self>, crate::database::DatabaseError> {
819        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
820            r#"
821            SELECT {}
822            FROM server_backups
823            WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
824            "#,
825            Self::columns_sql(None)
826        )))
827        .bind(server_uuid)
828        .fetch_all(database.read())
829        .await?;
830
831        rows.into_iter()
832            .map(|row| Self::map(None, &row))
833            .try_collect_vec()
834    }
835
836    pub async fn count_by_server_uuid(
837        database: &crate::database::Database,
838        server_uuid: uuid::Uuid,
839    ) -> Result<i64, sqlx::Error> {
840        sqlx::query_scalar(
841            r#"
842            SELECT COUNT(*)
843            FROM server_backups
844            WHERE server_backups.server_uuid = $1 AND server_backups.deleted IS NULL
845            "#,
846        )
847        .bind(server_uuid)
848        .fetch_one(database.read())
849        .await
850    }
851
852    pub async fn generate_metadata(
853        state: &crate::State,
854        server: &super::server::Server,
855    ) -> Result<serde_json::Value, anyhow::Error> {
856        let mut variables = serde_json::Map::new();
857
858        for variable in ServerVariable::all_by_server_uuid_egg_uuid(
859            &state.database,
860            server.uuid,
861            server.egg.uuid,
862        )
863        .await?
864        {
865            variables.insert(variable.variable.env_variable.into(), variable.value.into());
866        }
867
868        Ok(serde_json::json!({
869            "startup": server.startup,
870            "image": server.image,
871            "variables": variables,
872        }))
873    }
874
875    pub async fn download_url(
876        &self,
877        state: &crate::State,
878        user: &super::user::User,
879        node: &super::node::Node,
880        archive_format: wings_api::StreamableArchiveFormat,
881    ) -> Result<String, anyhow::Error> {
882        let backup_configuration = self
883            .backup_configuration
884            .as_ref()
885            .ok_or_else(|| {
886                crate::response::DisplayError::new(
887                    "no backup configuration available, unable to restore backup",
888                )
889                .with_status(StatusCode::EXPECTATION_FAILED)
890            })?
891            .fetch_cached(&state.database)
892            .await?;
893
894        if backup_configuration.maintenance_enabled {
895            return Err(crate::response::DisplayError::new(
896                "cannot restore backup while backup configuration is in maintenance mode",
897            )
898            .with_status(StatusCode::EXPECTATION_FAILED)
899            .into());
900        }
901
902        if matches!(self.disk, BackupDisk::S3)
903            && let Some(mut s3_configuration) = backup_configuration.backup_configs.s3
904        {
905            s3_configuration.decrypt(&state.database).await?;
906
907            let (client, bucket) = s3_configuration.into_client();
908
909            let file_path = match &self.upload_path {
910                Some(path) => path,
911                None => {
912                    return Err(crate::response::DisplayError::new(
913                        "backup does not have an upload path",
914                    )
915                    .with_status(StatusCode::EXPECTATION_FAILED)
916                    .into());
917                }
918            };
919
920            let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
921                std::time::Duration::from_mins(15),
922            )?;
923            let presigned = client
924                .get_object()
925                .bucket(bucket)
926                .key(&**file_path)
927                .presigned(presigning_config)
928                .await?;
929
930            return Ok(presigned.uri().to_string());
931        }
932
933        #[derive(Serialize)]
934        struct BackupDownloadJwt {
935            #[serde(flatten)]
936            base: BasePayload,
937
938            backup_uuid: uuid::Uuid,
939            unique_id: uuid::Uuid,
940        }
941
942        let token = node.create_jwt(
943            &state.database,
944            &state.jwt,
945            &BackupDownloadJwt {
946                base: BasePayload {
947                    scope: "backup-download".into(),
948                    issuer: "panel".into(),
949                    subject: None,
950                    audience: Vec::new(),
951                    expiration_time: Some(chrono::Utc::now().timestamp() + 900),
952                    not_before: None,
953                    issued_at: Some(chrono::Utc::now().timestamp()),
954                    jwt_id: user.uuid.to_compact_string(),
955                },
956                backup_uuid: self.uuid,
957                unique_id: uuid::Uuid::new_v4(),
958            },
959        )?;
960
961        let mut url = node.public_url(state, "/download/backup").await?;
962        url.set_query(Some(&format!(
963            "token={}&archive_format={}",
964            urlencoding::encode(&token),
965            archive_format
966        )));
967
968        Ok(url.to_string())
969    }
970
971    pub async fn restore(
972        self,
973        state: &crate::State,
974        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
975        mut server: super::server::Server,
976        options: ServerBackupRestoreOptions,
977    ) -> Result<(), anyhow::Error> {
978        let backup_configuration = self
979            .backup_configuration
980            .as_ref()
981            .ok_or_else(|| {
982                crate::response::DisplayError::new(
983                    "no backup configuration available, unable to restore backup",
984                )
985                .with_status(StatusCode::EXPECTATION_FAILED)
986            })?
987            .fetch_cached(&state.database)
988            .await?;
989
990        if backup_configuration.maintenance_enabled {
991            return Err(crate::response::DisplayError::new(
992                "cannot restore backup while backup configuration is in maintenance mode",
993            )
994            .with_status(StatusCode::EXPECTATION_FAILED)
995            .into());
996        }
997
998        if options.restore_startup {
999            self.restore_startup(state, transaction, &mut server)
1000                .await?;
1001        }
1002
1003        server
1004            .node
1005            .fetch_cached(&state.database)
1006            .await?
1007            .api_client(&state.database)
1008            .await?
1009            .post_servers_server_backup_backup_restore(
1010                server.uuid,
1011                self.uuid,
1012                &wings_api::servers_server_backup_backup_restore::post::RequestBody {
1013                    adapter: self.disk.to_wings_adapter(),
1014                    download_url: self.wings_restore_download_url(state, server.uuid).await?,
1015                    truncate_directory: options.truncate_directory,
1016                },
1017            )
1018            .await?;
1019
1020        Self::get_event_emitter().emit(
1021            state.clone(),
1022            ServerBackupEvent::RestoreStarted {
1023                backup: Box::new(self),
1024                server: Box::new(server),
1025            },
1026        );
1027
1028        Ok(())
1029    }
1030
1031    pub async fn restore_startup(
1032        &self,
1033        state: &crate::State,
1034        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1035        server: &mut super::server::Server,
1036    ) -> Result<(), anyhow::Error> {
1037        let startup_cmd = self
1038            .metadata
1039            .get("startup")
1040            .and_then(|v| v.as_str())
1041            .map(|s| s.to_compact_string());
1042        let image_str = self
1043            .metadata
1044            .get("image")
1045            .and_then(|v| v.as_str())
1046            .map(|s| s.to_compact_string());
1047        let variables = self
1048            .metadata
1049            .get("variables")
1050            .and_then(|v| v.as_object())
1051            .cloned()
1052            .unwrap_or_default();
1053
1054        if let Some(startup) = startup_cmd
1055            && let Ok(egg_config) = server.egg.configuration(&state.database).await
1056        {
1057            let is_predefined = server
1058                .egg
1059                .startup_commands
1060                .values()
1061                .any(|cmd| cmd == startup.as_str());
1062            let custom_allowed = egg_config
1063                .config_startup
1064                .as_ref()
1065                .is_some_and(|c| c.allow_custom_startup_command);
1066            if is_predefined || custom_allowed {
1067                server
1068                    .update_with_transaction(
1069                        state,
1070                        super::server::UpdateServerOptions {
1071                            startup: Some(startup),
1072                            ..Default::default()
1073                        },
1074                        transaction,
1075                    )
1076                    .await?;
1077            }
1078        }
1079
1080        if let Some(image) = image_str {
1081            let is_valid_image = server
1082                .egg
1083                .docker_images
1084                .values()
1085                .any(|img| img == image.as_str());
1086            if is_valid_image {
1087                let current_is_custom = !server
1088                    .egg
1089                    .docker_images
1090                    .values()
1091                    .any(|img| img == server.image.as_str());
1092                let allow_overwrite = state
1093                    .settings
1094                    .get()
1095                    .await
1096                    .map(|s| s.server.allow_overwriting_custom_docker_image)
1097                    .unwrap_or(false);
1098                if !current_is_custom || allow_overwrite {
1099                    server
1100                        .update_with_transaction(
1101                            state,
1102                            super::server::UpdateServerOptions {
1103                                image: Some(image),
1104                                ..Default::default()
1105                            },
1106                            transaction,
1107                        )
1108                        .await?;
1109                }
1110            }
1111        }
1112
1113        if !variables.is_empty() {
1114            let existing_variables = ServerVariable::all_by_server_uuid_egg_uuid(
1115                &state.database,
1116                server.uuid,
1117                server.egg.uuid,
1118            )
1119            .await?;
1120
1121            let mut validator_variables = HashMap::new();
1122            for variable in existing_variables.iter() {
1123                validator_variables.insert(
1124                    variable.variable.env_variable.as_str(),
1125                    (
1126                        variable.variable.rules.as_slice(),
1127                        if let Some(value) = variables
1128                            .iter()
1129                            .find(|v| v.0 == variable.variable.env_variable)
1130                            && variable.variable.user_editable
1131                            && let Some(value) = value.1.as_str()
1132                        {
1133                            value
1134                        } else {
1135                            variable.value.as_str()
1136                        },
1137                    ),
1138                );
1139            }
1140
1141            let validator = match rule_validator::Validator::new(validator_variables) {
1142                Ok(validator) => validator,
1143                Err(error) => {
1144                    return Err(crate::response::DisplayError::new(error)
1145                        .with_status(StatusCode::EXPECTATION_FAILED)
1146                        .into());
1147                }
1148            };
1149            if let Err(error) = validator.validate() {
1150                return Err(crate::response::DisplayError::new(error)
1151                    .with_status(StatusCode::EXPECTATION_FAILED)
1152                    .into());
1153            }
1154
1155            for (env_var, value) in &variables {
1156                let Some(value) = value.as_str() else {
1157                    continue;
1158                };
1159                let variable_uuid = match existing_variables
1160                    .iter()
1161                    .find(|v| v.variable.env_variable == env_var)
1162                {
1163                    Some(variable) if variable.variable.user_editable => variable.variable.uuid,
1164                    _ => continue,
1165                };
1166
1167                ServerVariable::create_with_transaction(
1168                    transaction,
1169                    server.uuid,
1170                    variable_uuid,
1171                    value,
1172                )
1173                .await?;
1174            }
1175        }
1176
1177        Ok(())
1178    }
1179
1180    pub async fn wings_restore_download_url(
1181        &self,
1182        state: &crate::State,
1183        server_uuid: uuid::Uuid,
1184    ) -> Result<Option<compact_str::CompactString>, anyhow::Error> {
1185        if !matches!(self.disk, BackupDisk::S3) {
1186            return Ok(None);
1187        }
1188
1189        let backup_configuration = self
1190            .backup_configuration
1191            .as_ref()
1192            .ok_or_else(|| {
1193                crate::response::DisplayError::new(
1194                    "no backup configuration available, unable to restore backup",
1195                )
1196                .with_status(StatusCode::EXPECTATION_FAILED)
1197            })?
1198            .fetch_cached(&state.database)
1199            .await?;
1200
1201        let Some(mut s3_configuration) = backup_configuration.backup_configs.s3 else {
1202            return Ok(None);
1203        };
1204
1205        s3_configuration.decrypt(&state.database).await?;
1206
1207        let compression_type = s3_configuration.compression_type;
1208        let (client, bucket) = s3_configuration.into_client();
1209
1210        let file_path = match &self.upload_path {
1211            Some(path) => path.as_str(),
1212            None => &Self::s3_path(server_uuid, self.uuid, compression_type),
1213        };
1214
1215        let presigning_config = aws_sdk_s3::presigning::PresigningConfig::expires_in(
1216            std::time::Duration::from_mins(60),
1217        )?;
1218        let presigned = client
1219            .get_object()
1220            .bucket(bucket)
1221            .key(file_path)
1222            .presigned(presigning_config)
1223            .await?;
1224
1225        Ok(Some(presigned.uri().to_compact_string()))
1226    }
1227
1228    pub async fn export(
1229        &self,
1230        state: &crate::State,
1231        server: &super::server::Server,
1232        path: compact_str::CompactString,
1233        archive_format: wings_api::StreamableArchiveFormat,
1234        foreground: bool,
1235    ) -> Result<wings_api::backups_backup_export::post::Response, anyhow::Error> {
1236        let backup_configuration = self
1237            .backup_configuration
1238            .as_ref()
1239            .ok_or_else(|| {
1240                crate::response::DisplayError::new(
1241                    "no backup configuration available, unable to export backup",
1242                )
1243                .with_status(StatusCode::EXPECTATION_FAILED)
1244            })?
1245            .fetch_cached(&state.database)
1246            .await?;
1247
1248        if backup_configuration.maintenance_enabled {
1249            return Err(crate::response::DisplayError::new(
1250                "cannot export backup while backup configuration is in maintenance mode",
1251            )
1252            .with_status(StatusCode::EXPECTATION_FAILED)
1253            .into());
1254        }
1255
1256        let client = server
1257            .node
1258            .fetch_cached(&state.database)
1259            .await?
1260            .api_client(&state.database)
1261            .await?;
1262
1263        match client
1264            .post_backups_backup_export(
1265                self.uuid,
1266                &wings_api::backups_backup_export::post::RequestBody {
1267                    adapter: self.disk.to_wings_adapter(),
1268                    server: server.uuid,
1269                    path,
1270                    archive_format,
1271                    foreground,
1272                },
1273            )
1274            .await
1275        {
1276            Ok(response) => Ok(response),
1277            Err(wings_api::client::ApiHttpError::Http(
1278                status @ (StatusCode::NOT_FOUND | StatusCode::EXPECTATION_FAILED),
1279                err,
1280            )) => Err(crate::response::DisplayError::new(
1281                crate::ApiError::new_wings_value(err).to_string(),
1282            )
1283            .with_status(status)
1284            .into()),
1285            Err(err) => Err(err.into()),
1286        }
1287    }
1288
1289    pub async fn query(
1290        &self,
1291        state: &crate::State,
1292        node: &super::node::Node,
1293    ) -> Result<wings_api::backups_backup_query::get::Response, anyhow::Error> {
1294        let client = node.api_client(&state.database).await?;
1295
1296        match client
1297            .get_backups_backup_query(
1298                self.uuid,
1299                &wings_api::backups_backup_query::get::Query {
1300                    adapter: Some(self.disk.to_wings_adapter()),
1301                    __priv: (),
1302                },
1303            )
1304            .await
1305        {
1306            Ok(response) => Ok(response),
1307            Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, err)) => {
1308                Err(crate::response::DisplayError::new(
1309                    crate::ApiError::new_wings_value(err).to_string(),
1310                )
1311                .with_status(StatusCode::NOT_FOUND)
1312                .into())
1313            }
1314            Err(err) => Err(err.into()),
1315        }
1316    }
1317
1318    pub async fn evict_one_by_server_uuid(
1319        state: &crate::State,
1320        server: &super::server::Server,
1321    ) -> Result<(), anyhow::Error> {
1322        let row = sqlx::query(
1323            r#"
1324            SELECT candidates.uuid, candidates.tier, candidates.group_name
1325            FROM (
1326                SELECT
1327                    server_backups.uuid,
1328                    server_backups.created,
1329                    (CASE
1330                        WHEN NOT server_backups.successful THEN 0
1331                        WHEN g.retention_count IS NOT NULL AND (
1332                            SELECT COUNT(*)
1333                            FROM server_backups b2
1334                            WHERE b2.backup_group_uuid = server_backups.backup_group_uuid
1335                                AND b2.deleted IS NULL
1336                                AND b2.deleting IS NULL
1337                                AND b2.successful
1338                                AND b2.completed IS NOT NULL
1339                                AND b2.created >= server_backups.created
1340                        ) > g.retention_count THEN 1
1341                        WHEN server_backups.backup_group_uuid IS NULL THEN 2
1342                        ELSE 3
1343                    END) AS tier,
1344                    g.name AS group_name
1345                FROM server_backups
1346                LEFT JOIN server_backup_groups g ON g.uuid = server_backups.backup_group_uuid
1347                WHERE server_backups.server_uuid = $1
1348                    AND server_backups.locked = false
1349                    AND server_backups.completed IS NOT NULL
1350                    AND server_backups.deleted IS NULL
1351                    AND server_backups.deleting IS NULL
1352            ) candidates
1353            ORDER BY candidates.tier ASC, candidates.created ASC
1354            LIMIT 1
1355            "#,
1356        )
1357        .bind(server.uuid)
1358        .fetch_optional(state.database.read())
1359        .await?;
1360
1361        let Some(row) = row else {
1362            return Err(sqlx::Error::RowNotFound.into());
1363        };
1364
1365        let row_uuid: uuid::Uuid = row.try_get("uuid")?;
1366        let row_tier: i32 = row.try_get("tier")?;
1367        let row_group_name: Option<String> = row.try_get("group_name")?;
1368
1369        let rule = match row_tier {
1370            0 => "failed",
1371            1 => "over-retention",
1372            2 => "ungrouped",
1373            _ => "in-retention",
1374        };
1375
1376        if row_tier == 3 {
1377            tracing::warn!(
1378                server = %server.uuid,
1379                backup = %row_uuid,
1380                group = ?row_group_name,
1381                "evicting an in-retention grouped backup to satisfy backup_limit; retention quota exceeds backup_limit"
1382            );
1383        }
1384
1385        let Some(backup) =
1386            Self::by_server_uuid_uuid(&state.database, server.uuid, row_uuid).await?
1387        else {
1388            return Err(sqlx::Error::RowNotFound.into());
1389        };
1390
1391        backup.delete(state, Default::default()).await?;
1392
1393        Self::log_eviction_activity(
1394            state,
1395            server.uuid,
1396            backup.uuid,
1397            &backup.name,
1398            rule,
1399            row_group_name.as_deref(),
1400        )
1401        .await;
1402
1403        Ok(())
1404    }
1405
1406    pub async fn rotate_group_for_create(
1407        state: &crate::State,
1408        group: &super::server_backup_group::ServerBackupGroup,
1409    ) -> Result<GroupRotationOutcome, anyhow::Error> {
1410        let Some(retention_count) = group.retention_count else {
1411            return Ok(GroupRotationOutcome::NotConfigured);
1412        };
1413
1414        let row = sqlx::query(
1415            r#"
1416            SELECT
1417                (SELECT COUNT(*)
1418                    FROM server_backups
1419                    WHERE server_backups.backup_group_uuid = $1
1420                        AND server_backups.deleted IS NULL
1421                        AND server_backups.deleting IS NULL
1422                        AND server_backups.successful
1423                        AND server_backups.completed IS NOT NULL) AS usable,
1424                (SELECT server_backups.uuid
1425                    FROM server_backups
1426                    WHERE server_backups.backup_group_uuid = $1
1427                        AND server_backups.deleted IS NULL
1428                        AND server_backups.deleting IS NULL
1429                        AND server_backups.successful
1430                        AND server_backups.completed IS NOT NULL
1431                        AND server_backups.locked = false
1432                    ORDER BY server_backups.created ASC
1433                    LIMIT 1) AS oldest_unlocked
1434            "#,
1435        )
1436        .bind(group.uuid)
1437        .fetch_one(state.database.read())
1438        .await?;
1439
1440        let usable: i64 = row.try_get("usable")?;
1441        let oldest_unlocked: Option<uuid::Uuid> = row.try_get("oldest_unlocked")?;
1442
1443        if usable < retention_count as i64 {
1444            return Ok(GroupRotationOutcome::WithinRetention);
1445        }
1446
1447        let Some(oldest_unlocked) = oldest_unlocked else {
1448            return Ok(GroupRotationOutcome::BlockedAllLocked);
1449        };
1450
1451        let Some(backup) =
1452            Self::by_server_uuid_uuid(&state.database, group.server_uuid, oldest_unlocked).await?
1453        else {
1454            return Ok(GroupRotationOutcome::WithinRetention);
1455        };
1456
1457        backup.delete(state, Default::default()).await?;
1458
1459        Self::log_eviction_activity(
1460            state,
1461            group.server_uuid,
1462            backup.uuid,
1463            &backup.name,
1464            "group-rotation",
1465            Some(group.name.as_str()),
1466        )
1467        .await;
1468
1469        Ok(GroupRotationOutcome::Evicted)
1470    }
1471
1472    pub async fn prune_expired_group_backups(state: &crate::State) -> Result<u64, anyhow::Error> {
1473        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
1474            r#"
1475            SELECT {}, g.name AS group_name
1476            FROM server_backups
1477            JOIN server_backup_groups g ON g.uuid = server_backups.backup_group_uuid
1478            WHERE g.retention_days IS NOT NULL
1479                AND server_backups.deleted IS NULL
1480                AND server_backups.deleting IS NULL
1481                AND server_backups.locked = false
1482                AND server_backups.completed IS NOT NULL
1483                AND server_backups.created < NOW() - make_interval(days => g.retention_days)
1484            "#,
1485            Self::columns_sql(None)
1486        )))
1487        .fetch_all(state.database.read())
1488        .await?;
1489
1490        let mut pruned = 0;
1491        for row in rows {
1492            let group_name: compact_str::CompactString = row.try_get("group_name")?;
1493            let server_uuid: Option<uuid::Uuid> = row.try_get("server_uuid")?;
1494            let backup = Self::map(None, &row)?;
1495
1496            if let Err(err) = backup.delete(state, Default::default()).await {
1497                tracing::error!(
1498                    backup = %backup.uuid,
1499                    "failed to prune expired group backup: {:#?}",
1500                    err
1501                );
1502                continue;
1503            }
1504
1505            if let Some(server_uuid) = server_uuid {
1506                Self::log_eviction_activity(
1507                    state,
1508                    server_uuid,
1509                    backup.uuid,
1510                    &backup.name,
1511                    "retention-days",
1512                    Some(group_name.as_str()),
1513                )
1514                .await;
1515            }
1516
1517            pruned += 1;
1518        }
1519
1520        Ok(pruned)
1521    }
1522
1523    async fn log_eviction_activity(
1524        state: &crate::State,
1525        server_uuid: uuid::Uuid,
1526        backup_uuid: uuid::Uuid,
1527        backup_name: &str,
1528        rule: &str,
1529        group_name: Option<&str>,
1530    ) {
1531        if let Err(err) = super::server_activity::ServerActivity::create(
1532            state,
1533            super::server_activity::CreateServerActivityOptions {
1534                server_uuid,
1535                user_uuid: None,
1536                impersonator_uuid: None,
1537                api_key_uuid: None,
1538                schedule_uuid: None,
1539                event: "server:backup.delete".into(),
1540                ip: None,
1541                data: serde_json::json!({
1542                    "source": "eviction",
1543                    "uuid": backup_uuid,
1544                    "name": backup_name,
1545                    "rule": rule,
1546                    "group": group_name,
1547                }),
1548                created: None,
1549            },
1550        )
1551        .await
1552        {
1553            tracing::warn!(
1554                server = %server_uuid,
1555                "failed to log backup eviction activity: {:#?}",
1556                err
1557            );
1558        }
1559    }
1560
1561    #[inline]
1562    pub fn default_name() -> compact_str::CompactString {
1563        let now = chrono::Local::now();
1564
1565        now.format("%Y-%m-%d %H:%M:%S %z").to_compact_string()
1566    }
1567
1568    #[inline]
1569    pub fn s3_path(
1570        server_uuid: uuid::Uuid,
1571        backup_uuid: uuid::Uuid,
1572        compression_type: wings_api::CompressionType,
1573    ) -> compact_str::CompactString {
1574        compact_str::format_compact!(
1575            "{server_uuid}/{backup_uuid}.tar{}",
1576            match compression_type {
1577                wings_api::CompressionType::None => "",
1578                wings_api::CompressionType::Gz => ".gz",
1579                wings_api::CompressionType::Xz => ".xz",
1580                wings_api::CompressionType::Lzip => ".lz",
1581                wings_api::CompressionType::Bz2 => ".bz2",
1582                wings_api::CompressionType::Lz4 => ".lz4",
1583                wings_api::CompressionType::Zstd => ".zst",
1584            }
1585        )
1586    }
1587
1588    #[inline]
1589    pub fn s3_content_type(name: &str) -> &'static str {
1590        if name.ends_with("tar") {
1591            "application/x-tar"
1592        } else if name.ends_with(".tar.gz") {
1593            "application/x-gzip"
1594        } else if name.ends_with(".tar.xz") {
1595            "application/x-xz"
1596        } else if name.ends_with(".tar.lz") {
1597            "application/x-lzip"
1598        } else if name.ends_with(".tar.bz2") {
1599            "application/x-bzip2"
1600        } else if name.ends_with(".tar.lz4") {
1601            "application/x-lz4"
1602        } else if name.ends_with(".tar.zst") {
1603            "application/zstd"
1604        } else {
1605            "application/octet-stream"
1606        }
1607    }
1608
1609    pub async fn into_admin_node_api_object(
1610        self,
1611        state: &crate::State,
1612        storage_url_retriever: &StorageUrlRetriever<'_>,
1613    ) -> Result<AdminApiNodeServerBackup, crate::database::DatabaseError> {
1614        let deletion_status = self.deletion_status();
1615
1616        Ok(AdminApiNodeServerBackup {
1617            uuid: self.uuid,
1618            server: match self.server {
1619                Some(server) => Some(
1620                    server
1621                        .fetch_cached(&state.database)
1622                        .await?
1623                        .into_admin_api_object(state, storage_url_retriever)
1624                        .await?,
1625                ),
1626                None => None,
1627            },
1628            node: self
1629                .node
1630                .fetch_cached(&state.database)
1631                .await?
1632                .into_admin_api_object(state, ())
1633                .await?,
1634            backup_group_uuid: self.backup_group_uuid,
1635            name: self.name,
1636            ignored_files: self.ignored_files,
1637            is_successful: self.successful,
1638            is_locked: self.locked,
1639            is_browsable: self.browsable,
1640            is_streaming: self.streaming,
1641            is_shared: self.shared,
1642            checksum: self.checksum,
1643            bytes: self.bytes,
1644            files: self.files,
1645            deletion_status,
1646            metadata: self.metadata,
1647            completed: self.completed.map(|dt| dt.and_utc()),
1648            created: self.created.and_utc(),
1649        })
1650    }
1651}
1652
1653#[async_trait::async_trait]
1654impl IntoAdminApiObject for ServerBackup {
1655    type AdminApiObject = AdminApiServerBackup;
1656    type ExtraArgs<'a> = &'a crate::storage::StorageUrlRetriever<'a>;
1657
1658    async fn into_admin_api_object<'a>(
1659        self,
1660        state: &crate::State,
1661        storage_url_retriever: Self::ExtraArgs<'a>,
1662    ) -> Result<Self::AdminApiObject, crate::database::DatabaseError> {
1663        let deletion_status = self.deletion_status();
1664        let api_object = AdminApiServerBackup::init_hooks(&self, state).await?;
1665
1666        let api_object = finish_extendible!(
1667            AdminApiServerBackup {
1668                uuid: self.uuid,
1669                server: match self.server {
1670                    Some(server) => Some(
1671                        server
1672                            .fetch_cached(&state.database)
1673                            .await?
1674                            .into_admin_api_object(state, storage_url_retriever)
1675                            .await?,
1676                    ),
1677                    None => None,
1678                },
1679                backup_group_uuid: self.backup_group_uuid,
1680                name: self.name,
1681                ignored_files: self.ignored_files,
1682                is_successful: self.successful,
1683                is_locked: self.locked,
1684                is_browsable: self.browsable,
1685                is_streaming: self.streaming,
1686                is_shared: self.shared,
1687                checksum: self.checksum,
1688                bytes: self.bytes,
1689                files: self.files,
1690                deletion_status,
1691                metadata: self.metadata,
1692                completed: self.completed.map(|dt| dt.and_utc()),
1693                created: self.created.and_utc(),
1694            },
1695            api_object,
1696            state
1697        )?;
1698
1699        Ok(api_object)
1700    }
1701}
1702
1703#[async_trait::async_trait]
1704impl IntoApiObject for ServerBackup {
1705    type ApiObject = ApiServerBackup;
1706    type ExtraArgs<'a> = ();
1707
1708    async fn into_api_object<'a>(
1709        self,
1710        state: &crate::State,
1711        _args: Self::ExtraArgs<'a>,
1712    ) -> Result<Self::ApiObject, crate::database::DatabaseError> {
1713        let deletion_status = self.deletion_status();
1714        let api_object = ApiServerBackup::init_hooks(&self, state).await?;
1715
1716        let api_object = finish_extendible!(
1717            ApiServerBackup {
1718                uuid: self.uuid,
1719                backup_group_uuid: self.backup_group_uuid,
1720                name: self.name,
1721                ignored_files: self.ignored_files,
1722                is_successful: self.successful,
1723                is_locked: self.locked,
1724                is_browsable: self.browsable,
1725                is_streaming: self.streaming,
1726                checksum: self.checksum,
1727                bytes: self.bytes,
1728                files: self.files,
1729                deletion_status,
1730                metadata: self.metadata,
1731                completed: self.completed.map(|dt| dt.and_utc()),
1732                created: self.created.and_utc(),
1733            },
1734            api_object,
1735            state
1736        )?;
1737
1738        Ok(api_object)
1739    }
1740}
1741
1742#[derive(Validate)]
1743pub struct CreateServerBackupOptions<'a> {
1744    #[garde(skip)]
1745    pub server: &'a super::server::Server,
1746    #[garde(length(chars, min = 1, max = 255))]
1747    pub name: compact_str::CompactString,
1748    #[garde(skip)]
1749    pub backup_group_uuid: Option<uuid::Uuid>,
1750    #[garde(skip)]
1751    pub ignored_files: Vec<compact_str::CompactString>,
1752    #[garde(skip)]
1753    pub metadata: serde_json::Value,
1754}
1755
1756#[async_trait::async_trait]
1757impl CreatableModel for ServerBackup {
1758    type CreateOptions<'a> = CreateServerBackupOptions<'a>;
1759    type CreateResult = Self;
1760
1761    fn get_create_handlers() -> &'static LazyLock<CreateListenerList<Self>> {
1762        static CREATE_LISTENERS: LazyLock<CreateListenerList<ServerBackup>> =
1763            LazyLock::new(|| Arc::new(ModelHandlerList::default()));
1764
1765        &CREATE_LISTENERS
1766    }
1767
1768    async fn create_with_transaction(
1769        _state: &crate::State,
1770        _options: Self::CreateOptions<'_>,
1771        _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1772    ) -> Result<Self, crate::database::DatabaseError> {
1773        Err(anyhow::anyhow!("create_with_transaction is not supported for ServerBackup").into())
1774    }
1775
1776    async fn create(
1777        state: &crate::State,
1778        mut options: Self::CreateOptions<'_>,
1779    ) -> Result<Self, crate::database::DatabaseError> {
1780        options.validate()?;
1781
1782        let backup_configuration = options
1783            .server
1784            .backup_configuration(&state.database)
1785            .await
1786            .ok_or_else(|| {
1787                anyhow::Error::new(
1788                    crate::response::DisplayError::new(
1789                        "no backup configuration available, unable to create backup",
1790                    )
1791                    .with_status(StatusCode::EXPECTATION_FAILED),
1792                )
1793            })?;
1794
1795        if backup_configuration.maintenance_enabled {
1796            return Err(anyhow::Error::new(
1797                crate::response::DisplayError::new(
1798                    "cannot create backup while backup configuration is in maintenance mode",
1799                )
1800                .with_status(StatusCode::EXPECTATION_FAILED),
1801            )
1802            .into());
1803        }
1804
1805        let mut transaction = state.database.write().begin().await?;
1806
1807        let mut query_builder = InsertQueryBuilder::new("server_backups");
1808
1809        Self::run_create_handlers(&mut options, &mut query_builder, state, &mut transaction)
1810            .await?;
1811
1812        query_builder
1813            .set("server_uuid", options.server.uuid)
1814            .set("node_uuid", options.server.node.uuid)
1815            .set("backup_configuration_uuid", backup_configuration.uuid)
1816            .set("backup_group_uuid", options.backup_group_uuid)
1817            .set("name", &options.name)
1818            .set("ignored_files", &options.ignored_files)
1819            .set("bytes", 0i64)
1820            .set("disk", backup_configuration.backup_disk)
1821            .set("shared", backup_configuration.shared)
1822            .set("metadata", &options.metadata);
1823
1824        let row = query_builder
1825            .returning(&Self::columns_sql(None))
1826            .fetch_one(&mut *transaction)
1827            .await?;
1828        let mut backup = Self::map(None, &row)?;
1829
1830        Self::run_after_create_handlers(&mut backup, &options, state, &mut transaction).await?;
1831
1832        transaction.commit().await?;
1833
1834        let server = options.server.clone();
1835        let database = Arc::clone(&state.database);
1836        let backup_uuid = backup.uuid;
1837        let backup_disk = backup_configuration.backup_disk;
1838        let ignored_files_str = options
1839            .ignored_files
1840            .iter()
1841            .map(|s| s.as_str())
1842            .collect::<Vec<_>>()
1843            .join("\n");
1844
1845        tokio::spawn(async move {
1846            tracing::debug!(backup = %backup_uuid, "creating server backup");
1847
1848            let node = match server.node.fetch_cached(&database).await {
1849                Ok(node) => node,
1850                Err(err) => {
1851                    tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
1852
1853                    if let Err(err) = sqlx::query!(
1854                        "UPDATE server_backups
1855                        SET successful = false, completed = NOW()
1856                        WHERE server_backups.uuid = $1",
1857                        backup_uuid
1858                    )
1859                    .execute(database.write())
1860                    .await
1861                    {
1862                        tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
1863                    }
1864
1865                    return;
1866                }
1867            };
1868
1869            let api_client = match node.api_client(&database).await {
1870                Ok(api_client) => api_client,
1871                Err(err) => {
1872                    tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
1873
1874                    if let Err(err) = sqlx::query!(
1875                        "UPDATE server_backups
1876                        SET successful = false, completed = NOW()
1877                        WHERE server_backups.uuid = $1",
1878                        backup_uuid
1879                    )
1880                    .execute(database.write())
1881                    .await
1882                    {
1883                        tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
1884                    }
1885
1886                    return;
1887                }
1888            };
1889
1890            if let Err(err) = api_client
1891                .post_servers_server_backup(
1892                    server.uuid,
1893                    &wings_api::servers_server_backup::post::RequestBody {
1894                        adapter: backup_disk.to_wings_adapter(),
1895                        uuid: backup_uuid,
1896                        ignore: ignored_files_str.into(),
1897                    },
1898                )
1899                .await
1900            {
1901                tracing::error!(backup = %backup_uuid, "failed to create server backup: {:?}", err);
1902
1903                if let Err(err) = sqlx::query!(
1904                    "UPDATE server_backups
1905                    SET successful = false, completed = NOW()
1906                    WHERE server_backups.uuid = $1",
1907                    backup_uuid
1908                )
1909                .execute(database.write())
1910                .await
1911                {
1912                    tracing::error!(backup = %backup_uuid, "failed to update server backup status: {:?}", err);
1913                }
1914            }
1915        });
1916
1917        Ok(backup)
1918    }
1919}
1920
1921#[derive(ToSchema, Serialize, Deserialize, Validate, Default)]
1922pub struct UpdateServerBackupOptions {
1923    #[garde(length(chars, min = 1, max = 255))]
1924    #[schema(min_length = 1, max_length = 255)]
1925    pub name: Option<compact_str::CompactString>,
1926    #[garde(skip)]
1927    #[serde(default, with = "::serde_with::rust::double_option")]
1928    pub backup_group_uuid: Option<Option<uuid::Uuid>>,
1929    #[garde(skip)]
1930    pub locked: Option<bool>,
1931}
1932
1933#[async_trait::async_trait]
1934impl UpdatableModel for ServerBackup {
1935    type UpdateOptions = UpdateServerBackupOptions;
1936
1937    fn get_update_handlers() -> &'static LazyLock<UpdateHandlerList<Self>> {
1938        static UPDATE_LISTENERS: LazyLock<UpdateHandlerList<ServerBackup>> =
1939            LazyLock::new(|| Arc::new(ModelHandlerList::default()));
1940
1941        &UPDATE_LISTENERS
1942    }
1943
1944    async fn update_with_transaction(
1945        &mut self,
1946        state: &crate::State,
1947        mut options: Self::UpdateOptions,
1948        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
1949    ) -> Result<(), crate::database::DatabaseError> {
1950        options.validate()?;
1951
1952        let mut query_builder = UpdateQueryBuilder::new("server_backups");
1953
1954        self.run_update_handlers(&mut options, &mut query_builder, state, transaction)
1955            .await?;
1956
1957        query_builder
1958            .set("name", options.name.as_ref())
1959            .set("backup_group_uuid", options.backup_group_uuid)
1960            .set("locked", options.locked)
1961            .where_eq("uuid", self.uuid);
1962
1963        query_builder.execute(&mut **transaction).await?;
1964
1965        if let Some(name) = options.name {
1966            self.name = name;
1967        }
1968        if let Some(backup_group_uuid) = options.backup_group_uuid {
1969            self.backup_group_uuid = backup_group_uuid;
1970        }
1971        if let Some(locked) = options.locked {
1972            self.locked = locked;
1973        }
1974
1975        self.run_after_update_handlers(state, transaction).await?;
1976
1977        Ok(())
1978    }
1979}
1980
1981#[async_trait::async_trait]
1982impl ByUuid for ServerBackup {
1983    async fn by_uuid(
1984        database: &crate::database::Database,
1985        uuid: uuid::Uuid,
1986    ) -> Result<Self, crate::database::DatabaseError> {
1987        let row = sqlx::query(sqlx::AssertSqlSafe(format!(
1988            r#"
1989            SELECT {}
1990            FROM server_backups
1991            WHERE server_backups.uuid = $1
1992            "#,
1993            Self::columns_sql(None)
1994        )))
1995        .bind(uuid)
1996        .fetch_one(database.read())
1997        .await?;
1998
1999        Self::map(None, &row)
2000    }
2001
2002    async fn by_uuid_with_transaction(
2003        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2004        uuid: uuid::Uuid,
2005    ) -> Result<Self, crate::database::DatabaseError> {
2006        let row = sqlx::query(sqlx::AssertSqlSafe(format!(
2007            r#"
2008            SELECT {}
2009            FROM server_backups
2010            WHERE server_backups.uuid = $1
2011            "#,
2012            Self::columns_sql(None)
2013        )))
2014        .bind(uuid)
2015        .fetch_one(&mut **transaction)
2016        .await?;
2017
2018        Self::map(None, &row)
2019    }
2020}
2021
2022#[derive(Clone, Default)]
2023pub struct DeleteServerBackupOptions {
2024    pub force: bool,
2025}
2026
2027impl ServerBackup {
2028    pub const MAX_DELETION_RETRIES: i32 = 8;
2029
2030    #[inline]
2031    pub fn deletion_status(&self) -> Option<ServerBackupDeletionStatus> {
2032        if self.deleted.is_some() || self.deleting.is_none() {
2033            return None;
2034        }
2035
2036        if self.deletion_retries >= Self::MAX_DELETION_RETRIES {
2037            Some(ServerBackupDeletionStatus::Failed)
2038        } else {
2039            Some(ServerBackupDeletionStatus::Deleting)
2040        }
2041    }
2042
2043    pub async fn dispatch_deletion(
2044        &self,
2045        state: &crate::State,
2046        options: &DeleteServerBackupOptions,
2047    ) -> Result<bool, anyhow::Error> {
2048        let node = self.node.fetch_cached(&state.database).await?;
2049
2050        let backup_configuration = match &self.backup_configuration {
2051            Some(backup_configuration) => {
2052                Some(backup_configuration.fetch_cached(&state.database).await?)
2053            }
2054            None if options.force => None,
2055            None => {
2056                return Err(crate::response::DisplayError::new(
2057                    "no backup configuration available, unable to delete backup",
2058                )
2059                .with_status(StatusCode::EXPECTATION_FAILED)
2060                .into());
2061            }
2062        };
2063
2064        if let Some(backup_configuration) = &backup_configuration
2065            && backup_configuration.maintenance_enabled
2066        {
2067            return Err(crate::response::DisplayError::new(
2068                "cannot delete backup while backup configuration is in maintenance mode",
2069            )
2070            .with_status(StatusCode::EXPECTATION_FAILED)
2071            .into());
2072        }
2073
2074        if self.disk == BackupDisk::S3 {
2075            let Some(mut s3_configuration) =
2076                backup_configuration.and_then(|c| c.backup_configs.s3.clone())
2077            else {
2078                if options.force {
2079                    tracing::warn!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "S3 backup deletion attempted but no S3 configuration found, ignoring");
2080
2081                    return Ok(true);
2082                }
2083
2084                return Err(anyhow::anyhow!(
2085                    "s3 backup deletion attempted but no S3 configuration found"
2086                ));
2087            };
2088
2089            s3_configuration.decrypt(&state.database).await?;
2090
2091            let compression_type = s3_configuration.compression_type;
2092            let (client, bucket) = s3_configuration.into_client();
2093
2094            let file_path = match &self.upload_path {
2095                Some(path) => path,
2096                None => {
2097                    if let Some(server) = &self.server {
2098                        &Self::s3_path(server.uuid, self.uuid, compression_type)
2099                    } else {
2100                        return Err(anyhow::anyhow!("backup upload path not found"));
2101                    }
2102                }
2103            };
2104
2105            if let Err(err) = client
2106                .delete_object()
2107                .bucket(bucket)
2108                .key(&**file_path)
2109                .send()
2110                .await
2111            {
2112                if options.force {
2113                    tracing::error!(server = ?self.server.as_ref().map(|s| s.uuid), backup = %self.uuid, "failed to delete S3 backup, ignoring: {:?}", err);
2114                } else {
2115                    return Err(err.into());
2116                }
2117            }
2118
2119            return Ok(true);
2120        }
2121
2122        match node
2123            .api_client(&state.database)
2124            .await?
2125            .delete_backups_backup(
2126                self.uuid,
2127                &wings_api::backups_backup::delete::RequestBody {
2128                    adapter: self.disk.to_wings_adapter(),
2129                    foreground: false,
2130                    server: self.server.as_ref().map(|s| s.uuid),
2131                },
2132            )
2133            .await
2134        {
2135            Ok(_) => Ok(false),
2136            Err(wings_api::client::ApiHttpError::Http(StatusCode::NOT_FOUND, _)) => Ok(true),
2137            Err(err) if options.force => {
2138                tracing::error!(node = %node.uuid, backup = %self.uuid, "unable to delete backup on node, finalizing anyway: {:?}", err);
2139
2140                Ok(true)
2141            }
2142            Err(err) => Err(err.into()),
2143        }
2144    }
2145
2146    pub async fn finish_deletion(
2147        &self,
2148        state: &crate::State,
2149        options: &DeleteServerBackupOptions,
2150    ) -> Result<(), anyhow::Error> {
2151        let mut transaction = state.database.write().begin().await?;
2152
2153        let finalized = sqlx::query(
2154            r#"
2155            UPDATE server_backups
2156            SET deleted = NOW(), deleting = NULL
2157            WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2158            "#,
2159        )
2160        .bind(self.uuid)
2161        .execute(&mut *transaction)
2162        .await?
2163        .rows_affected();
2164
2165        if finalized == 0 {
2166            return Ok(());
2167        }
2168
2169        self.run_after_delete_handlers(options, state, &mut transaction)
2170            .await?;
2171
2172        transaction.commit().await?;
2173
2174        Self::get_event_emitter().emit(
2175            state.clone(),
2176            ServerBackupEvent::DeletionCompleted {
2177                backup: Box::new(self.clone()),
2178                successful: true,
2179            },
2180        );
2181
2182        Ok(())
2183    }
2184
2185    pub async fn fail_deletion_attempt(&self, state: &crate::State) -> Result<i32, anyhow::Error> {
2186        let deletion_retries: Option<i32> = sqlx::query_scalar(
2187            r#"
2188            UPDATE server_backups
2189            SET deletion_retries = deletion_retries + 1
2190            WHERE
2191                server_backups.uuid = $1
2192                AND server_backups.deleted IS NULL
2193                AND server_backups.deleting IS NOT NULL
2194            RETURNING server_backups.deletion_retries
2195            "#,
2196        )
2197        .bind(self.uuid)
2198        .fetch_optional(state.database.write())
2199        .await?;
2200
2201        let Some(deletion_retries) = deletion_retries else {
2202            return Ok(0);
2203        };
2204
2205        if deletion_retries >= Self::MAX_DELETION_RETRIES {
2206            if let Some(server) = &self.server
2207                && let Err(err) = super::server_activity::ServerActivity::create(
2208                    state,
2209                    super::server_activity::CreateServerActivityOptions {
2210                        server_uuid: server.uuid,
2211                        user_uuid: None,
2212                        impersonator_uuid: None,
2213                        api_key_uuid: None,
2214                        schedule_uuid: None,
2215                        event: "server:backup.delete-failed".into(),
2216                        ip: None,
2217                        data: serde_json::json!({
2218                            "uuid": self.uuid,
2219                            "name": self.name,
2220                        }),
2221                        created: None,
2222                    },
2223                )
2224                .await
2225            {
2226                tracing::warn!(
2227                    backup = %self.uuid,
2228                    "failed to log backup deletion failure activity: {:#?}",
2229                    err
2230                );
2231            }
2232
2233            Self::get_event_emitter().emit(
2234                state.clone(),
2235                ServerBackupEvent::DeletionCompleted {
2236                    backup: Box::new(self.clone()),
2237                    successful: false,
2238                },
2239            );
2240        }
2241
2242        Ok(deletion_retries)
2243    }
2244
2245    pub async fn redispatch_stale_deletions(state: &crate::State) -> Result<u64, anyhow::Error> {
2246        let rows = sqlx::query(sqlx::AssertSqlSafe(format!(
2247            r#"
2248            SELECT {}
2249            FROM server_backups
2250            WHERE
2251                server_backups.deleted IS NULL
2252                AND server_backups.deleting IS NOT NULL
2253                AND server_backups.deletion_retries < $1
2254                AND server_backups.deleting < NOW() - make_interval(mins => LEAST(60.0, 5.0 * POWER(2.0, server_backups.deletion_retries))::int)
2255            ORDER BY server_backups.deleting
2256            LIMIT 32
2257            "#,
2258            Self::columns_sql(None)
2259        )))
2260        .bind(Self::MAX_DELETION_RETRIES)
2261        .fetch_all(state.database.read())
2262        .await?;
2263
2264        let mut redispatched = 0;
2265        for row in rows {
2266            let backup = Self::map(None, &row)?;
2267
2268            if let Some(backup_configuration) = &backup.backup_configuration
2269                && let Ok(backup_configuration) =
2270                    backup_configuration.fetch_cached(&state.database).await
2271                && backup_configuration.maintenance_enabled
2272            {
2273                sqlx::query(
2274                    "UPDATE server_backups
2275                    SET deleting = NOW()
2276                    WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL",
2277                )
2278                .bind(backup.uuid)
2279                .execute(state.database.write())
2280                .await?;
2281
2282                continue;
2283            }
2284
2285            let deletion_retries: i32 = sqlx::query_scalar(
2286                r#"
2287                UPDATE server_backups
2288                SET deleting = NOW(), deletion_retries = deletion_retries + 1
2289                WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2290                RETURNING server_backups.deletion_retries
2291                "#,
2292            )
2293            .bind(backup.uuid)
2294            .fetch_one(state.database.write())
2295            .await?;
2296
2297            match backup
2298                .dispatch_deletion(state, &DeleteServerBackupOptions::default())
2299                .await
2300            {
2301                Ok(true) => {
2302                    backup
2303                        .finish_deletion(state, &DeleteServerBackupOptions::default())
2304                        .await?;
2305                }
2306                Ok(false) => {}
2307                Err(err) => {
2308                    tracing::error!(
2309                        backup = %backup.uuid,
2310                        deletion_retries,
2311                        "failed to redispatch backup deletion: {:#?}",
2312                        err
2313                    );
2314
2315                    if deletion_retries >= Self::MAX_DELETION_RETRIES {
2316                        Self::get_event_emitter().emit(
2317                            state.clone(),
2318                            ServerBackupEvent::DeletionCompleted {
2319                                backup: Box::new(backup.clone()),
2320                                successful: false,
2321                            },
2322                        );
2323                    }
2324
2325                    continue;
2326                }
2327            }
2328
2329            redispatched += 1;
2330        }
2331
2332        Ok(redispatched)
2333    }
2334}
2335
2336#[async_trait::async_trait]
2337impl DeletableModel for ServerBackup {
2338    type DeleteOptions = DeleteServerBackupOptions;
2339
2340    fn get_delete_handlers() -> &'static LazyLock<DeleteHandlerList<Self>> {
2341        static DELETE_LISTENERS: LazyLock<DeleteHandlerList<ServerBackup>> =
2342            LazyLock::new(|| Arc::new(ModelHandlerList::default()));
2343
2344        &DELETE_LISTENERS
2345    }
2346
2347    async fn delete_with_transaction(
2348        &self,
2349        _state: &crate::State,
2350        _options: Self::DeleteOptions,
2351        _transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
2352    ) -> Result<(), anyhow::Error> {
2353        Err(anyhow::anyhow!(
2354            "delete_with_transaction is not supported for ServerBackup"
2355        ))
2356    }
2357
2358    async fn delete(
2359        &self,
2360        state: &crate::State,
2361        options: Self::DeleteOptions,
2362    ) -> Result<(), anyhow::Error> {
2363        if let Some(backup_configuration) = &self.backup_configuration
2364            && backup_configuration
2365                .fetch_cached(&state.database)
2366                .await?
2367                .maintenance_enabled
2368        {
2369            return Err(crate::response::DisplayError::new(
2370                "cannot delete backup while backup configuration is in maintenance mode",
2371            )
2372            .with_status(StatusCode::EXPECTATION_FAILED)
2373            .into());
2374        }
2375
2376        let mut transaction = state.database.write().begin().await?;
2377
2378        self.run_delete_handlers(&options, state, &mut transaction)
2379            .await?;
2380
2381        let claimed = sqlx::query(
2382            r#"
2383            UPDATE server_backups
2384            SET deleting = NOW(), deletion_retries = 0
2385            WHERE
2386                server_backups.uuid = $1
2387                AND server_backups.deleted IS NULL
2388                AND (server_backups.deleting IS NULL OR server_backups.deletion_retries >= $2)
2389            "#,
2390        )
2391        .bind(self.uuid)
2392        .bind(Self::MAX_DELETION_RETRIES)
2393        .execute(&mut *transaction)
2394        .await?
2395        .rows_affected();
2396
2397        if claimed == 0 {
2398            return Err(
2399                crate::response::DisplayError::new("backup is already being deleted")
2400                    .with_status(StatusCode::EXPECTATION_FAILED)
2401                    .into(),
2402            );
2403        }
2404
2405        transaction.commit().await?;
2406
2407        match self.dispatch_deletion(state, &options).await {
2408            Ok(true) => self.finish_deletion(state, &options).await,
2409            Ok(false) => Ok(()),
2410            Err(err) => {
2411                sqlx::query(
2412                    r#"
2413                    UPDATE server_backups
2414                    SET deleting = NULL, deletion_retries = 0
2415                    WHERE server_backups.uuid = $1 AND server_backups.deleted IS NULL
2416                    "#,
2417                )
2418                .bind(self.uuid)
2419                .execute(state.database.write())
2420                .await?;
2421
2422                Err(err)
2423            }
2424        }
2425    }
2426}
2427
2428#[derive(ToSchema, Serialize)]
2429#[schema(title = "AdminNodeServerBackup")]
2430pub struct AdminApiNodeServerBackup {
2431    pub uuid: uuid::Uuid,
2432    pub server: Option<super::server::AdminApiServer>,
2433    pub node: super::node::AdminApiNode,
2434    pub backup_group_uuid: Option<uuid::Uuid>,
2435
2436    pub name: compact_str::CompactString,
2437    pub ignored_files: Vec<compact_str::CompactString>,
2438
2439    pub is_successful: bool,
2440    pub is_locked: bool,
2441    pub is_browsable: bool,
2442    pub is_streaming: bool,
2443    pub is_shared: bool,
2444
2445    pub checksum: Option<compact_str::CompactString>,
2446    pub bytes: i64,
2447    pub files: i64,
2448
2449    pub metadata: serde_json::Value,
2450    pub deletion_status: Option<ServerBackupDeletionStatus>,
2451
2452    pub completed: Option<chrono::DateTime<chrono::Utc>>,
2453    pub created: chrono::DateTime<chrono::Utc>,
2454}
2455
2456#[schema_extension_derive::extendible]
2457#[init_args(ServerBackup, crate::State)]
2458#[hook_args(crate::State)]
2459#[derive(ToSchema, Serialize)]
2460#[schema(title = "AdminServerBackup")]
2461pub struct AdminApiServerBackup {
2462    pub uuid: uuid::Uuid,
2463    pub server: Option<super::server::AdminApiServer>,
2464    pub backup_group_uuid: Option<uuid::Uuid>,
2465
2466    pub name: compact_str::CompactString,
2467    pub ignored_files: Vec<compact_str::CompactString>,
2468
2469    pub is_successful: bool,
2470    pub is_locked: bool,
2471    pub is_browsable: bool,
2472    pub is_streaming: bool,
2473    pub is_shared: bool,
2474
2475    pub checksum: Option<compact_str::CompactString>,
2476    pub bytes: i64,
2477    pub files: i64,
2478
2479    pub metadata: serde_json::Value,
2480    pub deletion_status: Option<ServerBackupDeletionStatus>,
2481
2482    pub completed: Option<chrono::DateTime<chrono::Utc>>,
2483    pub created: chrono::DateTime<chrono::Utc>,
2484}
2485
2486#[schema_extension_derive::extendible]
2487#[init_args(ServerBackup, crate::State)]
2488#[hook_args(crate::State)]
2489#[derive(ToSchema, Serialize)]
2490#[schema(title = "ServerBackup")]
2491pub struct ApiServerBackup {
2492    pub uuid: uuid::Uuid,
2493    pub backup_group_uuid: Option<uuid::Uuid>,
2494
2495    pub name: compact_str::CompactString,
2496    pub ignored_files: Vec<compact_str::CompactString>,
2497
2498    pub is_successful: bool,
2499    pub is_locked: bool,
2500    pub is_browsable: bool,
2501    pub is_streaming: bool,
2502
2503    pub checksum: Option<compact_str::CompactString>,
2504    pub bytes: i64,
2505    pub files: i64,
2506
2507    pub metadata: serde_json::Value,
2508    pub deletion_status: Option<ServerBackupDeletionStatus>,
2509
2510    pub completed: Option<chrono::DateTime<chrono::Utc>>,
2511    pub created: chrono::DateTime<chrono::Utc>,
2512}