From 91d12ab9a7a1538c091bfabb5f80dd4864f045c7 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 26 Apr 2020 16:43:18 -0700 Subject: [PATCH] More abstract support for backups & misc code cleanup in that area --- router/router_server_backup.go | 32 ++++++++------- server/backup.go | 71 +++++++++++++++++++++------------- server/backup/backup.go | 25 +++++++++++- server/backup/backup_local.go | 4 ++ server/backup/backup_s3.go | 14 +++++++ 5 files changed, 106 insertions(+), 40 deletions(-) diff --git a/router/router_server_backup.go b/router/router_server_backup.go index a405042..ffd4977 100644 --- a/router/router_server_backup.go +++ b/router/router_server_backup.go @@ -17,25 +17,31 @@ func postServerBackup(c *gin.Context) { data := &backup.Request{} c.BindJSON(&data) + var adapter backup.Backup + var err error + switch data.Adapter { case backup.LocalBackupAdapter: - adapter, err := data.NewLocalBackup() - if err != nil { - TrackedServerError(err, s).AbortWithServerError(c) - return - } - - go func(b *backup.LocalBackup, serv *server.Server) { - if err := serv.BackupLocal(b); err != nil { - zap.S().Errorw("failed to generate backup for server", zap.Error(err)) - } - }(adapter, s) + adapter, err = data.NewLocalBackup() case backup.S3BackupAdapter: - TrackedServerError(errors.New(fmt.Sprintf("unsupported backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c) + adapter, err = data.NewS3Backup() default: - TrackedServerError(errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c) + err = errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter)) + return } + if err != nil { + TrackedServerError(err, s).AbortWithServerError(c) + return + } + + go func(b backup.Backup, serv *server.Server) { + if err := serv.Backup(b); err != nil { + zap.S().Errorw("failed to generate backup for server", zap.Error(err)) + } + }(adapter, s) + + c.Status(http.StatusAccepted) } diff --git a/server/backup.go b/server/backup.go index a3e2a28..74f0374 100644 --- a/server/backup.go +++ b/server/backup.go @@ -34,42 +34,61 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc return nil } -// Performs a server backup and then emits the event over the server websocket. We -// let the actual backup system handle notifying the panel of the status, but that -// won't emit a websocket event. -func (s *Server) BackupLocal(b *backup.LocalBackup) error { +// Get all of the ignored files for a server based on its .pteroignore file in the root. +func (s *Server) getServerwideIgnoredFiles() ([]string, error) { + var ignored []string + + f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore")) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + // Only include non-empty lines, for the sake of clarity... + if t := scanner.Text(); t != "" { + ignored = append(ignored, t) + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + } + + return ignored, nil +} + +// Get the backup files to include when generating it. +func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles, error) { // If no ignored files are present in the request, check for a .pteroignore file in the root // of the server files directory, and use that to generate the backup. - if len(b.IgnoredFiles) == 0 { - f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore")) - if err != nil { - if !os.IsNotExist(err) { - zap.S().Warnw("failed to open .pteroignore file in server directory", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err))) - } + if len(ignored) == 0 { + if i, err := s.getServerwideIgnoredFiles(); err != nil { + zap.S().Warnw("failed to retrieve server ignored files", zap.String("server", s.Uuid), zap.Error(err)) } else { - scanner := bufio.NewScanner(f) - for scanner.Scan() { - // Only include non-empty lines, for the sake of clarity... - if t := scanner.Text(); t != "" { - b.IgnoredFiles = append(b.IgnoredFiles, t) - } - } - - if err := scanner.Err(); err != nil { - zap.S().Warnw("failed to scan .pteroignore file for lines", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err))) - } + ignored = i } } // Get the included files based on the root path and the ignored files provided. - inc, err := s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), b.IgnoredFiles) + return s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), ignored) +} + +// Performs a server backup and then emits the event over the server websocket. We +// let the actual backup system handle notifying the panel of the status, but that +// won't emit a websocket event. +func (s *Server) Backup(b backup.Backup) error { + // Get the included files based on the root path and the ignored files provided. + inc, err := s.GetIncludedBackupFiles(b.Ignored()) if err != nil { return errors.WithStack(err) } if err := b.Backup(inc, s.Filesystem.Path()); err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { - zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Uuid), zap.Error(err)) + zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Identifier()), zap.Error(err)) } return errors.WithStack(err) @@ -86,11 +105,11 @@ func (s *Server) BackupLocal(b *backup.LocalBackup) error { // Emit an event over the socket so we can update the backup in realtime on // the frontend for the server. - s.Events().PublishJson(BackupCompletedEvent+":"+b.Uuid, map[string]interface{}{ - "uuid": b.Uuid, + s.Events().PublishJson(BackupCompletedEvent+":"+b.Identifier(), map[string]interface{}{ + "uuid": b.Identifier(), "sha256_hash": ad.Checksum, "file_size": ad.Size, }) return nil -} +} \ No newline at end of file diff --git a/server/backup/backup.go b/server/backup/backup.go index 5e25f3c..e8b082e 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -7,7 +7,7 @@ import ( ) const ( - LocalBackupAdapter = "local" + LocalBackupAdapter = "wings" S3BackupAdapter = "s3" ) @@ -30,6 +30,23 @@ func (r *Request) NewLocalBackup() (*LocalBackup, error) { }, nil } +// Generates a new S3 backup struct. +func (r *Request) NewS3Backup() (*S3Backup, error) { + if r.Adapter != S3BackupAdapter { + return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter)) + } + + if len(r.PresignedUrl) == 0 { + return nil, errors.New("a valid presigned S3 upload URL must be provided to use the [s3] adapter") + } + + return &S3Backup{ + Uuid: r.Uuid, + IgnoredFiles: r.IgnoredFiles, + PresignedUrl: r.PresignedUrl, + }, nil +} + type Backup interface { // Returns the UUID of this backup as tracked by the panel instance. Identifier() string @@ -38,6 +55,9 @@ type Backup interface { // implementation is. Backup(*IncludedFiles, string) error + // Returns the ignored files for this backup instance. + Ignored() []string + // Returns a SHA256 checksum for the generated backup. Checksum() ([]byte, error) @@ -51,6 +71,9 @@ type Backup interface { // Returns details about the archive. Details() *ArchiveDetails + + // Removes a backup file. + Remove() error } type ArchiveDetails struct { diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index b81efb9..f05a799 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -138,6 +138,10 @@ func (b *LocalBackup) Details() *ArchiveDetails { } } +func (b *LocalBackup) Ignored() []string { + return b.IgnoredFiles +} + // Ensures that the local backup destination for files exists. func (b *LocalBackup) ensureLocalBackupLocation() error { d := config.Get().System.BackupDirectory diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 4e78131..afb7af5 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -8,6 +8,12 @@ type S3Backup struct { // An array of files to ignore when generating this backup. This should be // compatible with a standard .gitignore structure. IgnoredFiles []string + + // The pre-signed upload endpoint for the generated backup. This must be + // provided otherwise this request will fail. This allows us to keep all + // of the keys off the daemon instances and the panel can handle generating + // the credentials for us. + PresignedUrl string } var _ Backup = (*S3Backup)(nil) @@ -35,3 +41,11 @@ func (s *S3Backup) Path() string { func (s *S3Backup) Details() *ArchiveDetails { return &ArchiveDetails{} } + +func (s *S3Backup) Ignored() []string { + return s.IgnoredFiles +} + +func (s *S3Backup) Remove() error { + return nil +}