More abstract support for backups & misc code cleanup in that area

router/router_server_backup.go

@@ -17,25 +17,31 @@ func postServerBackup(c *gin.Context) {
 	data := &backup.Request{}
 	c.BindJSON(&data)
 
+	var adapter backup.Backup
+	var err error
+
 	switch data.Adapter {
 	case backup.LocalBackupAdapter:
-		adapter, err := data.NewLocalBackup()
+		adapter, err = data.NewLocalBackup()
-		if err != nil {
-			TrackedServerError(err, s).AbortWithServerError(c)
-			return
-		}
-
-		go func(b *backup.LocalBackup, serv *server.Server) {
-			if err := serv.BackupLocal(b); err != nil {
-				zap.S().Errorw("failed to generate backup for server", zap.Error(err))
-			}
-		}(adapter, s)
 	case backup.S3BackupAdapter:
-		TrackedServerError(errors.New(fmt.Sprintf("unsupported backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c)
+		adapter, err = data.NewS3Backup()
 	default:
-		TrackedServerError(errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c)
+		err = errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter))
+		return
 	}
 
+	if err != nil {
+		TrackedServerError(err, s).AbortWithServerError(c)
+		return
+	}
+
+	go func(b backup.Backup, serv *server.Server) {
+		if err := serv.Backup(b); err != nil {
+			zap.S().Errorw("failed to generate backup for server", zap.Error(err))
+		}
+	}(adapter, s)
+
+
 	c.Status(http.StatusAccepted)
 }
 

server/backup.go

@@ -34,42 +34,61 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc
 	return nil
 }
 
-// Performs a server backup and then emits the event over the server websocket. We
+// Get all of the ignored files for a server based on its .pteroignore file in the root.
-// let the actual backup system handle notifying the panel of the status, but that
+func (s *Server) getServerwideIgnoredFiles() ([]string, error) {
-// won't emit a websocket event.
+	var ignored []string
-func (s *Server) BackupLocal(b *backup.LocalBackup) error {
+
+	f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore"))
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	} else {
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			// Only include non-empty lines, for the sake of clarity...
+			if t := scanner.Text(); t != "" {
+				ignored = append(ignored, t)
+			}
+		}
+
+		if err := scanner.Err(); err != nil {
+			return nil, err
+		}
+	}
+
+	return ignored, nil
+}
+
+// Get the backup files to include when generating it.
+func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles, error) {
 	// If no ignored files are present in the request, check for a .pteroignore file in the root
 	// of the server files directory, and use that to generate the backup.
-	if len(b.IgnoredFiles) == 0 {
+	if len(ignored) == 0 {
-		f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore"))
+		if i, err := s.getServerwideIgnoredFiles(); err != nil {
-		if err != nil {
+			zap.S().Warnw("failed to retrieve server ignored files", zap.String("server", s.Uuid), zap.Error(err))
-			if !os.IsNotExist(err) {
-				zap.S().Warnw("failed to open .pteroignore file in server directory", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err)))
-			}
 		} else {
-			scanner := bufio.NewScanner(f)
+			ignored = i
-			for scanner.Scan() {
-				// Only include non-empty lines, for the sake of clarity...
-				if t := scanner.Text(); t != "" {
-					b.IgnoredFiles = append(b.IgnoredFiles, t)
-				}
-			}
-
-			if err := scanner.Err(); err != nil {
-				zap.S().Warnw("failed to scan .pteroignore file for lines", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err)))
-			}
 		}
 	}
 
 	// Get the included files based on the root path and the ignored files provided.
-	inc, err := s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), b.IgnoredFiles)
+	return s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), ignored)
+}
+
+// Performs a server backup and then emits the event over the server websocket. We
+// let the actual backup system handle notifying the panel of the status, but that
+// won't emit a websocket event.
+func (s *Server) Backup(b backup.Backup) error {
+	// Get the included files based on the root path and the ignored files provided.
+	inc, err := s.GetIncludedBackupFiles(b.Ignored())
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
 	if err := b.Backup(inc, s.Filesystem.Path()); err != nil {
 		if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil {
-			zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Uuid), zap.Error(err))
+			zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Identifier()), zap.Error(err))
 		}
 
 		return errors.WithStack(err)
@@ -86,8 +105,8 @@ func (s *Server) BackupLocal(b *backup.LocalBackup) error {
 
 	// Emit an event over the socket so we can update the backup in realtime on
 	// the frontend for the server.
-	s.Events().PublishJson(BackupCompletedEvent+":"+b.Uuid, map[string]interface{}{
+	s.Events().PublishJson(BackupCompletedEvent+":"+b.Identifier(), map[string]interface{}{
-		"uuid":        b.Uuid,
+		"uuid":        b.Identifier(),
 		"sha256_hash": ad.Checksum,
 		"file_size":   ad.Size,
 	})

server/backup/backup.go

@@ -7,7 +7,7 @@ import (
 )
 
 const (
-	LocalBackupAdapter = "local"
+	LocalBackupAdapter = "wings"
 	S3BackupAdapter    = "s3"
 )
 
@@ -30,6 +30,23 @@ func (r *Request) NewLocalBackup() (*LocalBackup, error) {
 	}, nil
 }
 
+// Generates a new S3 backup struct.
+func (r *Request) NewS3Backup() (*S3Backup, error) {
+	if r.Adapter != S3BackupAdapter {
+		return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter))
+	}
+
+	if len(r.PresignedUrl) == 0 {
+		return nil, errors.New("a valid presigned S3 upload URL must be provided to use the [s3] adapter")
+	}
+
+	return &S3Backup{
+		Uuid:         r.Uuid,
+		IgnoredFiles: r.IgnoredFiles,
+		PresignedUrl: r.PresignedUrl,
+	}, nil
+}
+
 type Backup interface {
 	// Returns the UUID of this backup as tracked by the panel instance.
 	Identifier() string
@@ -38,6 +55,9 @@ type Backup interface {
 	// implementation is.
 	Backup(*IncludedFiles, string) error
 
+	// Returns the ignored files for this backup instance.
+	Ignored() []string
+
 	// Returns a SHA256 checksum for the generated backup.
 	Checksum() ([]byte, error)
 
@@ -51,6 +71,9 @@ type Backup interface {
 
 	// Returns details about the archive.
 	Details() *ArchiveDetails
+
+	// Removes a backup file.
+	Remove() error
 }
 
 type ArchiveDetails struct {

server/backup/backup_local.go

@@ -138,6 +138,10 @@ func (b *LocalBackup) Details() *ArchiveDetails {
 	}
 }
 
+func (b *LocalBackup) Ignored() []string {
+	return b.IgnoredFiles
+}
+
 // Ensures that the local backup destination for files exists.
 func (b *LocalBackup) ensureLocalBackupLocation() error {
 	d := config.Get().System.BackupDirectory

server/backup/backup_s3.go

@@ -8,6 +8,12 @@ type S3Backup struct {
 	// An array of files to ignore when generating this backup. This should be
 	// compatible with a standard .gitignore structure.
 	IgnoredFiles []string
+
+	// The pre-signed upload endpoint for the generated backup. This must be
+	// provided otherwise this request will fail. This allows us to keep all
+	// of the keys off the daemon instances and the panel can handle generating
+	// the credentials for us.
+	PresignedUrl string
 }
 
 var _ Backup = (*S3Backup)(nil)
@@ -35,3 +41,11 @@ func (s *S3Backup) Path() string {
 func (s *S3Backup) Details() *ArchiveDetails {
 	return &ArchiveDetails{}
 }
+
+func (s *S3Backup) Ignored() []string {
+	return s.IgnoredFiles
+}
+
+func (s *S3Backup) Remove() error {
+	return nil
+}