More abstract support for backups & misc code cleanup in that area

This commit is contained in:
Dane Everitt 2020-04-26 16:43:18 -07:00
parent 1e2da95d26
commit 91d12ab9a7
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
5 changed files with 106 additions and 40 deletions

View File

@ -17,25 +17,31 @@ func postServerBackup(c *gin.Context) {
data := &backup.Request{}
c.BindJSON(&data)
var adapter backup.Backup
var err error
switch data.Adapter {
case backup.LocalBackupAdapter:
adapter, err := data.NewLocalBackup()
if err != nil {
TrackedServerError(err, s).AbortWithServerError(c)
return
}
go func(b *backup.LocalBackup, serv *server.Server) {
if err := serv.BackupLocal(b); err != nil {
zap.S().Errorw("failed to generate backup for server", zap.Error(err))
}
}(adapter, s)
adapter, err = data.NewLocalBackup()
case backup.S3BackupAdapter:
TrackedServerError(errors.New(fmt.Sprintf("unsupported backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c)
adapter, err = data.NewS3Backup()
default:
TrackedServerError(errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter)), s).AbortWithServerError(c)
err = errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter))
return
}
if err != nil {
TrackedServerError(err, s).AbortWithServerError(c)
return
}
go func(b backup.Backup, serv *server.Server) {
if err := serv.Backup(b); err != nil {
zap.S().Errorw("failed to generate backup for server", zap.Error(err))
}
}(adapter, s)
c.Status(http.StatusAccepted)
}

View File

@ -34,42 +34,61 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc
return nil
}
// Performs a server backup and then emits the event over the server websocket. We
// let the actual backup system handle notifying the panel of the status, but that
// won't emit a websocket event.
func (s *Server) BackupLocal(b *backup.LocalBackup) error {
// Get all of the ignored files for a server based on its .pteroignore file in the root.
func (s *Server) getServerwideIgnoredFiles() ([]string, error) {
var ignored []string
f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore"))
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
} else {
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// Only include non-empty lines, for the sake of clarity...
if t := scanner.Text(); t != "" {
ignored = append(ignored, t)
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
}
return ignored, nil
}
// Get the backup files to include when generating it.
func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles, error) {
// If no ignored files are present in the request, check for a .pteroignore file in the root
// of the server files directory, and use that to generate the backup.
if len(b.IgnoredFiles) == 0 {
f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore"))
if err != nil {
if !os.IsNotExist(err) {
zap.S().Warnw("failed to open .pteroignore file in server directory", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err)))
}
if len(ignored) == 0 {
if i, err := s.getServerwideIgnoredFiles(); err != nil {
zap.S().Warnw("failed to retrieve server ignored files", zap.String("server", s.Uuid), zap.Error(err))
} else {
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// Only include non-empty lines, for the sake of clarity...
if t := scanner.Text(); t != "" {
b.IgnoredFiles = append(b.IgnoredFiles, t)
}
}
if err := scanner.Err(); err != nil {
zap.S().Warnw("failed to scan .pteroignore file for lines", zap.String("server", s.Uuid), zap.Error(errors.WithStack(err)))
}
ignored = i
}
}
// Get the included files based on the root path and the ignored files provided.
inc, err := s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), b.IgnoredFiles)
return s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), ignored)
}
// Performs a server backup and then emits the event over the server websocket. We
// let the actual backup system handle notifying the panel of the status, but that
// won't emit a websocket event.
func (s *Server) Backup(b backup.Backup) error {
// Get the included files based on the root path and the ignored files provided.
inc, err := s.GetIncludedBackupFiles(b.Ignored())
if err != nil {
return errors.WithStack(err)
}
if err := b.Backup(inc, s.Filesystem.Path()); err != nil {
if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil {
zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Uuid), zap.Error(err))
zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Identifier()), zap.Error(err))
}
return errors.WithStack(err)
@ -86,11 +105,11 @@ func (s *Server) BackupLocal(b *backup.LocalBackup) error {
// Emit an event over the socket so we can update the backup in realtime on
// the frontend for the server.
s.Events().PublishJson(BackupCompletedEvent+":"+b.Uuid, map[string]interface{}{
"uuid": b.Uuid,
s.Events().PublishJson(BackupCompletedEvent+":"+b.Identifier(), map[string]interface{}{
"uuid": b.Identifier(),
"sha256_hash": ad.Checksum,
"file_size": ad.Size,
})
return nil
}
}

View File

@ -7,7 +7,7 @@ import (
)
const (
LocalBackupAdapter = "local"
LocalBackupAdapter = "wings"
S3BackupAdapter = "s3"
)
@ -30,6 +30,23 @@ func (r *Request) NewLocalBackup() (*LocalBackup, error) {
}, nil
}
// Generates a new S3 backup struct.
func (r *Request) NewS3Backup() (*S3Backup, error) {
if r.Adapter != S3BackupAdapter {
return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter))
}
if len(r.PresignedUrl) == 0 {
return nil, errors.New("a valid presigned S3 upload URL must be provided to use the [s3] adapter")
}
return &S3Backup{
Uuid: r.Uuid,
IgnoredFiles: r.IgnoredFiles,
PresignedUrl: r.PresignedUrl,
}, nil
}
type Backup interface {
// Returns the UUID of this backup as tracked by the panel instance.
Identifier() string
@ -38,6 +55,9 @@ type Backup interface {
// implementation is.
Backup(*IncludedFiles, string) error
// Returns the ignored files for this backup instance.
Ignored() []string
// Returns a SHA256 checksum for the generated backup.
Checksum() ([]byte, error)
@ -51,6 +71,9 @@ type Backup interface {
// Returns details about the archive.
Details() *ArchiveDetails
// Removes a backup file.
Remove() error
}
type ArchiveDetails struct {

View File

@ -138,6 +138,10 @@ func (b *LocalBackup) Details() *ArchiveDetails {
}
}
func (b *LocalBackup) Ignored() []string {
return b.IgnoredFiles
}
// Ensures that the local backup destination for files exists.
func (b *LocalBackup) ensureLocalBackupLocation() error {
d := config.Get().System.BackupDirectory

View File

@ -8,6 +8,12 @@ type S3Backup struct {
// An array of files to ignore when generating this backup. This should be
// compatible with a standard .gitignore structure.
IgnoredFiles []string
// The pre-signed upload endpoint for the generated backup. This must be
// provided otherwise this request will fail. This allows us to keep all
// of the keys off the daemon instances and the panel can handle generating
// the credentials for us.
PresignedUrl string
}
var _ Backup = (*S3Backup)(nil)
@ -35,3 +41,11 @@ func (s *S3Backup) Path() string {
func (s *S3Backup) Details() *ArchiveDetails {
return &ArchiveDetails{}
}
func (s *S3Backup) Ignored() []string {
return s.IgnoredFiles
}
func (s *S3Backup) Remove() error {
return nil
}