From 7dd0acebc0cccd2bfb5a0525682faf09e27252c7 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 18:06:22 -0800 Subject: [PATCH] Initial untested pass at restoring from local backups --- router/router_server_backup.go | 126 ++++++++++++++++++++------------ server/backup/backup.go | 40 +++++++++- server/backup/backup_local.go | 31 ++++++-- server/backup/backup_request.go | 42 ----------- server/filesystem/decompress.go | 39 ++++++---- server/filesystem/filesystem.go | 21 +++++- 6 files changed, 183 insertions(+), 116 deletions(-) delete mode 100644 server/backup/backup_request.go diff --git a/router/router_server_backup.go b/router/router_server_backup.go index b0afd19..15be67b 100644 --- a/router/router_server_backup.go +++ b/router/router_server_backup.go @@ -1,64 +1,105 @@ package router import ( - "fmt" "net/http" "os" "emperror.dev/errors" + "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/pterodactyl/wings/router/middleware" "github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/server/backup" ) -// Backs up a server. +// postServerBackup performs a backup against a given server instance using the +// provided backup adapter. func postServerBackup(c *gin.Context) { - s := GetServer(c.Param("server")) - - data := &backup.Request{} - // BindJSON sends 400 if the request fails, all we need to do is return + s := middleware.ExtractServer(c) + logger := middleware.ExtractLogger(c) + var data backup.Request if err := c.BindJSON(&data); err != nil { return } - var adapter backup.BackupInterface - var err error - - switch data.Adapter { - case backup.LocalBackupAdapter: - adapter, err = data.NewLocalBackup() - case backup.S3BackupAdapter: - adapter, err = data.NewS3Backup() - default: - err = errors.New(fmt.Sprintf("unknown backup adapter [%s] provided", data.Adapter)) - return - } - + adapter, err := data.AsBackup() if err != nil { - NewServerError(err, s).Abort(c) + middleware.CaptureAndAbort(c, err) return } - - // Attach the server ID to the backup log output for easier parsing. + // Attach the server ID and the request ID to the adapter log context for easier + // parsing in the logs. adapter.WithLogContext(map[string]interface{}{ - "server": s.Id(), + "server": s.Id(), + "request_id": c.GetString("request_id"), }) - go func(b backup.BackupInterface, serv *server.Server) { - if err := serv.Backup(b); err != nil { - serv.Log().WithField("error", errors.WithStackIf(err)).Error("failed to generate backup for server") + go func(b backup.BackupInterface, s *server.Server, logger *log.Entry) { + if err := s.Backup(b); err != nil { + logger.WithField("error", errors.WithStackIf(err)).Error("router: failed to generate server backup") } - }(adapter, s) + }(adapter, s, logger) c.Status(http.StatusAccepted) } -// Deletes a local backup of a server. If the backup is not found on the machine just return -// a 404 error. The service calling this endpoint can make its own decisions as to how it wants -// to handle that response. -func deleteServerBackup(c *gin.Context) { - s := GetServer(c.Param("server")) +// postServerRestoreBackup handles restoring a backup for a server by downloading +// or finding the given backup on the system and then unpacking the archive into +// the server's data directory. If the TruncateDirectory field is provided and +// is true all of the files will be deleted for the server. +// +// This endpoint will block until the backup is fully restored allowing for a +// spinner to be displayed in the Panel UI effectively. +func postServerRestoreBackup(c *gin.Context) { + s := middleware.ExtractServer(c) + logger := middleware.ExtractLogger(c) + var data struct { + UUID string `binding:"required,uuid" json:"uuid"` + Adapter backup.AdapterType `binding:"required,oneof=wings s3" json:"adapter"` + TruncateDirectory bool `json:"truncate_directory"` + // A UUID is always required for this endpoint, however the download URL + // is only present when the given adapter type is s3. + DownloadUrl string `json:"download_url"` + } + if err := c.BindJSON(&data); err != nil { + return + } + if data.Adapter == backup.S3BackupAdapter && data.DownloadUrl == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "The download_url field is required when the backup adapter is set to S3."}) + return + } + + logger.Info("processing server backup restore request") + if data.TruncateDirectory { + logger.Info(`recieved "truncate_directory" flag in request: deleting server files`) + if err := s.Filesystem().TruncateRootDirectory(); err != nil { + middleware.CaptureAndAbort(c, err) + return + } + } + + // Now that we've cleaned up the data directory if necessary, grab the backup file + // and attempt to restore it into the server directory. + if data.Adapter == backup.LocalBackupAdapter { + b, _, err := backup.LocateLocal(data.UUID) + if err != nil { + middleware.CaptureAndAbort(c, err) + return + } + if err := b.Restore(s); err != nil { + middleware.CaptureAndAbort(c, err) + return + } + } + c.Status(http.StatusNoContent) +} + +// deleteServerBackup deletes a local backup of a server. If the backup is not +// found on the machine just return a 404 error. The service calling this +// endpoint can make its own decisions as to how it wants to handle that +// response. +func deleteServerBackup(c *gin.Context) { b, _, err := backup.LocateLocal(c.Param("backup")) if err != nil { // Just return from the function at this point if the backup was not located. @@ -68,20 +109,15 @@ func deleteServerBackup(c *gin.Context) { }) return } - - NewServerError(err, s).Abort(c) + middleware.CaptureAndAbort(c, err) return } - - if err := b.Remove(); err != nil { - // I'm not entirely sure how likely this is to happen, however if we did manage to locate - // the backup previously and it is now missing when we go to delete, just treat it as having - // been successful, rather than returning a 404. - if !errors.Is(err, os.ErrNotExist) { - NewServerError(err, s).Abort(c) - return - } + // I'm not entirely sure how likely this is to happen, however if we did manage to + // locate the backup previously and it is now missing when we go to delete, just + // treat it as having been successful, rather than returning a 404. + if err := b.Remove(); err != nil && !errors.Is(err, os.ErrNotExist) { + middleware.CaptureAndAbort(c, err) + return } - c.Status(http.StatusNoContent) -} +} \ No newline at end of file diff --git a/server/backup/backup.go b/server/backup/backup.go index 29e7b5d..9c50e84 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -3,13 +3,15 @@ package backup import ( "crypto/sha1" "encoding/hex" - "github.com/apex/log" - "github.com/pterodactyl/wings/api" - "github.com/pterodactyl/wings/config" "io" "os" "path" "sync" + + "emperror.dev/errors" + "github.com/apex/log" + "github.com/pterodactyl/wings/api" + "github.com/pterodactyl/wings/config" ) type AdapterType string @@ -19,6 +21,38 @@ const ( S3BackupAdapter AdapterType = "s3" ) +type Request struct { + Adapter AdapterType `json:"adapter"` + Uuid string `json:"uuid"` + Ignore string `json:"ignore"` +} + +// AsBackup returns a new backup adapter based on the request value. +func (r *Request) AsBackup() (BackupInterface, error) { + var adapter BackupInterface + switch r.Adapter { + case LocalBackupAdapter: + adapter = &LocalBackup{ + Backup{ + Uuid: r.Uuid, + Ignore: r.Ignore, + adapter: LocalBackupAdapter, + }, + } + case S3BackupAdapter: + adapter = &S3Backup{ + Backup: Backup{ + Uuid: r.Uuid, + Ignore: r.Ignore, + adapter: S3BackupAdapter, + }, + } + default: + return nil, errors.New("server/backup: unsupported adapter type: " + string(r.Adapter)) + } + return adapter, nil +} + type ArchiveDetails struct { Checksum string `json:"checksum"` ChecksumType string `json:"checksum_type"` diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index 6e07ff3..75305f9 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -2,8 +2,11 @@ package backup import ( "errors" - "github.com/pterodactyl/wings/server/filesystem" "os" + + "github.com/mholt/archiver/v3" + "github.com/pterodactyl/wings/server" + "github.com/pterodactyl/wings/server/filesystem" ) type LocalBackup struct { @@ -12,8 +15,8 @@ type LocalBackup struct { var _ BackupInterface = (*LocalBackup)(nil) -// Locates the backup for a server and returns the local path. This will obviously only -// work if the backup was created as a local backup. +// LocateLocal finds the backup for a server and returns the local path. This +// will obviously only work if the backup was created as a local backup. func LocateLocal(uuid string) (*LocalBackup, os.FileInfo, error) { b := &LocalBackup{ Backup{ @@ -34,18 +37,18 @@ func LocateLocal(uuid string) (*LocalBackup, os.FileInfo, error) { return b, st, nil } -// Removes a backup from the system. +// Remove removes a backup from the system. func (b *LocalBackup) Remove() error { return os.Remove(b.Path()) } -// Attaches additional context to the log output for this backup. +// WithLogContext attaches additional context to the log output for this backup. func (b *LocalBackup) WithLogContext(c map[string]interface{}) { b.logContext = c } -// Generates a backup of the selected files and pushes it to the defined location -// for this instance. +// Generate generates a backup of the selected files and pushes it to the +// defined location for this instance. func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) { a := &filesystem.Archive{ BasePath: basePath, @@ -60,3 +63,17 @@ func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) return b.Details(), nil } + +// Restore restores a backup to the provided server's root data directory. +func (b *LocalBackup) Restore(s *server.Server) error { + return archiver.Walk(b.Path(), func(f archiver.File) error { + if f.IsDir() { + return nil + } + name, err := filesystem.ExtractArchiveSourceName(f, "/") + if err != nil { + return err + } + return s.Filesystem().Writefile(name, f) + }) +} diff --git a/server/backup/backup_request.go b/server/backup/backup_request.go deleted file mode 100644 index 60ba4d7..0000000 --- a/server/backup/backup_request.go +++ /dev/null @@ -1,42 +0,0 @@ -package backup - -import ( - "errors" - "fmt" -) - -type Request struct { - Adapter AdapterType `json:"adapter"` - Uuid string `json:"uuid"` - Ignore string `json:"ignore"` -} - -// Generates a new local backup struct. -func (r *Request) NewLocalBackup() (*LocalBackup, error) { - if r.Adapter != LocalBackupAdapter { - return nil, errors.New(fmt.Sprintf("cannot create local backup using [%s] adapter", r.Adapter)) - } - - return &LocalBackup{ - Backup{ - Uuid: r.Uuid, - Ignore: r.Ignore, - adapter: LocalBackupAdapter, - }, - }, nil -} - -// Generates a new S3 backup struct. -func (r *Request) NewS3Backup() (*S3Backup, error) { - if r.Adapter != S3BackupAdapter { - return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter)) - } - - return &S3Backup{ - Backup: Backup{ - Uuid: r.Uuid, - Ignore: r.Ignore, - adapter: S3BackupAdapter, - }, - }, nil -} diff --git a/server/filesystem/decompress.go b/server/filesystem/decompress.go index 51c3bff..b7f18f2 100644 --- a/server/filesystem/decompress.go +++ b/server/filesystem/decompress.go @@ -74,23 +74,10 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { if f.IsDir() { return nil } - - var name string - switch s := f.Sys().(type) { - case *tar.Header: - name = s.Name - case *gzip.Header: - name = s.Name - case *zip.FileHeader: - name = s.Name - default: - return &Error{ - code: ErrCodeUnknownError, - resolved: filepath.Join(dir, f.Name()), - err: errors.New(fmt.Sprintf("could not parse underlying data source with type: %s", reflect.TypeOf(s).String())), - } + name, err := ExtractArchiveSourceName(f, dir) + if err != nil { + return err } - p := filepath.Join(dir, name) // If it is ignored, just don't do anything with the file and skip over it. if err := fs.IsIgnored(p); err != nil { @@ -109,3 +96,23 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { } return nil } + +// ExtractArchiveSourceName looks for the provided archiver.File's name if it is +// a type that is supported, otherwise it returns an error to the caller. +func ExtractArchiveSourceName(f archiver.File, dir string) (name string, err error) { + switch s := f.Sys().(type) { + case *tar.Header: + name = s.Name + case *gzip.Header: + name = s.Name + case *zip.FileHeader: + name = s.Name + default: + err = &Error{ + code: ErrCodeUnknownError, + resolved: filepath.Join(dir, f.Name()), + err: errors.New(fmt.Sprintf("could not parse underlying data source with type: %s", reflect.TypeOf(s).String())), + } + } + return name, err +} diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 52da443..7e4da94 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "emperror.dev/errors" @@ -124,7 +125,8 @@ func (fs *Filesystem) Readfile(p string, w io.Writer) error { } // Writefile writes a file to the system. If the file does not already exist one -// will be created. +// will be created. This will also properly recalculate the disk space used by +// the server when writing new files or modifying existing ones. func (fs *Filesystem) Writefile(p string, r io.Reader) error { cleaned, err := fs.SafePath(p) if err != nil { @@ -365,8 +367,21 @@ func (fs *Filesystem) Copy(p string) error { return fs.Writefile(path.Join(relative, n), source) } -// Deletes a file or folder from the system. Prevents the user from accidentally -// (or maliciously) removing their root server data directory. +// TruncateRootDirectory removes _all_ files and directories from a server's +// data directory and resets the used disk space to zero. +func (fs *Filesystem) TruncateRootDirectory() error { + if err := os.RemoveAll(fs.Path()); err != nil { + return err + } + if err := os.Mkdir(fs.Path(), 0755); err != nil { + return err + } + atomic.StoreInt64(&fs.diskUsed, 0) + return nil +} + +// Delete removes a file or folder from the system. Prevents the user from +// accidentally (or maliciously) removing their root server data directory. func (fs *Filesystem) Delete(p string) error { wg := sync.WaitGroup{} // This is one of the few (only?) places in the codebase where we're explicitly not using