MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break out the backup functions of the daemon in prep for S3 support

Browse Source
This commit is contained in:
Dane Everitt 2020-04-13 22:01:07 -07:00
parent fd9487ea4d
commit 11035b561a
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
7 changed files with 260 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
api/backup_endpoints.go Normal file
View File

@ -0,0 +1,35 @@
package api
import (
"encoding/json"
"fmt"
"github.com/pkg/errors"
)
type BackupRequest struct {
Checksum string `json:"checksum"`
Size int64 `json:"size"`
Successful bool `json:"successful"`
}
// Notifies the panel that a specific backup has been completed and is now
// available for a user to view and download.
func (r *PanelRequest) SendBackupStatus(backup string, data BackupRequest) (*RequestError, error) {
b, err := json.Marshal(data)
if err != nil {
return nil, errors.WithStack(err)
}
resp, err := r.Post(fmt.Sprintf("/backups/%s", backup), b)
if err != nil {
return nil, errors.WithStack(err)
}
defer resp.Body.Close()
r.Response = resp
if r.HasError() {
return r.Error(), nil
}
return nil, nil
}

28
api/server_endpoints.go
View File

@ -202,30 +202,4 @@ func (r *PanelRequest) SendTransferSuccess(uuid string) (*RequestError, error) {
}
return nil, nil
}
type BackupRequest struct {
Successful bool `json:"successful"`
Sha256Hash string `json:"sha256_hash"`
FileSize int64 `json:"file_size"`
}
func (r *PanelRequest) SendBackupStatus(uuid string, backup string, data BackupRequest) (*RequestError, error) {
b, err := json.Marshal(data)
if err != nil {
return nil, errors.WithStack(err)
}
resp, err := r.Post(fmt.Sprintf("/servers/%s/backup/%s", uuid, backup), b)
if err != nil {
return nil, errors.WithStack(err)
}
defer resp.Body.Close()
r.Response = resp
if r.HasError() {
return r.Error(), nil
}
return nil, nil
}
}

12
router/router_download.go
View File

@ -4,6 +4,7 @@ import (
"bufio"
"github.com/gin-gonic/gin"
"github.com/pterodactyl/wings/router/tokens"
"github.com/pterodactyl/wings/server/backup"
"net/http"
"os"
"strconv"
@ -25,13 +26,20 @@ func getDownloadBackup(c *gin.Context) {
return
}
p, st, err := s.LocateBackup(token.BackupUuid)
b, st, err := backup.LocateLocal(token.BackupUuid)
if err != nil {
if os.IsNotExist(err) {
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
"error": "The requested backup was not found on this server.",
})
return
}
TrackedServerError(err, s).AbortWithServerError(c)
return
}
f, err := os.Open(p)
f, err := os.Open(b.Path())
if err != nil {
TrackedServerError(err, s).AbortWithServerError(c)
return

17
router/router_server_backup.go
View File

@ -3,26 +3,23 @@ package router
import (
"github.com/gin-gonic/gin"
"github.com/pterodactyl/wings/server"
"github.com/pterodactyl/wings/server/backup"
"go.uber.org/zap"
"net/http"
"os"
)
// Backs up a server.
func postServerBackup(c *gin.Context) {
s := GetServer(c.Param("server"))
var data struct{
Uuid string `json:"uuid"`
IgnoredFiles []string `json:"ignored_files"`
}
data := &backup.Backup{}
c.BindJSON(&data)
go func(backup *server.Backup) {
if err := backup.BackupAndNotify(); err != nil {
go func(b *backup.Backup, serv *server.Server) {
if err := serv.BackupRoot(b); err != nil {
zap.S().Errorw("failed to generate backup for server", zap.Error(err))
}
}(s.NewBackup(data.Uuid, data.IgnoredFiles))
}(data, s)
c.Status(http.StatusAccepted)
}
@ -31,13 +28,13 @@ func postServerBackup(c *gin.Context) {
func deleteServerBackup(c *gin.Context) {
s := GetServer(c.Param("server"))
p, _, err := s.LocateBackup(c.Param("backup"))
b, _, err := backup.LocateLocal(c.Param("backup"))
if err != nil {
TrackedServerError(err, s).AbortWithServerError(c)
return
}
if err := os.Remove(p); err != nil {
if err := b.Remove(); err != nil {
TrackedServerError(err, s).AbortWithServerError(c)
return
}

205
server/backup.go
View File

@ -1,210 +1,39 @@
package server
import (
"crypto/sha256"
"encoding/hex"
"github.com/mholt/archiver/v3"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/server/backup"
"go.uber.org/zap"
"io"
"os"
"path"
"strings"
"sync"
)
type Backup struct {
Uuid string `json:"uuid"`
IgnoredFiles []string `json:"ignored_files"`
server *Server
localDirectory string
}
// Create a new Backup struct from data passed through in a request.
func (s *Server) NewBackup(uuid string, ignore []string) *Backup {
return &Backup{
Uuid: uuid,
IgnoredFiles: ignore,
server: s,
localDirectory: path.Join(config.Get().System.BackupDirectory, s.Uuid),
}
}
// Locates the backup for a server and returns the local path. This will obviously only
// work if the backup was created as a local backup.
func (s *Server) LocateBackup(uuid string) (string, os.FileInfo, error) {
p := path.Join(config.Get().System.BackupDirectory, s.Uuid, uuid+".tar.gz")
st, err := os.Stat(p)
// Performs a server backup and then emits the event over the server websocket. We
// let the actual backup system handle notifying the panel of the status, but that
// won't emit a websocket event.
func (s *Server) BackupRoot(b *backup.Backup) error {
r, err := b.LocalBackup(s.Filesystem.Path())
if err != nil {
return "", nil, err
}
if st.IsDir() {
return "", nil, errors.New("invalid archive found; is directory")
}
return p, st, nil
}
// Ensures that the local backup destination for files exists.
func (b *Backup) ensureLocalBackupLocation() error {
if _, err := os.Stat(b.localDirectory); err != nil {
if !os.IsNotExist(err) {
return errors.WithStack(err)
if notifyError := b.NotifyPanel(r, false); notifyError != nil {
zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Uuid), zap.Error(err))
}
return os.MkdirAll(b.localDirectory, 0700)
}
return nil
}
// Returns the path for this specific backup.
func (b *Backup) GetPath() string {
return path.Join(b.localDirectory, b.Uuid+".tar.gz")
}
func (b *Backup) GetChecksum() ([]byte, error) {
h := sha256.New()
f, err := os.Open(b.GetPath())
if err != nil {
return []byte{}, errors.WithStack(err)
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
return []byte{}, errors.WithStack(err)
}
return h.Sum(nil), nil
}
// Generates a backup of the selected files and pushes it to the defined location
// for this instance.
func (b *Backup) Backup() (*api.BackupRequest, error) {
rootPath := b.server.Filesystem.Path()
if err := b.ensureLocalBackupLocation(); err != nil {
return nil, errors.WithStack(err)
}
zap.S().Debugw("starting archive of server files for backup", zap.String("server", b.server.Uuid), zap.String("backup", b.Uuid))
if err := archiver.Archive([]string{rootPath}, b.GetPath()); err != nil {
if strings.HasPrefix(err.Error(), "file already exists") {
zap.S().Debugw("backup already exists on system, removing and re-attempting", zap.String("backup", b.Uuid))
if rerr := os.Remove(b.GetPath()); rerr != nil {
return nil, errors.WithStack(rerr)
}
// Re-attempt this backup.
return b.Backup()
}
// If there was some error with the archive, just go ahead and ensure the backup
// is completely destroyed at this point. Ignore any errors from this function.
os.Remove(b.GetPath())
return nil, err
}
wg := sync.WaitGroup{}
wg.Add(2)
var checksum string
// Calculate the checksum for the file.
go func() {
defer wg.Done()
resp, err := b.GetChecksum()
if err != nil {
zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err))
}
checksum = hex.EncodeToString(resp)
}()
var s int64
go func() {
defer wg.Done()
st, err := os.Stat(b.GetPath())
if err != nil {
return
}
s = st.Size()
}()
wg.Wait()
return &api.BackupRequest{
Successful: true,
Sha256Hash: checksum,
FileSize: s,
}, nil
}
// Performs a server backup and then notifies the Panel of the completed status
// so that the backup shows up for the user correctly.
func (b *Backup) BackupAndNotify() error {
resp, err := b.Backup()
if err != nil {
b.notifyPanel(resp)
return errors.WithStack(err)
}
if err := b.notifyPanel(resp); err != nil {
// These errors indicate that the Panel will not know about the status of this
// backup, so let's just go ahead and delete it, and let the Panel handle the
// cleanup process for the backups.
//
// @todo perhaps in the future we can sync the backups from the servers on boot?
os.Remove(b.GetPath())
// Try to notify the panel about the status of this backup. If for some reason this request
// fails, delete the archive from the daemon and return that error up the chain to the caller.
if notifyError := b.NotifyPanel(r, true); notifyError != nil {
b.Remove()
return err
return notifyError
}
// Emit an event over the socket so we can update the backup in realtime on
// the frontend for the server.
b.server.Events().PublishJson(BackupCompletedEvent+":"+b.Uuid, map[string]interface{}{
s.Events().PublishJson(BackupCompletedEvent+":"+b.Uuid, map[string]interface{}{
"uuid": b.Uuid,
"sha256_hash": resp.Sha256Hash,
"file_size": resp.FileSize,
"sha256_hash": r.Checksum,
"file_size": r.Size,
})
return nil
}
func (b *Backup) notifyPanel(request *api.BackupRequest) error {
r := api.NewRequester()
rerr, err := r.SendBackupStatus(b.server.Uuid, b.Uuid, *request)
if rerr != nil || err != nil {
if err != nil {
zap.S().Errorw(
"failed to notify panel of backup status due to internal code error",
zap.String("server", b.server.Uuid),
zap.String("backup", b.Uuid),
zap.Error(err),
)
return err
}
zap.S().Warnw(
rerr.String(),
zap.String("server", b.server.Uuid),
zap.String("backup", b.Uuid),
)
return errors.New(rerr.String())
}
return nil
}
}

103
server/backup/backup.go Normal file
View File

@ -0,0 +1,103 @@
package backup
import (
"crypto/sha256"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/config"
"go.uber.org/zap"
"io"
"os"
"path"
)
type Backup struct {
// The UUID of this backup object. This must line up with a backup from
// the panel instance.
Uuid string `json:"uuid"`
// An array of files to ignore when generating this backup. This should be
// compatible with a standard .gitignore structure.
IgnoredFiles []string `json:"ignored_files"`
}
type ArchiveDetails struct {
Checksum string `json:"checksum"`
Size int64 `json:"size"`
}
// Returns a request object.
func (ad *ArchiveDetails) ToRequest(successful bool) api.BackupRequest {
return api.BackupRequest{
Checksum: ad.Checksum,
Size: ad.Size,
Successful: successful,
}
}
// Returns the path for this specific backup.
func (b *Backup) Path() string {
return path.Join(config.Get().System.BackupDirectory, b.Uuid+".tar.gz")
}
// Returns the SHA256 checksum of a backup.
func (b *Backup) Checksum() ([]byte, error) {
h := sha256.New()
f, err := os.Open(b.Path())
if err != nil {
return []byte{}, errors.WithStack(err)
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
return []byte{}, errors.WithStack(err)
}
return h.Sum(nil), nil
}
// Removes a backup from the system.
func (b *Backup) Remove() error {
return os.Remove(b.Path())
}
// Notifies the panel of a backup's state and returns an error if one is encountered
// while performing this action.
func (b *Backup) NotifyPanel(ad *ArchiveDetails, successful bool) error {
r := api.NewRequester()
rerr, err := r.SendBackupStatus(b.Uuid, ad.ToRequest(successful))
if rerr != nil || err != nil {
if err != nil {
zap.S().Errorw(
"failed to notify panel of backup status due to internal code error",
zap.String("backup", b.Uuid),
zap.Error(err),
)
return err
}
zap.S().Warnw(rerr.String(), zap.String("backup", b.Uuid))
return errors.New(rerr.String())
}
return nil
}
// Ensures that the local backup destination for files exists.
func (b *Backup) ensureLocalBackupLocation() error {
d := config.Get().System.BackupDirectory
if _, err := os.Stat(d); err != nil {
if !os.IsNotExist(err) {
return errors.WithStack(err)
}
return os.MkdirAll(d, 0700)
}
return nil
}

87
server/backup/backup_local.go Normal file
View File

@ -0,0 +1,87 @@
package backup
import (
"encoding/hex"
"github.com/mholt/archiver/v3"
"github.com/pkg/errors"
"go.uber.org/zap"
"os"
"strings"
"sync"
)
// Locates the backup for a server and returns the local path. This will obviously only
// work if the backup was created as a local backup.
func LocateLocal(uuid string) (*Backup, os.FileInfo, error) {
b := &Backup{
Uuid: uuid,
IgnoredFiles: nil,
}
st, err := os.Stat(b.Path())
if err != nil {
return nil, nil, err
}
if st.IsDir() {
return nil, nil, errors.New("invalid archive found; is directory")
}
return b, st, nil
}
// Generates a backup of the selected files and pushes it to the defined location
// for this instance.
func (b *Backup) LocalBackup(dir string) (*ArchiveDetails, error) {
if err := archiver.Archive([]string{dir}, b.Path()); err != nil {
if strings.HasPrefix(err.Error(), "file already exists") {
if rerr := os.Remove(b.Path()); rerr != nil {
return nil, errors.WithStack(rerr)
}
// Re-attempt this backup by calling it with the same information.
return b.LocalBackup(dir)
}
// If there was some error with the archive, just go ahead and ensure the backup
// is completely destroyed at this point. Ignore any errors from this function.
os.Remove(b.Path())
return nil, err
}
wg := sync.WaitGroup{}
wg.Add(2)
var checksum string
// Calculate the checksum for the file.
go func() {
defer wg.Done()
resp, err := b.Checksum()
if err != nil {
zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err))
}
checksum = hex.EncodeToString(resp)
}()
var sz int64
go func() {
defer wg.Done()
st, err := os.Stat(b.Path())
if err != nil {
return
}
sz = st.Size()
}()
wg.Wait()
return &ArchiveDetails{
Checksum: checksum,
Size: sz,
}, nil
}