Better error handling and logging for restorations

router/router_server_backup.go

@@ -138,12 +138,14 @@ func postServerRestoreBackup(c *gin.Context) {
 		return
 	}
 
-	go func(uuid string, logger *log.Entry) {
+	go func(s *server.Server, uuid string, logger *log.Entry) {
-		logger.Info("restoring server from remote S3 backup...")
+		logger.Info("starting restoration process for server backup using S3 driver")
 		if err := s.RestoreBackup(backup.NewS3(uuid, ""), res.Body); err != nil {
 			logger.WithField("error", errors.WithStack(err)).Error("failed to restore remote S3 backup to server")
 		}
-	}(c.Param("backup"), logger)
+		s.Events().Publish(server.BackupRestoreCompletedEvent, "")
+		logger.Info("completed server restoration from S3 backup")
+	}(s, c.Param("backup"), logger)
 
 	c.Status(http.StatusAccepted)
 }

router/websocket/listeners.go

@@ -45,6 +45,7 @@ var e = []string{
 	server.InstallCompletedEvent,
 	server.DaemonMessageEvent,
 	server.BackupCompletedEvent,
+	server.BackupRestoreCompletedEvent,
 	server.TransferLogsEvent,
 	server.TransferStatusEvent,
 }

server/backup.go

@@ -7,7 +7,9 @@ import (
 
 	"emperror.dev/errors"
 	"github.com/apex/log"
+	"github.com/docker/docker/client"
 	"github.com/pterodactyl/wings/api"
+	"github.com/pterodactyl/wings/environment"
 	"github.com/pterodactyl/wings/server/backup"
 )
 
@@ -126,13 +128,6 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) (
 			reader.Close()
 		}
 	}()
-	// Don't try to restore the server until we have completely stopped the running
-	// instance, otherwise you'll likely hit all types of write errors due to the
-	// server being suspended.
-	err = s.Environment.WaitForStop(120, false)
-	if err != nil {
-		return errors.WithStackIf(err)
-	}
 	// Send an API call to the Panel as soon as this function is done running so that
 	// the Panel is informed of the restoration status of this backup.
 	defer func() {
@@ -141,8 +136,20 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) (
 		}
 	}()
 
+	// Don't try to restore the server until we have completely stopped the running
+	// instance, otherwise you'll likely hit all types of write errors due to the
+	// server being suspended.
+	if s.Environment.State() != environment.ProcessOfflineState {
+		if err = s.Environment.WaitForStop(120, false); err != nil {
+			if !client.IsErrNotFound(err) {
+				return errors.WrapIf(err, "server/backup: restore: failed to wait for container stop")
+			}
+		}
+	}
+
 	// Attempt to restore the backup to the server by running through each entry
 	// in the file one at a time and writing them to the disk.
+	s.Log().Debug("starting file writing process for backup restoration")
 	err = b.Restore(reader, func(file string, r io.Reader) error {
 		return s.Filesystem().Writefile(file, r)
 	})

server/events.go

@@ -7,16 +7,17 @@ import (
 // Defines all of the possible output events for a server.
 // noinspection GoNameStartsWithPackageName
 const (
-	DaemonMessageEvent    = "daemon message"
+	DaemonMessageEvent          = "daemon message"
-	InstallOutputEvent    = "install output"
+	InstallOutputEvent          = "install output"
-	InstallStartedEvent   = "install started"
+	InstallStartedEvent         = "install started"
-	InstallCompletedEvent = "install completed"
+	InstallCompletedEvent       = "install completed"
-	ConsoleOutputEvent    = "console output"
+	ConsoleOutputEvent          = "console output"
-	StatusEvent           = "status"
+	StatusEvent                 = "status"
-	StatsEvent            = "stats"
+	StatsEvent                  = "stats"
-	BackupCompletedEvent  = "backup completed"
+	BackupRestoreCompletedEvent = "backup restore completed"
-	TransferLogsEvent     = "transfer logs"
+	BackupCompletedEvent        = "backup completed"
-	TransferStatusEvent   = "transfer status"
+	TransferLogsEvent           = "transfer logs"
+	TransferStatusEvent         = "transfer status"
 )
 
 // Returns the server's emitter instance.

server/filesystem/disk_space.go

@@ -1,6 +1,7 @@
 package filesystem
 
 import (
+	"emperror.dev/errors"
 	"github.com/apex/log"
 	"github.com/karrick/godirwalk"
 	"sync"
@@ -189,7 +190,7 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
 		},
 	})
 
-	return size, err
+	return size, errors.WrapIf(err, "server/filesystem: directorysize: failed to walk directory")
 }
 
 // Helper function to determine if a server has space available for a file of a given size.

server/filesystem/filesystem.go

@@ -90,12 +90,12 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) {
 	}
 	// If the error is not because it doesn't exist then we just need to bail at this point.
 	if !errors.Is(err, os.ErrNotExist) {
-		return nil, err
+		return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file handle")
 	}
 	// Create the path leading up to the file we're trying to create, setting the final perms
 	// on it as we go.
 	if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "server/filesystem: touch: failed to create directory tree")
 	}
 	if err := fs.Chown(filepath.Dir(cleaned)); err != nil {
 		return nil, err
@@ -105,7 +105,7 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) {
 	// Chown that file so that the permissions don't mess with things.
 	f, err = o.open(cleaned, flag, 0644)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file with wait")
 	}
 	_ = fs.Chown(cleaned)
 	return f, nil
@@ -138,7 +138,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error {
 	// to it and an empty file. We'll then write to it later on after this completes.
 	stat, err := os.Stat(cleaned)
 	if err != nil && !os.IsNotExist(err) {
-		return err
+		return errors.Wrap(err, "server/filesystem: writefile: failed to stat file")
 	} else if err == nil {
 		if stat.IsDir() {
 			return &Error{code: ErrCodeIsDirectory, resolved: cleaned}
@@ -235,7 +235,7 @@ func (fs *Filesystem) Chown(path string) error {
 
 	// Start by just chowning the initial path that we received.
 	if err := os.Chown(cleaned, uid, gid); err != nil {
-		return err
+		return errors.Wrap(err, "server/filesystem: chown: failed to chown path")
 	}
 
 	// If this is not a directory we can now return from the function, there is nothing
@@ -246,7 +246,7 @@ func (fs *Filesystem) Chown(path string) error {
 
 	// If this was a directory, begin walking over its contents recursively and ensure that all
 	// of the subfiles and directories get their permissions updated as well.
-	return godirwalk.Walk(cleaned, &godirwalk.Options{
+	err = godirwalk.Walk(cleaned, &godirwalk.Options{
 		Unsorted: true,
 		Callback: func(p string, e *godirwalk.Dirent) error {
 			// Do not attempt to chmod a symlink. Go's os.Chown function will affect the symlink
@@ -263,6 +263,8 @@ func (fs *Filesystem) Chown(path string) error {
 			return os.Chown(p, uid, gid)
 		},
 	})
+
+	return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function")
 }
 
 func (fs *Filesystem) Chmod(path string, mode os.FileMode) error {

server/filesystem/path.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"sync"
 
+	"emperror.dev/errors"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -41,7 +42,7 @@ func (fs *Filesystem) SafePath(p string) (string, error) {
 	// is truly pointing to.
 	ep, err := filepath.EvalSymlinks(r)
 	if err != nil && !os.IsNotExist(err) {
-		return "", err
+		return "", errors.Wrap(err, "server/filesystem: failed to evaluate symlink")
 	} else if os.IsNotExist(err) {
 		// The requested directory doesn't exist, so at this point we need to iterate up the
 		// path chain until we hit a directory that _does_ exist and can be validated.