Merge branch 'develop' into feature/server-transfers

api/server_endpoints.go

@@ -176,3 +176,29 @@ func (r *PanelRequest) SendTransferSuccess(uuid string) (*RequestError, error) {
 
 	return nil, nil
 }
+
+type BackupRequest struct {
+	Successful bool `json:"successful"`
+	Sha256Hash string `json:"sha256_hash"`
+	FileSize int64 `json:"file_size"`
+}
+
+func (r *PanelRequest) SendBackupStatus(uuid string, backup string, data BackupRequest) (*RequestError, error) {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	resp, err := r.Post(fmt.Sprintf("/servers/%s/backup/%s", uuid, backup), b)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	defer resp.Body.Close()
+
+	r.Response = resp
+	if r.HasError() {
+		return r.Error(), nil
+	}
+
+	return nil, nil
+}

config/config.go

@@ -69,6 +69,9 @@ type SystemConfiguration struct {
 	// Directory where server archives for transferring will be stored.
 	ArchiveDirectory string `default:"/srv/daemon-data/.archives" yaml:"archive_directory"`
 
+	// Directory where local backups will be stored on the machine.
+	BackupDirectory string `default:"/srv/daemon-data/.backups" yaml:"backup_directory"`
+
 	// The user that should own all of the server files, and be used for containers.
 	Username string `default:"pterodactyl" yaml:"username"`
 

go.sum

@@ -50,8 +50,9 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
@@ -99,6 +100,8 @@ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2Em
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs=
 github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
+github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee h1:IquUs3fIykn10zWDIyddanhpTqBvAHMaPnFhQuyYw5U=
+github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

http.go

@@ -518,7 +518,57 @@ func (rt *Router) routeServerReinstall(w http.ResponseWriter, r *http.Request, p
 	w.WriteHeader(http.StatusAccepted)
 }
 
-func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	s := rt.GetServer(ps.ByName("server"))
+	defer r.Body.Close()
+
+	data := rt.ReaderToBytes(r.Body)
+	b, err := s.NewBackup(data)
+	if err != nil {
+		zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err))
+
+		http.Error(w, "failed to update data structure", http.StatusInternalServerError)
+		return
+	}
+
+	zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid))
+	go func(bk *server.Backup) {
+		if err := bk.BackupAndNotify(); err != nil {
+			zap.S().Errorw("failed to generate backup for server", zap.Error(err))
+		} else {
+			zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid))
+		}
+	}(b)
+
+	w.WriteHeader(http.StatusAccepted)
+}
+
+func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	s := rt.GetServer(ps.ByName("server"))
+	defer r.Body.Close()
+
+	data := rt.ReaderToBytes(r.Body)
+	b, err := s.NewBackup(data)
+	if err != nil {
+		zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err))
+
+		http.Error(w, "failed to update data structure", http.StatusInternalServerError)
+		return
+	}
+
+	zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid))
+	go func(bk *server.Backup) {
+		if err := bk.BackupAndNotify(); err != nil {
+			zap.S().Errorw("failed to generate backup for server", zap.Error(err))
+		} else {
+			zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid))
+		}
+	}(b)
+
+	w.WriteHeader(http.StatusAccepted)
+}
+
+func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	defer r.Body.Close()
 
 	s, err := GetSystemInformation()
@@ -841,6 +891,7 @@ func (rt *Router) ConfigureRouter() *httprouter.Router {
 	router.POST("/api/servers/:server/power", rt.AuthenticateRequest(rt.routeServerPower))
 	router.POST("/api/servers/:server/commands", rt.AuthenticateRequest(rt.routeServerSendCommand))
 	router.POST("/api/servers/:server/reinstall", rt.AuthenticateRequest(rt.routeServerReinstall))
+	router.POST("/api/servers/:server/backup", rt.AuthenticateRequest(rt.routeServerBackup))
 	router.PATCH("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerUpdate))
 	router.DELETE("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerDelete))
 

server/backup.go

@@ -0,0 +1,190 @@
+package server
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"github.com/mholt/archiver/v3"
+	"github.com/pkg/errors"
+	"github.com/pterodactyl/wings/api"
+	"github.com/pterodactyl/wings/config"
+	"go.uber.org/zap"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"sync"
+)
+
+type Backup struct {
+	Uuid           string   `json:"uuid"`
+	IgnoredFiles   []string `json:"ignored_files"`
+	server         *Server
+	localDirectory string
+}
+
+// Create a new Backup struct from data passed through in a request.
+func (s *Server) NewBackup(data []byte) (*Backup, error) {
+	backup := &Backup{}
+
+	if err := json.Unmarshal(data, backup); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	backup.server = s
+	backup.localDirectory = path.Join(config.Get().System.BackupDirectory, s.Uuid)
+
+	return backup, nil
+}
+
+// Ensures that the local backup destination for files exists.
+func (b *Backup) ensureLocalBackupLocation() error {
+	if _, err := os.Stat(b.localDirectory); err != nil {
+		if !os.IsNotExist(err) {
+			return errors.WithStack(err)
+		}
+
+		return os.MkdirAll(b.localDirectory, 0700)
+	}
+
+	return nil
+}
+
+// Returns the path for this specific backup.
+func (b *Backup) GetPath() string {
+	return path.Join(b.localDirectory, b.Uuid+".tar.gz")
+}
+
+func (b *Backup) GetChecksum() ([]byte, error) {
+	h := sha256.New()
+
+	f, err := os.Open(b.GetPath())
+	if err != nil {
+		return []byte{}, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(h, f); err != nil {
+		return []byte{}, errors.WithStack(err)
+	}
+
+	return h.Sum(nil), nil
+}
+
+// Generates a backup of the selected files and pushes it to the defined location
+// for this instance.
+func (b *Backup) Backup() (*api.BackupRequest, error) {
+	rootPath := b.server.Filesystem.Path()
+
+	if err := b.ensureLocalBackupLocation(); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	zap.S().Debugw("starting archive of server files for backup", zap.String("server", b.server.Uuid), zap.String("backup", b.Uuid))
+	if err := archiver.Archive([]string{rootPath}, b.GetPath()); err != nil {
+		if strings.HasPrefix(err.Error(), "file already exists") {
+			zap.S().Debugw("backup already exists on system, removing and re-attempting", zap.String("backup", b.Uuid))
+
+			if rerr := os.Remove(b.GetPath()); rerr != nil {
+				return nil, errors.WithStack(rerr)
+			}
+
+			// Re-attempt this backup.
+			return b.Backup()
+		}
+
+		// If there was some error with the archive, just go ahead and ensure the backup
+		// is completely destroyed at this point. Ignore any errors from this function.
+		os.Remove(b.GetPath())
+
+		return nil, err
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+
+	var checksum string
+	// Calculate the checksum for the file.
+	go func() {
+		defer wg.Done()
+
+		resp, err := b.GetChecksum()
+		if err != nil {
+			zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err))
+		}
+
+		checksum = hex.EncodeToString(resp)
+	}()
+
+	var s int64
+	go func() {
+		defer wg.Done()
+
+		st, err := os.Stat(b.GetPath())
+		if err != nil {
+			return
+		}
+
+		s = st.Size()
+	}()
+
+	wg.Wait()
+
+	return &api.BackupRequest{
+		Successful: true,
+		Sha256Hash: checksum,
+		FileSize:   s,
+	}, nil
+}
+
+// Performs a server backup and then notifies the Panel of the completed status
+// so that the backup shows up for the user correctly.
+func (b *Backup) BackupAndNotify() error {
+	resp, err := b.Backup()
+	if err != nil {
+		b.notifyPanel(resp)
+
+		return errors.WithStack(err)
+	}
+
+	if err := b.notifyPanel(resp); err != nil {
+		// These errors indicate that the Panel will not know about the status of this
+		// backup, so let's just go ahead and delete it, and let the Panel handle the
+		// cleanup process for the backups.
+		//
+		// @todo perhaps in the future we can sync the backups from the servers on boot?
+		os.Remove(b.GetPath())
+
+		return err
+	}
+
+	return nil
+}
+
+func (b *Backup) notifyPanel(request *api.BackupRequest) error {
+	r := api.NewRequester()
+
+	rerr, err := r.SendBackupStatus(b.server.Uuid, b.Uuid, *request)
+	if rerr != nil || err != nil {
+		if err != nil {
+			zap.S().Errorw(
+				"failed to notify panel of backup status due to internal code error",
+				zap.String("server", b.server.Uuid),
+				zap.String("backup", b.Uuid),
+				zap.Error(err),
+			)
+
+			return err
+		}
+
+		zap.S().Warnw(
+			rerr.String(),
+			zap.String("server", b.server.Uuid),
+			zap.String("backup", b.Uuid),
+		)
+
+		return errors.New(rerr.String())
+	}
+
+	return nil
+}