Add first implementation of local backups for a server

api/server_endpoints.go

@@ -122,3 +122,29 @@ func (r *PanelRequest) SendInstallationStatus(uuid string, successful bool) (*Re
 
 	return nil, nil
 }
+
+type BackupRequest struct {
+	Successful bool `json:"successful"`
+	Sha256Hash string `json:"sha256_hash"`
+	FileSize int64 `json:"file_size"`
+}
+
+func (r *PanelRequest) SendBackupStatus(uuid string, backup string, data BackupRequest) (*RequestError, error) {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	resp, err := r.Post(fmt.Sprintf("/servers/%s/backup/%s", uuid, backup), b)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	defer resp.Body.Close()
+
+	r.Response = resp
+	if r.HasError() {
+		return r.Error(), nil
+	}
+
+	return nil, nil
+}

config/config.go

@@ -66,6 +66,9 @@ type SystemConfiguration struct {
 	// Directory where the server data is stored at.
 	Data string `default:"/srv/daemon-data" yaml:"data"`
 
+	// Directory where local backups will be stored on the machine.
+	BackupDirectory string `default:"/srv/daemon-data/.backups" yaml:"backup_directory"`
+
 	// The user that should own all of the server files, and be used for containers.
 	Username string `default:"pterodactyl" yaml:"username"`
 

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/imdario/mergo v0.3.8
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/magiconair/properties v1.8.1
+	github.com/mholt/archiver/v3 v3.3.0
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee
 	github.com/onsi/ginkgo v1.8.0 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.4.7 h1:vOvDiY/F1avSWlCWiKJjdYKz2jVjTK3pWPHndeG4
 github.com/Microsoft/go-winio v0.4.7/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
+github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 h1:bZ28Hqta7TFAK3Q08CMvv8y3/8ATaEqv2nGoc6yff6c=
+github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
@@ -31,6 +33,9 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gabriel-vasile/mimetype v0.1.4 h1:5mcsq3+DXypREUkW+1juhjeKmE/XnWgs+paHMJn7lf8=
@@ -41,10 +46,15 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw=
+github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
@@ -63,6 +73,12 @@ github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY=
+github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=
+github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -74,8 +90,12 @@ github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig=
+github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs=
+github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee h1:IquUs3fIykn10zWDIyddanhpTqBvAHMaPnFhQuyYw5U=
 github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -89,6 +109,8 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -113,6 +135,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/uber-go/zap v1.9.1/go.mod h1:GY+83l3yxBcBw2kmHu/sAWwItnTn+ynxHCRo+WiIQOY=
+github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=

http.go

@@ -510,6 +510,31 @@ func (rt *Router) routeServerReinstall(w http.ResponseWriter, r *http.Request, p
 	w.WriteHeader(http.StatusAccepted)
 }
 
+func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	s := rt.GetServer(ps.ByName("server"))
+	defer r.Body.Close()
+
+	data := rt.ReaderToBytes(r.Body)
+	b, err := s.NewBackup(data)
+	if err != nil {
+		zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err))
+
+		http.Error(w, "failed to update data structure", http.StatusInternalServerError)
+		return
+	}
+
+	zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid))
+	go func(bk *server.Backup) {
+		if err := bk.BackupAndNotify(); err != nil {
+			zap.S().Errorw("failed to generate backup for server", zap.Error(err))
+		} else {
+			zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid))
+		}
+	}(b)
+
+	w.WriteHeader(http.StatusAccepted)
+}
+
 func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	defer r.Body.Close()
 
@@ -605,6 +630,7 @@ func (rt *Router) ConfigureRouter() *httprouter.Router {
 	router.POST("/api/servers/:server/power", rt.AuthenticateRequest(rt.routeServerPower))
 	router.POST("/api/servers/:server/commands", rt.AuthenticateRequest(rt.routeServerSendCommand))
 	router.POST("/api/servers/:server/reinstall", rt.AuthenticateRequest(rt.routeServerReinstall))
+	router.POST("/api/servers/:server/backup", rt.AuthenticateRequest(rt.routeServerBackup))
 	router.PATCH("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerUpdate))
 	router.DELETE("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerDelete))
 

server/backup.go

@@ -0,0 +1,190 @@
+package server
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"github.com/mholt/archiver/v3"
+	"github.com/pkg/errors"
+	"github.com/pterodactyl/wings/api"
+	"github.com/pterodactyl/wings/config"
+	"go.uber.org/zap"
+	"io"
+	"os"
+	"path"
+	"strings"
+	"sync"
+)
+
+type Backup struct {
+	Uuid           string   `json:"uuid"`
+	IgnoredFiles   []string `json:"ignored_files"`
+	server         *Server
+	localDirectory string
+}
+
+// Create a new Backup struct from data passed through in a request.
+func (s *Server) NewBackup(data []byte) (*Backup, error) {
+	backup := &Backup{}
+
+	if err := json.Unmarshal(data, backup); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	backup.server = s
+	backup.localDirectory = path.Join(config.Get().System.BackupDirectory, s.Uuid)
+
+	return backup, nil
+}
+
+// Ensures that the local backup destination for files exists.
+func (b *Backup) ensureLocalBackupLocation() error {
+	if _, err := os.Stat(b.localDirectory); err != nil {
+		if !os.IsNotExist(err) {
+			return errors.WithStack(err)
+		}
+
+		return os.MkdirAll(b.localDirectory, 0700)
+	}
+
+	return nil
+}
+
+// Returns the path for this specific backup.
+func (b *Backup) GetPath() string {
+	return path.Join(b.localDirectory, b.Uuid+".tar.gz")
+}
+
+func (b *Backup) GetChecksum() ([]byte, error) {
+	h := sha256.New()
+
+	f, err := os.Open(b.GetPath())
+	if err != nil {
+		return []byte{}, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(h, f); err != nil {
+		return []byte{}, errors.WithStack(err)
+	}
+
+	return h.Sum(nil), nil
+}
+
+// Generates a backup of the selected files and pushes it to the defined location
+// for this instance.
+func (b *Backup) Backup() (*api.BackupRequest, error) {
+	rootPath := b.server.Filesystem.Path()
+
+	if err := b.ensureLocalBackupLocation(); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	zap.S().Debugw("starting archive of server files for backup", zap.String("server", b.server.Uuid), zap.String("backup", b.Uuid))
+	if err := archiver.Archive([]string{rootPath}, b.GetPath()); err != nil {
+		if strings.HasPrefix(err.Error(), "file already exists") {
+			zap.S().Debugw("backup already exists on system, removing and re-attempting", zap.String("backup", b.Uuid))
+
+			if rerr := os.Remove(b.GetPath()); rerr != nil {
+				return nil, errors.WithStack(rerr)
+			}
+
+			// Re-attempt this backup.
+			return b.Backup()
+		}
+
+		// If there was some error with the archive, just go ahead and ensure the backup
+		// is completely destroyed at this point. Ignore any errors from this function.
+		os.Remove(b.GetPath())
+
+		return nil, err
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+
+	var checksum string
+	// Calculate the checksum for the file.
+	go func() {
+		defer wg.Done()
+
+		resp, err := b.GetChecksum()
+		if err != nil {
+			zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err))
+		}
+
+		checksum = hex.EncodeToString(resp)
+	}()
+
+	var s int64
+	go func() {
+		defer wg.Done()
+
+		st, err := os.Stat(b.GetPath())
+		if err != nil {
+			return
+		}
+
+		s = st.Size()
+	}()
+
+	wg.Wait()
+
+	return &api.BackupRequest{
+		Successful: true,
+		Sha256Hash: checksum,
+		FileSize:   s,
+	}, nil
+}
+
+// Performs a server backup and then notifies the Panel of the completed status
+// so that the backup shows up for the user correctly.
+func (b *Backup) BackupAndNotify() error {
+	resp, err := b.Backup()
+	if err != nil {
+		b.notifyPanel(resp)
+
+		return errors.WithStack(err)
+	}
+
+	if err := b.notifyPanel(resp); err != nil {
+		// These errors indicate that the Panel will not know about the status of this
+		// backup, so let's just go ahead and delete it, and let the Panel handle the
+		// cleanup process for the backups.
+		//
+		// @todo perhaps in the future we can sync the backups from the servers on boot?
+		os.Remove(b.GetPath())
+
+		return err
+	}
+
+	return nil
+}
+
+func (b *Backup) notifyPanel(request *api.BackupRequest) error {
+	r := api.NewRequester()
+
+	rerr, err := r.SendBackupStatus(b.server.Uuid, b.Uuid, *request)
+	if rerr != nil || err != nil {
+		if err != nil {
+			zap.S().Errorw(
+				"failed to notify panel of backup status due to internal code error",
+				zap.String("server", b.server.Uuid),
+				zap.String("backup", b.Uuid),
+				zap.Error(err),
+			)
+
+			return err
+		}
+
+		zap.S().Warnw(
+			rerr.String(),
+			zap.String("server", b.server.Uuid),
+			zap.String("backup", b.Uuid),
+		)
+
+		return errors.New(rerr.String())
+	}
+
+	return nil
+}