diff --git a/api/server_endpoints.go b/api/server_endpoints.go index 7f6692c..0290799 100644 --- a/api/server_endpoints.go +++ b/api/server_endpoints.go @@ -122,3 +122,29 @@ func (r *PanelRequest) SendInstallationStatus(uuid string, successful bool) (*Re return nil, nil } + +type BackupRequest struct { + Successful bool `json:"successful"` + Sha256Hash string `json:"sha256_hash"` + FileSize int64 `json:"file_size"` +} + +func (r *PanelRequest) SendBackupStatus(uuid string, backup string, data BackupRequest) (*RequestError, error) { + b, err := json.Marshal(data) + if err != nil { + return nil, errors.WithStack(err) + } + + resp, err := r.Post(fmt.Sprintf("/servers/%s/backup/%s", uuid, backup), b) + if err != nil { + return nil, errors.WithStack(err) + } + defer resp.Body.Close() + + r.Response = resp + if r.HasError() { + return r.Error(), nil + } + + return nil, nil +} \ No newline at end of file diff --git a/config/config.go b/config/config.go index ddb905a..b397a88 100644 --- a/config/config.go +++ b/config/config.go @@ -66,6 +66,9 @@ type SystemConfiguration struct { // Directory where the server data is stored at. Data string `default:"/srv/daemon-data" yaml:"data"` + // Directory where local backups will be stored on the machine. + BackupDirectory string `default:"/srv/daemon-data/.backups" yaml:"backup_directory"` + // The user that should own all of the server files, and be used for containers. Username string `default:"pterodactyl" yaml:"username"` diff --git a/go.mod b/go.mod index 60ce73c..8322bbc 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/imdario/mergo v0.3.8 github.com/julienschmidt/httprouter v1.2.0 github.com/magiconair/properties v1.8.1 + github.com/mholt/archiver/v3 v3.3.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index f03429c..2c974f5 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.4.7 h1:vOvDiY/F1avSWlCWiKJjdYKz2jVjTK3pWPHndeG4 github.com/Microsoft/go-winio v0.4.7/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 h1:bZ28Hqta7TFAK3Q08CMvv8y3/8ATaEqv2nGoc6yff6c= +github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= @@ -31,6 +33,9 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v0.1.4 h1:5mcsq3+DXypREUkW+1juhjeKmE/XnWgs+paHMJn7lf8= @@ -41,10 +46,15 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= +github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= @@ -63,6 +73,12 @@ github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= +github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= +github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -74,8 +90,12 @@ github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig= +github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= +github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee h1:IquUs3fIykn10zWDIyddanhpTqBvAHMaPnFhQuyYw5U= github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -89,6 +109,8 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -113,6 +135,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/uber-go/zap v1.9.1/go.mod h1:GY+83l3yxBcBw2kmHu/sAWwItnTn+ynxHCRo+WiIQOY= +github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/http.go b/http.go index 02f282d..8a126f2 100644 --- a/http.go +++ b/http.go @@ -510,6 +510,31 @@ func (rt *Router) routeServerReinstall(w http.ResponseWriter, r *http.Request, p w.WriteHeader(http.StatusAccepted) } +func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + s := rt.GetServer(ps.ByName("server")) + defer r.Body.Close() + + data := rt.ReaderToBytes(r.Body) + b, err := s.NewBackup(data) + if err != nil { + zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err)) + + http.Error(w, "failed to update data structure", http.StatusInternalServerError) + return + } + + zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid)) + go func(bk *server.Backup) { + if err := bk.BackupAndNotify(); err != nil { + zap.S().Errorw("failed to generate backup for server", zap.Error(err)) + } else { + zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid)) + } + }(b) + + w.WriteHeader(http.StatusAccepted) +} + func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { defer r.Body.Close() @@ -605,6 +630,7 @@ func (rt *Router) ConfigureRouter() *httprouter.Router { router.POST("/api/servers/:server/power", rt.AuthenticateRequest(rt.routeServerPower)) router.POST("/api/servers/:server/commands", rt.AuthenticateRequest(rt.routeServerSendCommand)) router.POST("/api/servers/:server/reinstall", rt.AuthenticateRequest(rt.routeServerReinstall)) + router.POST("/api/servers/:server/backup", rt.AuthenticateRequest(rt.routeServerBackup)) router.PATCH("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerUpdate)) router.DELETE("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerDelete)) diff --git a/server/backup.go b/server/backup.go new file mode 100644 index 0000000..e41e844 --- /dev/null +++ b/server/backup.go @@ -0,0 +1,190 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "github.com/mholt/archiver/v3" + "github.com/pkg/errors" + "github.com/pterodactyl/wings/api" + "github.com/pterodactyl/wings/config" + "go.uber.org/zap" + "io" + "os" + "path" + "strings" + "sync" +) + +type Backup struct { + Uuid string `json:"uuid"` + IgnoredFiles []string `json:"ignored_files"` + server *Server + localDirectory string +} + +// Create a new Backup struct from data passed through in a request. +func (s *Server) NewBackup(data []byte) (*Backup, error) { + backup := &Backup{} + + if err := json.Unmarshal(data, backup); err != nil { + return nil, errors.WithStack(err) + } + + backup.server = s + backup.localDirectory = path.Join(config.Get().System.BackupDirectory, s.Uuid) + + return backup, nil +} + +// Ensures that the local backup destination for files exists. +func (b *Backup) ensureLocalBackupLocation() error { + if _, err := os.Stat(b.localDirectory); err != nil { + if !os.IsNotExist(err) { + return errors.WithStack(err) + } + + return os.MkdirAll(b.localDirectory, 0700) + } + + return nil +} + +// Returns the path for this specific backup. +func (b *Backup) GetPath() string { + return path.Join(b.localDirectory, b.Uuid+".tar.gz") +} + +func (b *Backup) GetChecksum() ([]byte, error) { + h := sha256.New() + + f, err := os.Open(b.GetPath()) + if err != nil { + return []byte{}, errors.WithStack(err) + } + defer f.Close() + + if _, err := io.Copy(h, f); err != nil { + return []byte{}, errors.WithStack(err) + } + + return h.Sum(nil), nil +} + +// Generates a backup of the selected files and pushes it to the defined location +// for this instance. +func (b *Backup) Backup() (*api.BackupRequest, error) { + rootPath := b.server.Filesystem.Path() + + if err := b.ensureLocalBackupLocation(); err != nil { + return nil, errors.WithStack(err) + } + + zap.S().Debugw("starting archive of server files for backup", zap.String("server", b.server.Uuid), zap.String("backup", b.Uuid)) + if err := archiver.Archive([]string{rootPath}, b.GetPath()); err != nil { + if strings.HasPrefix(err.Error(), "file already exists") { + zap.S().Debugw("backup already exists on system, removing and re-attempting", zap.String("backup", b.Uuid)) + + if rerr := os.Remove(b.GetPath()); rerr != nil { + return nil, errors.WithStack(rerr) + } + + // Re-attempt this backup. + return b.Backup() + } + + // If there was some error with the archive, just go ahead and ensure the backup + // is completely destroyed at this point. Ignore any errors from this function. + os.Remove(b.GetPath()) + + return nil, err + } + + wg := sync.WaitGroup{} + wg.Add(2) + + var checksum string + // Calculate the checksum for the file. + go func() { + defer wg.Done() + + resp, err := b.GetChecksum() + if err != nil { + zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err)) + } + + checksum = hex.EncodeToString(resp) + }() + + var s int64 + go func() { + defer wg.Done() + + st, err := os.Stat(b.GetPath()) + if err != nil { + return + } + + s = st.Size() + }() + + wg.Wait() + + return &api.BackupRequest{ + Successful: true, + Sha256Hash: checksum, + FileSize: s, + }, nil +} + +// Performs a server backup and then notifies the Panel of the completed status +// so that the backup shows up for the user correctly. +func (b *Backup) BackupAndNotify() error { + resp, err := b.Backup() + if err != nil { + b.notifyPanel(resp) + + return errors.WithStack(err) + } + + if err := b.notifyPanel(resp); err != nil { + // These errors indicate that the Panel will not know about the status of this + // backup, so let's just go ahead and delete it, and let the Panel handle the + // cleanup process for the backups. + // + // @todo perhaps in the future we can sync the backups from the servers on boot? + os.Remove(b.GetPath()) + + return err + } + + return nil +} + +func (b *Backup) notifyPanel(request *api.BackupRequest) error { + r := api.NewRequester() + + rerr, err := r.SendBackupStatus(b.server.Uuid, b.Uuid, *request) + if rerr != nil || err != nil { + if err != nil { + zap.S().Errorw( + "failed to notify panel of backup status due to internal code error", + zap.String("server", b.server.Uuid), + zap.String("backup", b.Uuid), + zap.Error(err), + ) + + return err + } + + zap.S().Warnw( + rerr.String(), + zap.String("server", b.server.Uuid), + zap.String("backup", b.Uuid), + ) + + return errors.New(rerr.String()) + } + + return nil +}