Merge branch 'develop' into feature/server-transfers

This commit is contained in:
Matthew Penner 2020-04-04 17:17:09 -06:00 committed by GitHub
commit 5693d0431e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 275 additions and 2 deletions

View File

@ -176,3 +176,29 @@ func (r *PanelRequest) SendTransferSuccess(uuid string) (*RequestError, error) {
return nil, nil return nil, nil
} }
type BackupRequest struct {
Successful bool `json:"successful"`
Sha256Hash string `json:"sha256_hash"`
FileSize int64 `json:"file_size"`
}
func (r *PanelRequest) SendBackupStatus(uuid string, backup string, data BackupRequest) (*RequestError, error) {
b, err := json.Marshal(data)
if err != nil {
return nil, errors.WithStack(err)
}
resp, err := r.Post(fmt.Sprintf("/servers/%s/backup/%s", uuid, backup), b)
if err != nil {
return nil, errors.WithStack(err)
}
defer resp.Body.Close()
r.Response = resp
if r.HasError() {
return r.Error(), nil
}
return nil, nil
}

View File

@ -69,6 +69,9 @@ type SystemConfiguration struct {
// Directory where server archives for transferring will be stored. // Directory where server archives for transferring will be stored.
ArchiveDirectory string `default:"/srv/daemon-data/.archives" yaml:"archive_directory"` ArchiveDirectory string `default:"/srv/daemon-data/.archives" yaml:"archive_directory"`
// Directory where local backups will be stored on the machine.
BackupDirectory string `default:"/srv/daemon-data/.backups" yaml:"backup_directory"`
// The user that should own all of the server files, and be used for containers. // The user that should own all of the server files, and be used for containers.
Username string `default:"pterodactyl" yaml:"username"` Username string `default:"pterodactyl" yaml:"username"`

5
go.sum
View File

@ -50,8 +50,9 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
@ -99,6 +100,8 @@ github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2Em
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs=
github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee h1:IquUs3fIykn10zWDIyddanhpTqBvAHMaPnFhQuyYw5U=
github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

53
http.go
View File

@ -518,7 +518,57 @@ func (rt *Router) routeServerReinstall(w http.ResponseWriter, r *http.Request, p
w.WriteHeader(http.StatusAccepted) w.WriteHeader(http.StatusAccepted)
} }
func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
s := rt.GetServer(ps.ByName("server"))
defer r.Body.Close()
data := rt.ReaderToBytes(r.Body)
b, err := s.NewBackup(data)
if err != nil {
zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err))
http.Error(w, "failed to update data structure", http.StatusInternalServerError)
return
}
zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid))
go func(bk *server.Backup) {
if err := bk.BackupAndNotify(); err != nil {
zap.S().Errorw("failed to generate backup for server", zap.Error(err))
} else {
zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid))
}
}(b)
w.WriteHeader(http.StatusAccepted)
}
func (rt *Router) routeServerBackup(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
s := rt.GetServer(ps.ByName("server"))
defer r.Body.Close()
data := rt.ReaderToBytes(r.Body)
b, err := s.NewBackup(data)
if err != nil {
zap.S().Errorw("failed to create backup struct for server", zap.String("server", s.Uuid), zap.Error(err))
http.Error(w, "failed to update data structure", http.StatusInternalServerError)
return
}
zap.S().Infow("starting backup process for server", zap.String("server", s.Uuid), zap.String("backup", b.Uuid))
go func(bk *server.Backup) {
if err := bk.BackupAndNotify(); err != nil {
zap.S().Errorw("failed to generate backup for server", zap.Error(err))
} else {
zap.S().Infow("completed backup process for server", zap.String("backup", b.Uuid))
}
}(b)
w.WriteHeader(http.StatusAccepted)
}
func (rt *Router) routeSystemInformation(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
defer r.Body.Close() defer r.Body.Close()
s, err := GetSystemInformation() s, err := GetSystemInformation()
@ -841,6 +891,7 @@ func (rt *Router) ConfigureRouter() *httprouter.Router {
router.POST("/api/servers/:server/power", rt.AuthenticateRequest(rt.routeServerPower)) router.POST("/api/servers/:server/power", rt.AuthenticateRequest(rt.routeServerPower))
router.POST("/api/servers/:server/commands", rt.AuthenticateRequest(rt.routeServerSendCommand)) router.POST("/api/servers/:server/commands", rt.AuthenticateRequest(rt.routeServerSendCommand))
router.POST("/api/servers/:server/reinstall", rt.AuthenticateRequest(rt.routeServerReinstall)) router.POST("/api/servers/:server/reinstall", rt.AuthenticateRequest(rt.routeServerReinstall))
router.POST("/api/servers/:server/backup", rt.AuthenticateRequest(rt.routeServerBackup))
router.PATCH("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerUpdate)) router.PATCH("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerUpdate))
router.DELETE("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerDelete)) router.DELETE("/api/servers/:server", rt.AuthenticateRequest(rt.routeServerDelete))

190
server/backup.go Normal file
View File

@ -0,0 +1,190 @@
package server
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"github.com/mholt/archiver/v3"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/config"
"go.uber.org/zap"
"io"
"os"
"path"
"strings"
"sync"
)
type Backup struct {
Uuid string `json:"uuid"`
IgnoredFiles []string `json:"ignored_files"`
server *Server
localDirectory string
}
// Create a new Backup struct from data passed through in a request.
func (s *Server) NewBackup(data []byte) (*Backup, error) {
backup := &Backup{}
if err := json.Unmarshal(data, backup); err != nil {
return nil, errors.WithStack(err)
}
backup.server = s
backup.localDirectory = path.Join(config.Get().System.BackupDirectory, s.Uuid)
return backup, nil
}
// Ensures that the local backup destination for files exists.
func (b *Backup) ensureLocalBackupLocation() error {
if _, err := os.Stat(b.localDirectory); err != nil {
if !os.IsNotExist(err) {
return errors.WithStack(err)
}
return os.MkdirAll(b.localDirectory, 0700)
}
return nil
}
// Returns the path for this specific backup.
func (b *Backup) GetPath() string {
return path.Join(b.localDirectory, b.Uuid+".tar.gz")
}
func (b *Backup) GetChecksum() ([]byte, error) {
h := sha256.New()
f, err := os.Open(b.GetPath())
if err != nil {
return []byte{}, errors.WithStack(err)
}
defer f.Close()
if _, err := io.Copy(h, f); err != nil {
return []byte{}, errors.WithStack(err)
}
return h.Sum(nil), nil
}
// Generates a backup of the selected files and pushes it to the defined location
// for this instance.
func (b *Backup) Backup() (*api.BackupRequest, error) {
rootPath := b.server.Filesystem.Path()
if err := b.ensureLocalBackupLocation(); err != nil {
return nil, errors.WithStack(err)
}
zap.S().Debugw("starting archive of server files for backup", zap.String("server", b.server.Uuid), zap.String("backup", b.Uuid))
if err := archiver.Archive([]string{rootPath}, b.GetPath()); err != nil {
if strings.HasPrefix(err.Error(), "file already exists") {
zap.S().Debugw("backup already exists on system, removing and re-attempting", zap.String("backup", b.Uuid))
if rerr := os.Remove(b.GetPath()); rerr != nil {
return nil, errors.WithStack(rerr)
}
// Re-attempt this backup.
return b.Backup()
}
// If there was some error with the archive, just go ahead and ensure the backup
// is completely destroyed at this point. Ignore any errors from this function.
os.Remove(b.GetPath())
return nil, err
}
wg := sync.WaitGroup{}
wg.Add(2)
var checksum string
// Calculate the checksum for the file.
go func() {
defer wg.Done()
resp, err := b.GetChecksum()
if err != nil {
zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err))
}
checksum = hex.EncodeToString(resp)
}()
var s int64
go func() {
defer wg.Done()
st, err := os.Stat(b.GetPath())
if err != nil {
return
}
s = st.Size()
}()
wg.Wait()
return &api.BackupRequest{
Successful: true,
Sha256Hash: checksum,
FileSize: s,
}, nil
}
// Performs a server backup and then notifies the Panel of the completed status
// so that the backup shows up for the user correctly.
func (b *Backup) BackupAndNotify() error {
resp, err := b.Backup()
if err != nil {
b.notifyPanel(resp)
return errors.WithStack(err)
}
if err := b.notifyPanel(resp); err != nil {
// These errors indicate that the Panel will not know about the status of this
// backup, so let's just go ahead and delete it, and let the Panel handle the
// cleanup process for the backups.
//
// @todo perhaps in the future we can sync the backups from the servers on boot?
os.Remove(b.GetPath())
return err
}
return nil
}
func (b *Backup) notifyPanel(request *api.BackupRequest) error {
r := api.NewRequester()
rerr, err := r.SendBackupStatus(b.server.Uuid, b.Uuid, *request)
if rerr != nil || err != nil {
if err != nil {
zap.S().Errorw(
"failed to notify panel of backup status due to internal code error",
zap.String("server", b.server.Uuid),
zap.String("backup", b.Uuid),
zap.Error(err),
)
return err
}
zap.S().Warnw(
rerr.String(),
zap.String("server", b.server.Uuid),
zap.String("backup", b.Uuid),
)
return errors.New(rerr.String())
}
return nil
}