MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

transfers: use backup archiver

Browse Source
This commit is contained in:
Matthew Penner 2021-03-07 11:02:03 -07:00
parent ad2618bc6f
commit 0e3778ac47
9 changed files with 50 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
router/router_server.go
View File

@ -204,12 +204,6 @@ func deleteServer(c *gin.Context) {
s.Events().Destroy()
s.Websockets().CancelAll()
// Delete the server's archive if it exists. We intentionally don't return
// here, if the archive fails to delete, the server can still be removed.
if err := s.Archiver.DeleteIfExists(); err != nil {
s.Log().WithField("error", err).Warn("failed to delete server archive during deletion process")
}
// Remove any pending remote file downloads for the server.
for _, dl := range downloader.ByServer(s.Id()) {
dl.Cancel()

62
router/router_transfer.go
View File

@ -29,6 +29,7 @@ import (
"github.com/pterodactyl/wings/router/middleware"
"github.com/pterodactyl/wings/router/tokens"
"github.com/pterodactyl/wings/server"
"github.com/pterodactyl/wings/server/filesystem"
"github.com/pterodactyl/wings/system"
)
@ -51,6 +52,10 @@ type serverTransferRequest struct {
Server json.RawMessage `json:"server"`
}
func getArchivePath(sID string) string {
return filepath.Join(config.Get().System.ArchiveDirectory, sID+".tar.gz")
}
// Returns the archive for a server so that it can be transferred to a new node.
func getServerArchive(c *gin.Context) {
auth := strings.SplitN(c.GetHeader("Authorization"), " ", 2)
@ -77,36 +82,51 @@ func getServerArchive(c *gin.Context) {
return
}
st, err := s.Archiver.Stat()
archivePath := getArchivePath(s.Id())
// Stat the archive file.
st, err := os.Lstat(archivePath)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
WithError(c, err)
_ = WithError(c, err)
return
}
c.AbortWithStatus(http.StatusNotFound)
return
}
checksum, err := s.Archiver.Checksum()
// Compute sha1 checksum.
h := sha256.New()
f, err := os.Open(archivePath)
if err != nil {
NewServerError(err, s).SetMessage("failed to calculate checksum").Abort(c)
return
}
if _, err := io.Copy(h, bufio.NewReader(f)); err != nil {
_ = f.Close()
_ = WithError(c, err)
return
}
if err := f.Close(); err != nil {
_ = WithError(c, err)
return
}
checksum := hex.EncodeToString(h.Sum(nil))
file, err := os.Open(s.Archiver.Path())
// Stream the file to the client.
f, err = os.Open(archivePath)
if err != nil {
WithError(c, err)
_ = WithError(c, err)
return
}
defer file.Close()
defer f.Close()
c.Header("X-Checksum", checksum)
c.Header("X-Mime-Type", st.Mimetype)
c.Header("X-Mime-Type", "application/tar+gzip")
c.Header("Content-Length", strconv.Itoa(int(st.Size())))
c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(s.Archiver.Name()))
c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(s.Id()+".tar.gz"))
c.Header("Content-Type", "application/octet-stream")
bufio.NewReader(file).WriteTo(c.Writer)
_, _ = bufio.NewReader(f).WriteTo(c.Writer)
}
func postServerArchive(c *gin.Context) {
@ -164,8 +184,13 @@ func postServerArchive(c *gin.Context) {
return
}
// Create an archive of the entire server's data directory.
a := &filesystem.Archive{
BasePath: s.Filesystem().Path(),
}
// Attempt to get an archive of the server.
if err := s.Archiver.Archive(); err != nil {
if err := a.Create(getArchivePath(s.Id())); err != nil {
sendTransferLog("An error occurred while archiving the server: " + err.Error())
l.WithField("error", err).Error("failed to get transfer archive for server")
return
@ -227,7 +252,7 @@ func (str serverTransferRequest) downloadArchive() (*http.Response, error) {
// Returns the path to the local archive on the system.
func (str serverTransferRequest) path() string {
return filepath.Join(config.Get().System.ArchiveDirectory, str.ServerID+".tar.gz")
return getArchivePath(str.ServerID)
}
// Creates the archive location on this machine by first checking that the required file
@ -260,17 +285,16 @@ func (str serverTransferRequest) removeArchivePath() {
// expected value from the transfer request. The string value returned is the computed
// checksum on the system.
func (str serverTransferRequest) verifyChecksum(matches string) (bool, string, error) {
file, err := os.Open(str.path())
f, err := os.Open(str.path())
if err != nil {
return false, "", err
}
defer file.Close()
hash := sha256.New()
buf := make([]byte, 1024*4)
if _, err := io.CopyBuffer(hash, file, buf); err != nil {
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, bufio.NewReader(f)); err != nil {
return false, "", err
}
checksum := hex.EncodeToString(hash.Sum(nil))
checksum := hex.EncodeToString(h.Sum(nil))
return checksum == matches, checksum, nil
}
@ -362,7 +386,7 @@ func postTransfer(c *gin.Context) {
return
}
defer res.Body.Close()
if res.StatusCode != 200 {
if res.StatusCode != http.StatusOK {
data.log().WithField("error", err).WithField("status", res.StatusCode).Error("unexpected error response from transfer endpoint")
return
}

120
server/archiver.go
View File

@ -1,120 +0,0 @@
package server
import (
"crypto/sha256"
"encoding/hex"
"io"
"io/ioutil"
"os"
"path/filepath"
"emperror.dev/errors"
"github.com/mholt/archiver/v3"
"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/server/filesystem"
)
// Archiver represents a Server Archiver.
type Archiver struct {
Server *Server
}
// Path returns the path to the server's archive.
func (a *Archiver) Path() string {
return filepath.Join(config.Get().System.ArchiveDirectory, a.Name())
}
// Name returns the name of the server's archive.
func (a *Archiver) Name() string {
return a.Server.Id() + ".tar.gz"
}
// Exists returns a boolean based off if the archive exists.
func (a *Archiver) Exists() bool {
if _, err := os.Stat(a.Path()); os.IsNotExist(err) {
return false
}
return true
}
// Stat stats the archive file.
func (a *Archiver) Stat() (*filesystem.Stat, error) {
s, err := os.Stat(a.Path())
if err != nil {
return nil, err
}
return &filesystem.Stat{
FileInfo: s,
Mimetype: "application/tar+gzip",
}, nil
}
// Archive creates an archive of the server and deletes the previous one.
func (a *Archiver) Archive() error {
path := a.Server.Filesystem().Path()
// Get the list of root files and directories to archive.
var files []string
fileInfo, err := ioutil.ReadDir(path)
if err != nil {
return err
}
for _, file := range fileInfo {
f := filepath.Join(path, file.Name())
// If the file is a symlink we cannot safely assume that the result of a filepath.Join() will be
// a safe destination. We need to check if the file is a symlink, and if so pass off to the SafePath
// function to resolve it to the final destination.
//
// ioutil.ReadDir() calls Lstat, so this will work correctly. If it did not call Lstat, but rather
// just did a normal Stat call, this would fail since that would be looking at the symlink destination
// and not the actual file in this listing.
if file.Mode()&os.ModeSymlink != 0 {
f, err = a.Server.Filesystem().SafePath(filepath.Join(path, file.Name()))
if err != nil {
return err
}
}
files = append(files, f)
}
if err := a.DeleteIfExists(); err != nil {
return err
}
return archiver.NewTarGz().Archive(files, a.Path())
}
// DeleteIfExists deletes the archive if it exists.
func (a *Archiver) DeleteIfExists() error {
if _, err := a.Stat(); err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
return errors.WithMessage(os.Remove(a.Path()), "archiver: failed to delete archive from system")
}
// Checksum computes a SHA256 checksum of the server's archive.
func (a *Archiver) Checksum() (string, error) {
file, err := os.Open(a.Path())
if err != nil {
return "", err
}
defer file.Close()
hash := sha256.New()
buf := make([]byte, 1024*4)
if _, err := io.CopyBuffer(hash, file, buf); err != nil {
return "", err
}
return hex.EncodeToString(hash.Sum(nil)), nil
}

3
server/backup/backup_local.go
View File

@ -2,6 +2,7 @@ package backup
import (
"errors"
"github.com/pterodactyl/wings/server/filesystem"
"io"
"os"
@ -56,7 +57,7 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) {
// Generate generates a backup of the selected files and pushes it to the
// defined location for this instance.
func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
a := &Archive{
a := &filesystem.Archive{
BasePath: basePath,
Ignore: ignore,
}

3
server/backup/backup_s3.go
View File

@ -5,6 +5,7 @@ import (
"compress/gzip"
"context"
"fmt"
"github.com/pterodactyl/wings/server/filesystem"
"io"
"net/http"
"os"
@ -47,7 +48,7 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) {
func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
defer s.Remove()
a := &Archive{
a := &filesystem.Archive{
BasePath: basePath,
Ignore: ignore,
}

2
server/backup/archive.go → server/filesystem/archive.go
View File

@ -1,4 +1,4 @@
package backup
package filesystem
import (
"archive/tar"

4
server/filesystem/compress.go
View File

@ -10,7 +10,6 @@ import (
"time"
"github.com/mholt/archiver/v3"
"github.com/pterodactyl/wings/server/backup"
"github.com/pterodactyl/wings/system"
)
@ -39,7 +38,7 @@ func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, er
return nil, err
}
a := &backup.Archive{BasePath: cleanedRootDir, Files: cleaned}
a := &Archive{BasePath: cleanedRootDir, Files: cleaned}
d := path.Join(
cleanedRootDir,
fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")),
@ -144,4 +143,3 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error {
}
return nil
}

1
server/manager.go
View File

@ -175,7 +175,6 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server,
return nil, err
}
s.Archiver = Archiver{Server: s}
s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist)
// Right now we only support a Docker based environment, so I'm going to hard code

1
server/server.go
View File

@ -42,7 +42,6 @@ type Server struct {
crasher CrashHandler
resources ResourceUsage
Archiver Archiver `json:"-"`
Environment environment.ProcessEnvironment `json:"-"`
fs *filesystem.Filesystem