transfers: use backup archiver
This commit is contained in:
parent
ad2618bc6f
commit
0e3778ac47
|
@ -204,12 +204,6 @@ func deleteServer(c *gin.Context) {
|
||||||
s.Events().Destroy()
|
s.Events().Destroy()
|
||||||
s.Websockets().CancelAll()
|
s.Websockets().CancelAll()
|
||||||
|
|
||||||
// Delete the server's archive if it exists. We intentionally don't return
|
|
||||||
// here, if the archive fails to delete, the server can still be removed.
|
|
||||||
if err := s.Archiver.DeleteIfExists(); err != nil {
|
|
||||||
s.Log().WithField("error", err).Warn("failed to delete server archive during deletion process")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove any pending remote file downloads for the server.
|
// Remove any pending remote file downloads for the server.
|
||||||
for _, dl := range downloader.ByServer(s.Id()) {
|
for _, dl := range downloader.ByServer(s.Id()) {
|
||||||
dl.Cancel()
|
dl.Cancel()
|
||||||
|
|
|
@ -29,6 +29,7 @@ import (
|
||||||
"github.com/pterodactyl/wings/router/middleware"
|
"github.com/pterodactyl/wings/router/middleware"
|
||||||
"github.com/pterodactyl/wings/router/tokens"
|
"github.com/pterodactyl/wings/router/tokens"
|
||||||
"github.com/pterodactyl/wings/server"
|
"github.com/pterodactyl/wings/server"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"github.com/pterodactyl/wings/system"
|
"github.com/pterodactyl/wings/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -51,6 +52,10 @@ type serverTransferRequest struct {
|
||||||
Server json.RawMessage `json:"server"`
|
Server json.RawMessage `json:"server"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getArchivePath(sID string) string {
|
||||||
|
return filepath.Join(config.Get().System.ArchiveDirectory, sID+".tar.gz")
|
||||||
|
}
|
||||||
|
|
||||||
// Returns the archive for a server so that it can be transferred to a new node.
|
// Returns the archive for a server so that it can be transferred to a new node.
|
||||||
func getServerArchive(c *gin.Context) {
|
func getServerArchive(c *gin.Context) {
|
||||||
auth := strings.SplitN(c.GetHeader("Authorization"), " ", 2)
|
auth := strings.SplitN(c.GetHeader("Authorization"), " ", 2)
|
||||||
|
@ -77,36 +82,51 @@ func getServerArchive(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
st, err := s.Archiver.Stat()
|
archivePath := getArchivePath(s.Id())
|
||||||
|
|
||||||
|
// Stat the archive file.
|
||||||
|
st, err := os.Lstat(archivePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, os.ErrNotExist) {
|
if !errors.Is(err, os.ErrNotExist) {
|
||||||
WithError(c, err)
|
_ = WithError(c, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.AbortWithStatus(http.StatusNotFound)
|
c.AbortWithStatus(http.StatusNotFound)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
checksum, err := s.Archiver.Checksum()
|
// Compute sha1 checksum.
|
||||||
|
h := sha256.New()
|
||||||
|
f, err := os.Open(archivePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
NewServerError(err, s).SetMessage("failed to calculate checksum").Abort(c)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if _, err := io.Copy(h, bufio.NewReader(f)); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
_ = WithError(c, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
_ = WithError(c, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksum := hex.EncodeToString(h.Sum(nil))
|
||||||
|
|
||||||
file, err := os.Open(s.Archiver.Path())
|
// Stream the file to the client.
|
||||||
|
f, err = os.Open(archivePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
WithError(c, err)
|
_ = WithError(c, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer f.Close()
|
||||||
|
|
||||||
c.Header("X-Checksum", checksum)
|
c.Header("X-Checksum", checksum)
|
||||||
c.Header("X-Mime-Type", st.Mimetype)
|
c.Header("X-Mime-Type", "application/tar+gzip")
|
||||||
c.Header("Content-Length", strconv.Itoa(int(st.Size())))
|
c.Header("Content-Length", strconv.Itoa(int(st.Size())))
|
||||||
c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(s.Archiver.Name()))
|
c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(s.Id()+".tar.gz"))
|
||||||
c.Header("Content-Type", "application/octet-stream")
|
c.Header("Content-Type", "application/octet-stream")
|
||||||
|
|
||||||
bufio.NewReader(file).WriteTo(c.Writer)
|
_, _ = bufio.NewReader(f).WriteTo(c.Writer)
|
||||||
}
|
}
|
||||||
|
|
||||||
func postServerArchive(c *gin.Context) {
|
func postServerArchive(c *gin.Context) {
|
||||||
|
@ -164,8 +184,13 @@ func postServerArchive(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create an archive of the entire server's data directory.
|
||||||
|
a := &filesystem.Archive{
|
||||||
|
BasePath: s.Filesystem().Path(),
|
||||||
|
}
|
||||||
|
|
||||||
// Attempt to get an archive of the server.
|
// Attempt to get an archive of the server.
|
||||||
if err := s.Archiver.Archive(); err != nil {
|
if err := a.Create(getArchivePath(s.Id())); err != nil {
|
||||||
sendTransferLog("An error occurred while archiving the server: " + err.Error())
|
sendTransferLog("An error occurred while archiving the server: " + err.Error())
|
||||||
l.WithField("error", err).Error("failed to get transfer archive for server")
|
l.WithField("error", err).Error("failed to get transfer archive for server")
|
||||||
return
|
return
|
||||||
|
@ -227,7 +252,7 @@ func (str serverTransferRequest) downloadArchive() (*http.Response, error) {
|
||||||
|
|
||||||
// Returns the path to the local archive on the system.
|
// Returns the path to the local archive on the system.
|
||||||
func (str serverTransferRequest) path() string {
|
func (str serverTransferRequest) path() string {
|
||||||
return filepath.Join(config.Get().System.ArchiveDirectory, str.ServerID+".tar.gz")
|
return getArchivePath(str.ServerID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Creates the archive location on this machine by first checking that the required file
|
// Creates the archive location on this machine by first checking that the required file
|
||||||
|
@ -260,17 +285,16 @@ func (str serverTransferRequest) removeArchivePath() {
|
||||||
// expected value from the transfer request. The string value returned is the computed
|
// expected value from the transfer request. The string value returned is the computed
|
||||||
// checksum on the system.
|
// checksum on the system.
|
||||||
func (str serverTransferRequest) verifyChecksum(matches string) (bool, string, error) {
|
func (str serverTransferRequest) verifyChecksum(matches string) (bool, string, error) {
|
||||||
file, err := os.Open(str.path())
|
f, err := os.Open(str.path())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, "", err
|
return false, "", err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer f.Close()
|
||||||
hash := sha256.New()
|
h := sha256.New()
|
||||||
buf := make([]byte, 1024*4)
|
if _, err := io.Copy(h, bufio.NewReader(f)); err != nil {
|
||||||
if _, err := io.CopyBuffer(hash, file, buf); err != nil {
|
|
||||||
return false, "", err
|
return false, "", err
|
||||||
}
|
}
|
||||||
checksum := hex.EncodeToString(hash.Sum(nil))
|
checksum := hex.EncodeToString(h.Sum(nil))
|
||||||
return checksum == matches, checksum, nil
|
return checksum == matches, checksum, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -362,7 +386,7 @@ func postTransfer(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer res.Body.Close()
|
defer res.Body.Close()
|
||||||
if res.StatusCode != 200 {
|
if res.StatusCode != http.StatusOK {
|
||||||
data.log().WithField("error", err).WithField("status", res.StatusCode).Error("unexpected error response from transfer endpoint")
|
data.log().WithField("error", err).WithField("status", res.StatusCode).Error("unexpected error response from transfer endpoint")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,120 +0,0 @@
|
||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"emperror.dev/errors"
|
|
||||||
"github.com/mholt/archiver/v3"
|
|
||||||
"github.com/pterodactyl/wings/config"
|
|
||||||
"github.com/pterodactyl/wings/server/filesystem"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Archiver represents a Server Archiver.
|
|
||||||
type Archiver struct {
|
|
||||||
Server *Server
|
|
||||||
}
|
|
||||||
|
|
||||||
// Path returns the path to the server's archive.
|
|
||||||
func (a *Archiver) Path() string {
|
|
||||||
return filepath.Join(config.Get().System.ArchiveDirectory, a.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name returns the name of the server's archive.
|
|
||||||
func (a *Archiver) Name() string {
|
|
||||||
return a.Server.Id() + ".tar.gz"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exists returns a boolean based off if the archive exists.
|
|
||||||
func (a *Archiver) Exists() bool {
|
|
||||||
if _, err := os.Stat(a.Path()); os.IsNotExist(err) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stat stats the archive file.
|
|
||||||
func (a *Archiver) Stat() (*filesystem.Stat, error) {
|
|
||||||
s, err := os.Stat(a.Path())
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &filesystem.Stat{
|
|
||||||
FileInfo: s,
|
|
||||||
Mimetype: "application/tar+gzip",
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Archive creates an archive of the server and deletes the previous one.
|
|
||||||
func (a *Archiver) Archive() error {
|
|
||||||
path := a.Server.Filesystem().Path()
|
|
||||||
|
|
||||||
// Get the list of root files and directories to archive.
|
|
||||||
var files []string
|
|
||||||
fileInfo, err := ioutil.ReadDir(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, file := range fileInfo {
|
|
||||||
f := filepath.Join(path, file.Name())
|
|
||||||
// If the file is a symlink we cannot safely assume that the result of a filepath.Join() will be
|
|
||||||
// a safe destination. We need to check if the file is a symlink, and if so pass off to the SafePath
|
|
||||||
// function to resolve it to the final destination.
|
|
||||||
//
|
|
||||||
// ioutil.ReadDir() calls Lstat, so this will work correctly. If it did not call Lstat, but rather
|
|
||||||
// just did a normal Stat call, this would fail since that would be looking at the symlink destination
|
|
||||||
// and not the actual file in this listing.
|
|
||||||
if file.Mode()&os.ModeSymlink != 0 {
|
|
||||||
f, err = a.Server.Filesystem().SafePath(filepath.Join(path, file.Name()))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
files = append(files, f)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := a.DeleteIfExists(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return archiver.NewTarGz().Archive(files, a.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteIfExists deletes the archive if it exists.
|
|
||||||
func (a *Archiver) DeleteIfExists() error {
|
|
||||||
if _, err := a.Stat(); err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.WithMessage(os.Remove(a.Path()), "archiver: failed to delete archive from system")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checksum computes a SHA256 checksum of the server's archive.
|
|
||||||
func (a *Archiver) Checksum() (string, error) {
|
|
||||||
file, err := os.Open(a.Path())
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer file.Close()
|
|
||||||
|
|
||||||
hash := sha256.New()
|
|
||||||
|
|
||||||
buf := make([]byte, 1024*4)
|
|
||||||
if _, err := io.CopyBuffer(hash, file, buf); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
|
||||||
}
|
|
|
@ -2,6 +2,7 @@ package backup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
@ -56,7 +57,7 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) {
|
||||||
// Generate generates a backup of the selected files and pushes it to the
|
// Generate generates a backup of the selected files and pushes it to the
|
||||||
// defined location for this instance.
|
// defined location for this instance.
|
||||||
func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
|
func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
|
||||||
a := &Archive{
|
a := &filesystem.Archive{
|
||||||
BasePath: basePath,
|
BasePath: basePath,
|
||||||
Ignore: ignore,
|
Ignore: ignore,
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
@ -47,7 +48,7 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) {
|
||||||
func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
|
func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
|
||||||
defer s.Remove()
|
defer s.Remove()
|
||||||
|
|
||||||
a := &Archive{
|
a := &filesystem.Archive{
|
||||||
BasePath: basePath,
|
BasePath: basePath,
|
||||||
Ignore: ignore,
|
Ignore: ignore,
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package backup
|
package filesystem
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mholt/archiver/v3"
|
"github.com/mholt/archiver/v3"
|
||||||
"github.com/pterodactyl/wings/server/backup"
|
|
||||||
"github.com/pterodactyl/wings/system"
|
"github.com/pterodactyl/wings/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -39,7 +38,7 @@ func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, er
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
a := &backup.Archive{BasePath: cleanedRootDir, Files: cleaned}
|
a := &Archive{BasePath: cleanedRootDir, Files: cleaned}
|
||||||
d := path.Join(
|
d := path.Join(
|
||||||
cleanedRootDir,
|
cleanedRootDir,
|
||||||
fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")),
|
fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")),
|
||||||
|
@ -144,4 +143,3 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -175,7 +175,6 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server,
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
s.Archiver = Archiver{Server: s}
|
|
||||||
s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist)
|
s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist)
|
||||||
|
|
||||||
// Right now we only support a Docker based environment, so I'm going to hard code
|
// Right now we only support a Docker based environment, so I'm going to hard code
|
||||||
|
|
|
@ -42,7 +42,6 @@ type Server struct {
|
||||||
crasher CrashHandler
|
crasher CrashHandler
|
||||||
|
|
||||||
resources ResourceUsage
|
resources ResourceUsage
|
||||||
Archiver Archiver `json:"-"`
|
|
||||||
Environment environment.ProcessEnvironment `json:"-"`
|
Environment environment.ProcessEnvironment `json:"-"`
|
||||||
|
|
||||||
fs *filesystem.Filesystem
|
fs *filesystem.Filesystem
|
||||||
|
|
Loading…
Reference in New Issue
Block a user