Modify backup process to utilize contexts and exponential backoffs
If a request to upload a file part to S3 fails for any 5xx reason it will begin using an exponential backoff to keep re-trying the upload until we've reached a minute of trying to access the endpoint. This should resolve temporary resolution issues with URLs and certain S3 compatiable systems such as B2 that sometimes return a 5xx error and just need a retry to be successful. Also supports using the server context to ensure backups are terminated when a server is deleted, and removes the http call without a timeout, replacing it with a 2 hour timeout to account for connections as slow as 10Mbps on a huge file upload.
This commit is contained in:
Dane Everitt
@@ -1,16 +1,18 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"sync"
|
||||
|
||||
"emperror.dev/errors"
|
||||
"github.com/apex/log"
|
||||
"github.com/pterodactyl/wings/config"
|
||||
"github.com/pterodactyl/wings/remote"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type AdapterType string
|
||||
@@ -24,20 +26,37 @@ const (
|
||||
// and remote backups allowing the files to be restored.
|
||||
type RestoreCallback func(file string, r io.Reader) error
|
||||
|
||||
type ArchiveDetails struct {
|
||||
Checksum string `json:"checksum"`
|
||||
ChecksumType string `json:"checksum_type"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// ToRequest returns a request object.
|
||||
func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
|
||||
return remote.BackupRequest{
|
||||
Checksum: ad.Checksum,
|
||||
ChecksumType: ad.ChecksumType,
|
||||
Size: ad.Size,
|
||||
Successful: successful,
|
||||
}
|
||||
// noinspection GoNameStartsWithPackageName
|
||||
type BackupInterface interface {
|
||||
// SetClient sets the API request client on the backup interface.
|
||||
SetClient(c remote.Client)
|
||||
// Identifier returns the UUID of this backup as tracked by the panel
|
||||
// instance.
|
||||
Identifier() string
|
||||
// WithLogContext attaches additional context to the log output for this
|
||||
// backup.
|
||||
WithLogContext(map[string]interface{})
|
||||
// Generate creates a backup in whatever the configured source for the
|
||||
// specific implementation is.
|
||||
Generate(ctx context.Context, basePath string, ignore string) (*ArchiveDetails, error)
|
||||
// Ignored returns the ignored files for this backup instance.
|
||||
Ignored() string
|
||||
// Checksum returns a SHA1 checksum for the generated backup.
|
||||
Checksum() ([]byte, error)
|
||||
// Size returns the size of the generated backup.
|
||||
Size() (int64, error)
|
||||
// Path returns the path to the backup on the machine. This is not always
|
||||
// the final storage location of the backup, simply the location we're using
|
||||
// to store it until it is moved to the final spot.
|
||||
Path() string
|
||||
// Details returns details about the archive.
|
||||
Details(ctx context.Context) (*ArchiveDetails, error)
|
||||
// Remove removes a backup file.
|
||||
Remove() error
|
||||
// Restore is called when a backup is ready to be restored to the disk from
|
||||
// the given source. Not every backup implementation will support this nor
|
||||
// will every implementation require a reader be provided.
|
||||
Restore(ctx context.Context, reader io.Reader, callback RestoreCallback) error
|
||||
}
|
||||
|
||||
type Backup struct {
|
||||
@@ -54,39 +73,6 @@ type Backup struct {
|
||||
logContext map[string]interface{}
|
||||
}
|
||||
|
||||
// noinspection GoNameStartsWithPackageName
|
||||
type BackupInterface interface {
|
||||
// SetClient sets the API request client on the backup interface.
|
||||
SetClient(c remote.Client)
|
||||
// Identifier returns the UUID of this backup as tracked by the panel
|
||||
// instance.
|
||||
Identifier() string
|
||||
// WithLogContext attaches additional context to the log output for this
|
||||
// backup.
|
||||
WithLogContext(map[string]interface{})
|
||||
// Generate creates a backup in whatever the configured source for the
|
||||
// specific implementation is.
|
||||
Generate(string, string) (*ArchiveDetails, error)
|
||||
// Ignored returns the ignored files for this backup instance.
|
||||
Ignored() string
|
||||
// Checksum returns a SHA1 checksum for the generated backup.
|
||||
Checksum() ([]byte, error)
|
||||
// Size returns the size of the generated backup.
|
||||
Size() (int64, error)
|
||||
// Path returns the path to the backup on the machine. This is not always
|
||||
// the final storage location of the backup, simply the location we're using
|
||||
// to store it until it is moved to the final spot.
|
||||
Path() string
|
||||
// Details returns details about the archive.
|
||||
Details() *ArchiveDetails
|
||||
// Remove removes a backup file.
|
||||
Remove() error
|
||||
// Restore is called when a backup is ready to be restored to the disk from
|
||||
// the given source. Not every backup implementation will support this nor
|
||||
// will every implementation require a reader be provided.
|
||||
Restore(reader io.Reader, callback RestoreCallback) error
|
||||
}
|
||||
|
||||
func (b *Backup) SetClient(c remote.Client) {
|
||||
b.client = c
|
||||
}
|
||||
@@ -95,12 +81,12 @@ func (b *Backup) Identifier() string {
|
||||
return b.Uuid
|
||||
}
|
||||
|
||||
// Returns the path for this specific backup.
|
||||
// Path returns the path for this specific backup.
|
||||
func (b *Backup) Path() string {
|
||||
return path.Join(config.Get().System.BackupDirectory, b.Identifier()+".tar.gz")
|
||||
}
|
||||
|
||||
// Return the size of the generated backup.
|
||||
// Size returns the size of the generated backup.
|
||||
func (b *Backup) Size() (int64, error) {
|
||||
st, err := os.Stat(b.Path())
|
||||
if err != nil {
|
||||
@@ -110,7 +96,7 @@ func (b *Backup) Size() (int64, error) {
|
||||
return st.Size(), nil
|
||||
}
|
||||
|
||||
// Returns the SHA256 checksum of a backup.
|
||||
// Checksum returns the SHA256 checksum of a backup.
|
||||
func (b *Backup) Checksum() ([]byte, error) {
|
||||
h := sha1.New()
|
||||
|
||||
@@ -128,51 +114,34 @@ func (b *Backup) Checksum() ([]byte, error) {
|
||||
return h.Sum(nil), nil
|
||||
}
|
||||
|
||||
// Returns details of the archive by utilizing two go-routines to get the checksum and
|
||||
// the size of the archive.
|
||||
func (b *Backup) Details() *ArchiveDetails {
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(2)
|
||||
// Details returns both the checksum and size of the archive currently stored on
|
||||
// the disk to the caller.
|
||||
func (b *Backup) Details(ctx context.Context) (*ArchiveDetails, error) {
|
||||
ad := ArchiveDetails{ChecksumType: "sha1"}
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
|
||||
l := log.WithField("backup_id", b.Uuid)
|
||||
|
||||
var checksum string
|
||||
// Calculate the checksum for the file.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
l.Info("computing checksum for backup...")
|
||||
g.Go(func() error {
|
||||
resp, err := b.Checksum()
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"backup": b.Identifier(),
|
||||
"error": err,
|
||||
}).Error("failed to calculate checksum for backup")
|
||||
return
|
||||
return err
|
||||
}
|
||||
ad.Checksum = hex.EncodeToString(resp)
|
||||
return nil
|
||||
})
|
||||
|
||||
checksum = hex.EncodeToString(resp)
|
||||
l.WithField("checksum", checksum).Info("computed checksum for backup")
|
||||
}()
|
||||
|
||||
var sz int64
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
if s, err := b.Size(); err != nil {
|
||||
return
|
||||
} else {
|
||||
sz = s
|
||||
g.Go(func() error {
|
||||
s, err := b.Size()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}()
|
||||
ad.Size = s
|
||||
return nil
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return &ArchiveDetails{
|
||||
Checksum: checksum,
|
||||
ChecksumType: "sha1",
|
||||
Size: sz,
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, errors.WithStackDepth(err, 1)
|
||||
}
|
||||
return &ad, nil
|
||||
}
|
||||
|
||||
func (b *Backup) Ignored() string {
|
||||
@@ -188,3 +157,19 @@ func (b *Backup) log() *log.Entry {
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
type ArchiveDetails struct {
|
||||
Checksum string `json:"checksum"`
|
||||
ChecksumType string `json:"checksum_type"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// ToRequest returns a request object.
|
||||
func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
|
||||
return remote.BackupRequest{
|
||||
Checksum: ad.Checksum,
|
||||
ChecksumType: ad.ChecksumType,
|
||||
Size: ad.Size,
|
||||
Successful: successful,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user