MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backups(s3): send uploaded parts in complete request

Browse Source
This commit is contained in:
Matthew Penner 2022-09-26 11:14:57 -06:00
parent 02cbf2df5b
commit 7245791214
No known key found for this signature in database
GPG Key ID: 31311906AD4CF6D6
5 changed files with 44 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
remote/servers.go
View File

@ -3,10 +3,11 @@ package remote
import (
"context"
"fmt"
"github.com/pterodactyl/wings/internal/models"
"strconv"
"sync"
"github.com/pterodactyl/wings/internal/models"
"emperror.dev/errors"
"github.com/apex/log"
"golang.org/x/sync/errgroup"

21
remote/types.go
View File

@ -2,11 +2,12 @@ package remote
import (
"bytes"
"github.com/apex/log"
"github.com/goccy/go-json"
"regexp"
"strings"
"github.com/apex/log"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/parser"
)
@ -156,9 +157,15 @@ type BackupRemoteUploadResponse struct {
PartSize int64 `json:"part_size"`
}
type BackupRequest struct {
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
Successful bool `json:"successful"`
type BackupPart struct {
ETag string `json:"etag"`
PartNumber int `json:"part_number"`
}
type BackupRequest struct {
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
Successful bool `json:"successful"`
Parts []BackupPart `json:"parts"`
}

20
server/backup/backup.go
View File

@ -32,7 +32,7 @@ type RestoreCallback func(file string, r io.Reader, mode fs.FileMode, atime, mti
// noinspection GoNameStartsWithPackageName
type BackupInterface interface {
// SetClient sets the API request client on the backup interface.
SetClient(c remote.Client)
SetClient(remote.Client)
// Identifier returns the UUID of this backup as tracked by the panel
// instance.
Identifier() string
@ -41,7 +41,7 @@ type BackupInterface interface {
WithLogContext(map[string]interface{})
// Generate creates a backup in whatever the configured source for the
// specific implementation is.
Generate(ctx context.Context, basePath string, ignore string) (*ArchiveDetails, error)
Generate(context.Context, string, string) (*ArchiveDetails, error)
// Ignored returns the ignored files for this backup instance.
Ignored() string
// Checksum returns a SHA1 checksum for the generated backup.
@ -53,13 +53,13 @@ type BackupInterface interface {
// to store it until it is moved to the final spot.
Path() string
// Details returns details about the archive.
Details(ctx context.Context) (*ArchiveDetails, error)
Details(context.Context, []remote.BackupPart) (*ArchiveDetails, error)
// Remove removes a backup file.
Remove() error
// Restore is called when a backup is ready to be restored to the disk from
// the given source. Not every backup implementation will support this nor
// will every implementation require a reader be provided.
Restore(ctx context.Context, reader io.Reader, callback RestoreCallback) error
Restore(context.Context, io.Reader, RestoreCallback) error
}
type Backup struct {
@ -119,8 +119,8 @@ func (b *Backup) Checksum() ([]byte, error) {
// Details returns both the checksum and size of the archive currently stored on
// the disk to the caller.
func (b *Backup) Details(ctx context.Context) (*ArchiveDetails, error) {
ad := ArchiveDetails{ChecksumType: "sha1"}
func (b *Backup) Details(ctx context.Context, parts []remote.BackupPart) (*ArchiveDetails, error) {
ad := ArchiveDetails{ChecksumType: "sha1", Parts: parts}
g, ctx := errgroup.WithContext(ctx)
g.Go(func() error {
@ -162,9 +162,10 @@ func (b *Backup) log() *log.Entry {
}
type ArchiveDetails struct {
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
Checksum string `json:"checksum"`
ChecksumType string `json:"checksum_type"`
Size int64 `json:"size"`
Parts []remote.BackupPart `json:"parts"`
}
// ToRequest returns a request object.
@ -174,5 +175,6 @@ func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
ChecksumType: ad.ChecksumType,
Size: ad.Size,
Successful: successful,
Parts: ad.Parts,
}
}

2
server/backup/backup_local.go
View File

@ -69,7 +69,7 @@ func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*A
}
b.log().Info("created backup successfully")
ad, err := b.Details(ctx)
ad, err := b.Details(ctx, nil)
if err != nil {
return nil, errors.WrapIf(err, "backup: failed to get archive details for local backup")
}

27
server/backup/backup_s3.go
View File

@ -71,10 +71,11 @@ func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*Arch
}
defer rc.Close()
if err := s.generateRemoteRequest(ctx, rc); err != nil {
parts, err := s.generateRemoteRequest(ctx, rc)
if err != nil {
return nil, err
}
ad, err := s.Details(ctx)
ad, err := s.Details(ctx, parts)
if err != nil {
return nil, errors.WrapIf(err, "backup: failed to get archive details after upload")
}
@ -125,20 +126,20 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal
}
// Generates the remote S3 request and begins the upload.
func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) error {
func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) ([]remote.BackupPart, error) {
defer rc.Close()
s.log().Debug("attempting to get size of backup...")
size, err := s.Backup.Size()
if err != nil {
return err
return nil, err
}
s.log().WithField("size", size).Debug("got size of backup")
s.log().Debug("attempting to get S3 upload urls from Panel...")
urls, err := s.client.GetBackupRemoteUploadURLs(context.Background(), s.Backup.Uuid, size)
if err != nil {
return err
return nil, err
}
s.log().Debug("got S3 upload urls from the Panel")
s.log().WithField("parts", len(urls.Parts)).Info("attempting to upload backup to s3 endpoint...")
@ -156,22 +157,26 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser)
}
// Attempt to upload the part.
if _, err := uploader.uploadPart(ctx, part, partSize); err != nil {
etag, err := uploader.uploadPart(ctx, part, partSize)
if err != nil {
s.log().WithField("part_id", i+1).WithError(err).Warn("failed to upload part")
return err
return nil, err
}
uploader.uploadedParts = append(uploader.uploadedParts, remote.BackupPart{
ETag: etag,
PartNumber: i + 1,
})
s.log().WithField("part_id", i+1).Info("successfully uploaded backup part")
}
s.log().WithField("parts", len(urls.Parts)).Info("backup has been successfully uploaded")
return nil
return uploader.uploadedParts, nil
}
type s3FileUploader struct {
io.ReadCloser
client *http.Client
client *http.Client
uploadedParts []remote.BackupPart
}
// newS3FileUploader returns a new file uploader instance.