diff --git a/remote/servers.go b/remote/servers.go index 776113a..0b7aca2 100644 --- a/remote/servers.go +++ b/remote/servers.go @@ -3,10 +3,11 @@ package remote import ( "context" "fmt" - "github.com/pterodactyl/wings/internal/models" "strconv" "sync" + "github.com/pterodactyl/wings/internal/models" + "emperror.dev/errors" "github.com/apex/log" "golang.org/x/sync/errgroup" diff --git a/remote/types.go b/remote/types.go index 7e2773f..afbb974 100644 --- a/remote/types.go +++ b/remote/types.go @@ -2,11 +2,12 @@ package remote import ( "bytes" - "github.com/apex/log" - "github.com/goccy/go-json" "regexp" "strings" + "github.com/apex/log" + "github.com/goccy/go-json" + "github.com/pterodactyl/wings/parser" ) @@ -156,9 +157,15 @@ type BackupRemoteUploadResponse struct { PartSize int64 `json:"part_size"` } -type BackupRequest struct { - Checksum string `json:"checksum"` - ChecksumType string `json:"checksum_type"` - Size int64 `json:"size"` - Successful bool `json:"successful"` +type BackupPart struct { + ETag string `json:"etag"` + PartNumber int `json:"part_number"` +} + +type BackupRequest struct { + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` + Successful bool `json:"successful"` + Parts []BackupPart `json:"parts"` } diff --git a/server/backup/backup.go b/server/backup/backup.go index 496720e..78f4280 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -32,7 +32,7 @@ type RestoreCallback func(file string, r io.Reader, mode fs.FileMode, atime, mti // noinspection GoNameStartsWithPackageName type BackupInterface interface { // SetClient sets the API request client on the backup interface. - SetClient(c remote.Client) + SetClient(remote.Client) // Identifier returns the UUID of this backup as tracked by the panel // instance. Identifier() string @@ -41,7 +41,7 @@ type BackupInterface interface { WithLogContext(map[string]interface{}) // Generate creates a backup in whatever the configured source for the // specific implementation is. - Generate(ctx context.Context, basePath string, ignore string) (*ArchiveDetails, error) + Generate(context.Context, string, string) (*ArchiveDetails, error) // Ignored returns the ignored files for this backup instance. Ignored() string // Checksum returns a SHA1 checksum for the generated backup. @@ -53,13 +53,13 @@ type BackupInterface interface { // to store it until it is moved to the final spot. Path() string // Details returns details about the archive. - Details(ctx context.Context) (*ArchiveDetails, error) + Details(context.Context, []remote.BackupPart) (*ArchiveDetails, error) // Remove removes a backup file. Remove() error // Restore is called when a backup is ready to be restored to the disk from // the given source. Not every backup implementation will support this nor // will every implementation require a reader be provided. - Restore(ctx context.Context, reader io.Reader, callback RestoreCallback) error + Restore(context.Context, io.Reader, RestoreCallback) error } type Backup struct { @@ -119,8 +119,8 @@ func (b *Backup) Checksum() ([]byte, error) { // Details returns both the checksum and size of the archive currently stored on // the disk to the caller. -func (b *Backup) Details(ctx context.Context) (*ArchiveDetails, error) { - ad := ArchiveDetails{ChecksumType: "sha1"} +func (b *Backup) Details(ctx context.Context, parts []remote.BackupPart) (*ArchiveDetails, error) { + ad := ArchiveDetails{ChecksumType: "sha1", Parts: parts} g, ctx := errgroup.WithContext(ctx) g.Go(func() error { @@ -162,9 +162,10 @@ func (b *Backup) log() *log.Entry { } type ArchiveDetails struct { - Checksum string `json:"checksum"` - ChecksumType string `json:"checksum_type"` - Size int64 `json:"size"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + Size int64 `json:"size"` + Parts []remote.BackupPart `json:"parts"` } // ToRequest returns a request object. @@ -174,5 +175,6 @@ func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest { ChecksumType: ad.ChecksumType, Size: ad.Size, Successful: successful, + Parts: ad.Parts, } } diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index c652b13..37d70d2 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -69,7 +69,7 @@ func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*A } b.log().Info("created backup successfully") - ad, err := b.Details(ctx) + ad, err := b.Details(ctx, nil) if err != nil { return nil, errors.WrapIf(err, "backup: failed to get archive details for local backup") } diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 85faa59..304d323 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -71,10 +71,11 @@ func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*Arch } defer rc.Close() - if err := s.generateRemoteRequest(ctx, rc); err != nil { + parts, err := s.generateRemoteRequest(ctx, rc) + if err != nil { return nil, err } - ad, err := s.Details(ctx) + ad, err := s.Details(ctx, parts) if err != nil { return nil, errors.WrapIf(err, "backup: failed to get archive details after upload") } @@ -125,20 +126,20 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal } // Generates the remote S3 request and begins the upload. -func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) error { +func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) ([]remote.BackupPart, error) { defer rc.Close() s.log().Debug("attempting to get size of backup...") size, err := s.Backup.Size() if err != nil { - return err + return nil, err } s.log().WithField("size", size).Debug("got size of backup") s.log().Debug("attempting to get S3 upload urls from Panel...") urls, err := s.client.GetBackupRemoteUploadURLs(context.Background(), s.Backup.Uuid, size) if err != nil { - return err + return nil, err } s.log().Debug("got S3 upload urls from the Panel") s.log().WithField("parts", len(urls.Parts)).Info("attempting to upload backup to s3 endpoint...") @@ -156,22 +157,26 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) } // Attempt to upload the part. - if _, err := uploader.uploadPart(ctx, part, partSize); err != nil { + etag, err := uploader.uploadPart(ctx, part, partSize) + if err != nil { s.log().WithField("part_id", i+1).WithError(err).Warn("failed to upload part") - return err + return nil, err } - + uploader.uploadedParts = append(uploader.uploadedParts, remote.BackupPart{ + ETag: etag, + PartNumber: i + 1, + }) s.log().WithField("part_id", i+1).Info("successfully uploaded backup part") } - s.log().WithField("parts", len(urls.Parts)).Info("backup has been successfully uploaded") - return nil + return uploader.uploadedParts, nil } type s3FileUploader struct { io.ReadCloser - client *http.Client + client *http.Client + uploadedParts []remote.BackupPart } // newS3FileUploader returns a new file uploader instance.