backups(s3): send uploaded parts in complete request

remote/servers.go

@@ -3,10 +3,11 @@ package remote
 import (
 	"context"
 	"fmt"
-	"github.com/pterodactyl/wings/internal/models"
 	"strconv"
 	"sync"
 
+	"github.com/pterodactyl/wings/internal/models"
+
 	"emperror.dev/errors"
 	"github.com/apex/log"
 	"golang.org/x/sync/errgroup"

remote/types.go

@@ -2,11 +2,12 @@ package remote
 
 import (
 	"bytes"
-	"github.com/apex/log"
-	"github.com/goccy/go-json"
 	"regexp"
 	"strings"
 
+	"github.com/apex/log"
+	"github.com/goccy/go-json"
+
 	"github.com/pterodactyl/wings/parser"
 )
 
@@ -156,9 +157,15 @@ type BackupRemoteUploadResponse struct {
 	PartSize int64    `json:"part_size"`
 }
 
-type BackupRequest struct {
+type BackupPart struct {
-	Checksum     string `json:"checksum"`
+	ETag       string `json:"etag"`
-	ChecksumType string `json:"checksum_type"`
+	PartNumber int    `json:"part_number"`
-	Size         int64  `json:"size"`
+}
-	Successful   bool   `json:"successful"`
+
+type BackupRequest struct {
+	Checksum     string       `json:"checksum"`
+	ChecksumType string       `json:"checksum_type"`
+	Size         int64        `json:"size"`
+	Successful   bool         `json:"successful"`
+	Parts        []BackupPart `json:"parts"`
 }

server/backup/backup.go

@@ -32,7 +32,7 @@ type RestoreCallback func(file string, r io.Reader, mode fs.FileMode, atime, mti
 // noinspection GoNameStartsWithPackageName
 type BackupInterface interface {
 	// SetClient sets the API request client on the backup interface.
-	SetClient(c remote.Client)
+	SetClient(remote.Client)
 	// Identifier returns the UUID of this backup as tracked by the panel
 	// instance.
 	Identifier() string
@@ -41,7 +41,7 @@ type BackupInterface interface {
 	WithLogContext(map[string]interface{})
 	// Generate creates a backup in whatever the configured source for the
 	// specific implementation is.
-	Generate(ctx context.Context, basePath string, ignore string) (*ArchiveDetails, error)
+	Generate(context.Context, string, string) (*ArchiveDetails, error)
 	// Ignored returns the ignored files for this backup instance.
 	Ignored() string
 	// Checksum returns a SHA1 checksum for the generated backup.
@@ -53,13 +53,13 @@ type BackupInterface interface {
 	// to store it until it is moved to the final spot.
 	Path() string
 	// Details returns details about the archive.
-	Details(ctx context.Context) (*ArchiveDetails, error)
+	Details(context.Context, []remote.BackupPart) (*ArchiveDetails, error)
 	// Remove removes a backup file.
 	Remove() error
 	// Restore is called when a backup is ready to be restored to the disk from
 	// the given source. Not every backup implementation will support this nor
 	// will every implementation require a reader be provided.
-	Restore(ctx context.Context, reader io.Reader, callback RestoreCallback) error
+	Restore(context.Context, io.Reader, RestoreCallback) error
 }
 
 type Backup struct {
@@ -119,8 +119,8 @@ func (b *Backup) Checksum() ([]byte, error) {
 
 // Details returns both the checksum and size of the archive currently stored on
 // the disk to the caller.
-func (b *Backup) Details(ctx context.Context) (*ArchiveDetails, error) {
+func (b *Backup) Details(ctx context.Context, parts []remote.BackupPart) (*ArchiveDetails, error) {
-	ad := ArchiveDetails{ChecksumType: "sha1"}
+	ad := ArchiveDetails{ChecksumType: "sha1", Parts: parts}
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
@@ -162,9 +162,10 @@ func (b *Backup) log() *log.Entry {
 }
 
 type ArchiveDetails struct {
-	Checksum     string `json:"checksum"`
+	Checksum     string              `json:"checksum"`
-	ChecksumType string `json:"checksum_type"`
+	ChecksumType string              `json:"checksum_type"`
-	Size         int64  `json:"size"`
+	Size         int64               `json:"size"`
+	Parts        []remote.BackupPart `json:"parts"`
 }
 
 // ToRequest returns a request object.
@@ -174,5 +175,6 @@ func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
 		ChecksumType: ad.ChecksumType,
 		Size:         ad.Size,
 		Successful:   successful,
+		Parts:        ad.Parts,
 	}
 }

server/backup/backup_local.go

@@ -69,7 +69,7 @@ func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*A
 	}
 	b.log().Info("created backup successfully")
 
-	ad, err := b.Details(ctx)
+	ad, err := b.Details(ctx, nil)
 	if err != nil {
 		return nil, errors.WrapIf(err, "backup: failed to get archive details for local backup")
 	}

server/backup/backup_s3.go

@@ -71,10 +71,11 @@ func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*Arch
 	}
 	defer rc.Close()
 
-	if err := s.generateRemoteRequest(ctx, rc); err != nil {
+	parts, err := s.generateRemoteRequest(ctx, rc)
+	if err != nil {
 		return nil, err
 	}
-	ad, err := s.Details(ctx)
+	ad, err := s.Details(ctx, parts)
 	if err != nil {
 		return nil, errors.WrapIf(err, "backup: failed to get archive details after upload")
 	}
@@ -125,20 +126,20 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal
 }
 
 // Generates the remote S3 request and begins the upload.
-func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) error {
+func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) ([]remote.BackupPart, error) {
 	defer rc.Close()
 
 	s.log().Debug("attempting to get size of backup...")
 	size, err := s.Backup.Size()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	s.log().WithField("size", size).Debug("got size of backup")
 
 	s.log().Debug("attempting to get S3 upload urls from Panel...")
 	urls, err := s.client.GetBackupRemoteUploadURLs(context.Background(), s.Backup.Uuid, size)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	s.log().Debug("got S3 upload urls from the Panel")
 	s.log().WithField("parts", len(urls.Parts)).Info("attempting to upload backup to s3 endpoint...")
@@ -156,22 +157,26 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser)
 		}
 
 		// Attempt to upload the part.
-		if _, err := uploader.uploadPart(ctx, part, partSize); err != nil {
+		etag, err := uploader.uploadPart(ctx, part, partSize)
+		if err != nil {
 			s.log().WithField("part_id", i+1).WithError(err).Warn("failed to upload part")
-			return err
+			return nil, err
 		}
-
+		uploader.uploadedParts = append(uploader.uploadedParts, remote.BackupPart{
+			ETag:       etag,
+			PartNumber: i + 1,
+		})
 		s.log().WithField("part_id", i+1).Info("successfully uploaded backup part")
 	}
-
 	s.log().WithField("parts", len(urls.Parts)).Info("backup has been successfully uploaded")
 
-	return nil
+	return uploader.uploadedParts, nil
 }
 
 type s3FileUploader struct {
 	io.ReadCloser
-	client *http.Client
+	client        *http.Client
+	uploadedParts []remote.BackupPart
 }
 
 // newS3FileUploader returns a new file uploader instance.