MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backups(s3): send uploaded parts in complete request

Browse Source
This commit is contained in:
Matthew Penner
2022-09-26 11:14:57 -06:00
parent 02cbf2df5b
commit 7245791214
5 changed files with 44 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
server/backup/backup_s3.go
View File

@@ -71,10 +71,11 @@ func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*Arch
}
defer rc.Close()
if err := s.generateRemoteRequest(ctx, rc); err != nil {
parts, err := s.generateRemoteRequest(ctx, rc)
if err != nil {
return nil, err
}
ad, err := s.Details(ctx)
ad, err := s.Details(ctx, parts)
if err != nil {
return nil, errors.WrapIf(err, "backup: failed to get archive details after upload")
}
@@ -125,20 +126,20 @@ func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCal
}
// Generates the remote S3 request and begins the upload.
func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) error {
func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) ([]remote.BackupPart, error) {
defer rc.Close()
s.log().Debug("attempting to get size of backup...")
size, err := s.Backup.Size()
if err != nil {
return err
return nil, err
}
s.log().WithField("size", size).Debug("got size of backup")
s.log().Debug("attempting to get S3 upload urls from Panel...")
urls, err := s.client.GetBackupRemoteUploadURLs(context.Background(), s.Backup.Uuid, size)
if err != nil {
return err
return nil, err
}
s.log().Debug("got S3 upload urls from the Panel")
s.log().WithField("parts", len(urls.Parts)).Info("attempting to upload backup to s3 endpoint...")
@@ -156,22 +157,26 @@ func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser)
}
// Attempt to upload the part.
if _, err := uploader.uploadPart(ctx, part, partSize); err != nil {
etag, err := uploader.uploadPart(ctx, part, partSize)
if err != nil {
s.log().WithField("part_id", i+1).WithError(err).Warn("failed to upload part")
return err
return nil, err
}
uploader.uploadedParts = append(uploader.uploadedParts, remote.BackupPart{
ETag: etag,
PartNumber: i + 1,
})
s.log().WithField("part_id", i+1).Info("successfully uploaded backup part")
}
s.log().WithField("parts", len(urls.Parts)).Info("backup has been successfully uploaded")
return nil
return uploader.uploadedParts, nil
}
type s3FileUploader struct {
io.ReadCloser
client *http.Client
client *http.Client
uploadedParts []remote.BackupPart
}
// newS3FileUploader returns a new file uploader instance.