Switch to s3 multipart uploads for backups
This commit is contained in:
parent
b8598e90d4
commit
1239b1c0ca
|
@ -185,7 +185,7 @@ func (r *Response) Error() error {
|
|||
var bag RequestErrorBag
|
||||
_ = r.Bind(&bag)
|
||||
|
||||
e := new(RequestError)
|
||||
e := &RequestError{}
|
||||
if len(bag.Errors) > 0 {
|
||||
e = &bag.Errors[0]
|
||||
}
|
||||
|
|
|
@ -3,8 +3,35 @@ package api
|
|||
import (
|
||||
"fmt"
|
||||
"github.com/pkg/errors"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type BackupRemoteUploadResponse struct {
|
||||
CompleteMultipartUpload string
|
||||
AbortMultipartUpload string
|
||||
Parts []string
|
||||
PartSize int64
|
||||
}
|
||||
|
||||
func (r *Request) GetBackupRemoteUploadURLs(backup string, size int64) (*BackupRemoteUploadResponse, error) {
|
||||
resp, err := r.Get(fmt.Sprintf("/backups/%s", backup), Q{"size": strconv.FormatInt(size, 10)})
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.HasError() {
|
||||
return nil, resp.Error()
|
||||
}
|
||||
|
||||
var res BackupRemoteUploadResponse
|
||||
if err := resp.Bind(&res); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
type BackupRequest struct {
|
||||
Checksum string `json:"checksum"`
|
||||
ChecksumType string `json:"checksum_type"`
|
||||
|
|
|
@ -9,7 +9,6 @@ type Request struct {
|
|||
Adapter string `json:"adapter"`
|
||||
Uuid string `json:"uuid"`
|
||||
IgnoredFiles []string `json:"ignored_files"`
|
||||
PresignedUrl string `json:"presigned_url"`
|
||||
}
|
||||
|
||||
// Generates a new local backup struct.
|
||||
|
@ -32,15 +31,10 @@ func (r *Request) NewS3Backup() (*S3Backup, error) {
|
|||
return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter))
|
||||
}
|
||||
|
||||
if len(r.PresignedUrl) == 0 {
|
||||
return nil, errors.New("a valid presigned S3 upload URL must be provided to use the [s3] adapter")
|
||||
}
|
||||
|
||||
return &S3Backup{
|
||||
Backup: Backup{
|
||||
Uuid: r.Uuid,
|
||||
IgnoredFiles: r.IgnoredFiles,
|
||||
},
|
||||
PresignedUrl: r.PresignedUrl,
|
||||
}, nil
|
||||
}
|
||||
|
|
|
@ -1,10 +1,12 @@
|
|||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/apex/log"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pterodactyl/wings/api"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
@ -13,12 +15,6 @@ import (
|
|||
|
||||
type S3Backup struct {
|
||||
Backup
|
||||
|
||||
// The pre-signed upload endpoint for the generated backup. This must be
|
||||
// provided otherwise this request will fail. This allows us to keep all
|
||||
// of the keys off the daemon instances and the panel can handle generating
|
||||
// the credentials for us.
|
||||
PresignedUrl string
|
||||
}
|
||||
|
||||
var _ BackupInterface = (*S3Backup)(nil)
|
||||
|
@ -43,14 +39,8 @@ func (s *S3Backup) Generate(included *IncludedFiles, prefix string) (*ArchiveDet
|
|||
}
|
||||
defer rc.Close()
|
||||
|
||||
if resp, err := s.generateRemoteRequest(rc); err != nil {
|
||||
if err := s.generateRemoteRequest(rc); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
return s.Details(), err
|
||||
|
@ -61,27 +51,123 @@ func (s *S3Backup) Remove() error {
|
|||
return os.Remove(s.Path())
|
||||
}
|
||||
|
||||
// Reader provides a wrapper around an existing io.Reader
|
||||
// but implements io.Closer in order to satisfy an io.ReadCloser.
|
||||
type Reader struct {
|
||||
io.Reader
|
||||
}
|
||||
|
||||
func (Reader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Generates the remote S3 request and begins the upload.
|
||||
func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) (*http.Response, error) {
|
||||
r, err := http.NewRequest(http.MethodPut, s.PresignedUrl, nil)
|
||||
func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
|
||||
defer rc.Close()
|
||||
|
||||
size, err := s.Backup.Size()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
if sz, err := s.Size(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
r.ContentLength = sz
|
||||
r.Header.Add("Content-Length", strconv.Itoa(int(sz)))
|
||||
urls, err := api.New().GetBackupRemoteUploadURLs(s.Backup.Uuid, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("attempting to upload backup to remote S3 endpoint")
|
||||
|
||||
handlePart := func(part string, size int64) (string, error) {
|
||||
r, err := http.NewRequest(http.MethodPut, part, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
r.ContentLength = size
|
||||
r.Header.Add("Content-Length", strconv.Itoa(int(size)))
|
||||
r.Header.Add("Content-Type", "application/x-gzip")
|
||||
|
||||
r.Body = Reader{io.LimitReader(rc, size)}
|
||||
|
||||
res, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
r.Body = rc
|
||||
defer res.Body.Close()
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"endpoint": s.PresignedUrl,
|
||||
"headers": r.Header,
|
||||
}).Debug("uploading backup to remote S3 endpoint")
|
||||
|
||||
return http.DefaultClient.Do(r)
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("failed to put S3 object part, %d:%s", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
return res.Header.Get("ETag"), nil
|
||||
}
|
||||
|
||||
// Keep track of errors from individual part uploads.
|
||||
hasError := true
|
||||
defer func() {
|
||||
if !hasError {
|
||||
return
|
||||
}
|
||||
|
||||
r, err := http.NewRequest(http.MethodPost, urls.AbortMultipartUpload, nil)
|
||||
if err != nil {
|
||||
log.WithError(err).Warn("failed to create http request (AbortMultipartUpload)")
|
||||
return
|
||||
}
|
||||
|
||||
res, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
log.WithError(err).Warn("failed to make http request (AbortMultipartUpload)")
|
||||
return
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
log.Warnf("failed to abort S3 multipart upload, %d:%s", res.StatusCode, res.Status)
|
||||
}
|
||||
}()
|
||||
|
||||
var completeBody bytes.Buffer
|
||||
completeBody.WriteString("<CompleteMultipartUpload>\n")
|
||||
|
||||
partCount := len(urls.Parts)
|
||||
for i, part := range urls.Parts {
|
||||
var s int64
|
||||
if i+1 < partCount {
|
||||
s = urls.PartSize
|
||||
} else {
|
||||
s = size - (int64(i) * urls.PartSize)
|
||||
}
|
||||
|
||||
etag, err := handlePart(part, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
completeBody.WriteString("\t<Part>\n")
|
||||
completeBody.WriteString("\t\t<ETag>\"" + etag + "\"</ETag>\n")
|
||||
completeBody.WriteString("\t\t<PartNumber>" + strconv.Itoa(i+1) + "</PartNumber>\n")
|
||||
completeBody.WriteString("\t</Part>\n")
|
||||
}
|
||||
hasError = false
|
||||
|
||||
completeBody.WriteString("</CompleteMultipartUpload>")
|
||||
|
||||
r, err := http.NewRequest(http.MethodPost, urls.CompleteMultipartUpload, &completeBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := http.DefaultClient.Do(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("failed to complete S3 multipart upload, %d:%s", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user