Switch to s3 multipart uploads for backups

api/api.go

@@ -185,7 +185,7 @@ func (r *Response) Error() error {
 	var bag RequestErrorBag
 	_ = r.Bind(&bag)
 
-	e := new(RequestError)
+	e := &RequestError{}
 	if len(bag.Errors) > 0 {
 		e = &bag.Errors[0]
 	}

api/backup_endpoints.go

@@ -3,8 +3,35 @@ package api
 import (
 	"fmt"
 	"github.com/pkg/errors"
+	"strconv"
 )
 
+type BackupRemoteUploadResponse struct {
+	CompleteMultipartUpload string
+	AbortMultipartUpload    string
+	Parts                   []string
+	PartSize                int64
+}
+
+func (r *Request) GetBackupRemoteUploadURLs(backup string, size int64) (*BackupRemoteUploadResponse, error) {
+	resp, err := r.Get(fmt.Sprintf("/backups/%s", backup), Q{"size": strconv.FormatInt(size, 10)})
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.HasError() {
+		return nil, resp.Error()
+	}
+
+	var res BackupRemoteUploadResponse
+	if err := resp.Bind(&res); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return &res, nil
+}
+
 type BackupRequest struct {
 	Checksum     string `json:"checksum"`
 	ChecksumType string `json:"checksum_type"`

server/backup/backup_request.go

@@ -9,7 +9,6 @@ type Request struct {
 	Adapter      string   `json:"adapter"`
 	Uuid         string   `json:"uuid"`
 	IgnoredFiles []string `json:"ignored_files"`
-	PresignedUrl string   `json:"presigned_url"`
 }
 
 // Generates a new local backup struct.
@@ -32,15 +31,10 @@ func (r *Request) NewS3Backup() (*S3Backup, error) {
 		return nil, errors.New(fmt.Sprintf("cannot create s3 backup using [%s] adapter", r.Adapter))
 	}
 
-	if len(r.PresignedUrl) == 0 {
-		return nil, errors.New("a valid presigned S3 upload URL must be provided to use the [s3] adapter")
-	}
-
 	return &S3Backup{
 		Backup: Backup{
 			Uuid:         r.Uuid,
 			IgnoredFiles: r.IgnoredFiles,
 		},
-		PresignedUrl: r.PresignedUrl,
 	}, nil
 }

server/backup/backup_s3.go

@@ -1,10 +1,12 @@
 package backup
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"github.com/apex/log"
 	"github.com/pkg/errors"
+	"github.com/pterodactyl/wings/api"
 	"io"
 	"net/http"
 	"os"
@@ -13,12 +15,6 @@ import (
 
 type S3Backup struct {
 	Backup
-
-	// The pre-signed upload endpoint for the generated backup. This must be
-	// provided otherwise this request will fail. This allows us to keep all
-	// of the keys off the daemon instances and the panel can handle generating
-	// the credentials for us.
-	PresignedUrl string
 }
 
 var _ BackupInterface = (*S3Backup)(nil)
@@ -43,14 +39,8 @@ func (s *S3Backup) Generate(included *IncludedFiles, prefix string) (*ArchiveDet
 	}
 	defer rc.Close()
 
-	if resp, err := s.generateRemoteRequest(rc); err != nil {
+	if err := s.generateRemoteRequest(rc); err != nil {
 		return nil, errors.WithStack(err)
-	} else {
-		resp.Body.Close()
-
-		if resp.StatusCode != http.StatusOK {
-			return nil, fmt.Errorf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status)
-		}
 	}
 
 	return s.Details(), err
@@ -61,27 +51,123 @@ func (s *S3Backup) Remove() error {
 	return os.Remove(s.Path())
 }
 
-// Generates the remote S3 request and begins the upload.
+// Reader provides a wrapper around an existing io.Reader
-func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) (*http.Response, error) {
+// but implements io.Closer in order to satisfy an io.ReadCloser.
-	r, err := http.NewRequest(http.MethodPut, s.PresignedUrl, nil)
+type Reader struct {
-	if err != nil {
+	io.Reader
-		return nil, err
+}
-	}
+
-
+func (Reader) Close() error {
-	if sz, err := s.Size(); err != nil {
+	return nil
-		return nil, err
+}
-	} else {
+
-		r.ContentLength = sz
+// Generates the remote S3 request and begins the upload.
-		r.Header.Add("Content-Length", strconv.Itoa(int(sz)))
+func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
-		r.Header.Add("Content-Type", "application/x-gzip")
+	defer rc.Close()
-	}
+
-
+	size, err := s.Backup.Size()
-	r.Body = rc
+	if err != nil {
-
+		return err
-	log.WithFields(log.Fields{
+	}
-		"endpoint": s.PresignedUrl,
+
-		"headers":  r.Header,
+	urls, err := api.New().GetBackupRemoteUploadURLs(s.Backup.Uuid, size)
-	}).Debug("uploading backup to remote S3 endpoint")
+	if err != nil {
-
+		return err
-	return http.DefaultClient.Do(r)
+	}
+
+	log.Debug("attempting to upload backup to remote S3 endpoint")
+
+	handlePart := func(part string, size int64) (string, error) {
+		r, err := http.NewRequest(http.MethodPut, part, nil)
+		if err != nil {
+			return "", err
+		}
+
+		r.ContentLength = size
+		r.Header.Add("Content-Length", strconv.Itoa(int(size)))
+		r.Header.Add("Content-Type", "application/x-gzip")
+
+		r.Body = Reader{io.LimitReader(rc, size)}
+
+		res, err := http.DefaultClient.Do(r)
+		if err != nil {
+			return "", err
+		}
+
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return "", fmt.Errorf("failed to put S3 object part, %d:%s", res.StatusCode, res.Status)
+		}
+
+		return res.Header.Get("ETag"), nil
+	}
+
+	// Keep track of errors from individual part uploads.
+	hasError := true
+	defer func() {
+		if !hasError {
+			return
+		}
+
+		r, err := http.NewRequest(http.MethodPost, urls.AbortMultipartUpload, nil)
+		if err != nil {
+			log.WithError(err).Warn("failed to create http request (AbortMultipartUpload)")
+			return
+		}
+
+		res, err := http.DefaultClient.Do(r)
+		if err != nil {
+			log.WithError(err).Warn("failed to make http request (AbortMultipartUpload)")
+			return
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			log.Warnf("failed to abort S3 multipart upload, %d:%s", res.StatusCode, res.Status)
+		}
+	}()
+
+	var completeBody bytes.Buffer
+	completeBody.WriteString("<CompleteMultipartUpload>\n")
+
+	partCount := len(urls.Parts)
+	for i, part := range urls.Parts {
+		var s int64
+		if i+1 < partCount {
+			s = urls.PartSize
+		} else {
+			s = size - (int64(i) * urls.PartSize)
+		}
+
+		etag, err := handlePart(part, s)
+		if err != nil {
+			return err
+		}
+
+		completeBody.WriteString("\t<Part>\n")
+		completeBody.WriteString("\t\t<ETag>\"" + etag + "\"</ETag>\n")
+		completeBody.WriteString("\t\t<PartNumber>" + strconv.Itoa(i+1) + "</PartNumber>\n")
+		completeBody.WriteString("\t</Part>\n")
+	}
+	hasError = false
+
+	completeBody.WriteString("</CompleteMultipartUpload>")
+
+	r, err := http.NewRequest(http.MethodPost, urls.CompleteMultipartUpload, &completeBody)
+	if err != nil {
+		return err
+	}
+
+	res, err := http.DefaultClient.Do(r)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to complete S3 multipart upload, %d:%s", res.StatusCode, res.Status)
+	}
+
+	return nil
 }