Update CHANGELOG.md

Also implements more logic error returns from the Get/Post functions in the client, rather than making the developer call r.Error() on responses.

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v1.4.2
+### Fixed
+* Fixes the `~` character not being properly trimmed from container image names when creating a new server.
+
+### Changed
+* Implemented exponential backoff for S3 uploads when working with backups. This should resolve many issues with external S3 compatiable systems that sometimes return 5xx level errors that should be re-attempted automatically.
+* Implements exponential backoff behavior for all API calls to the Panel that do not immediately return a 401, 403, or 429 error response. This should address fragiligty in some API calls and address random call failures due to connection drops or random DNS resolution errors.
+
+## v1.4.1
+### Fixed
+* Fixes a bug that would cause the file unarchiving process to put all files in the base directory rather than the directory in which the files should be located.
+
 ## v1.4.0
 ### Fixed
 * **[Breaking]** Fixes `/api/servers` and `/api/servers/:server` not properly returning all of the relevant server information and resource usage.

environment/docker/container.go

@@ -178,7 +178,7 @@ func (e *Environment) Create() error {
 		OpenStdin:    true,
 		Tty:          true,
 		ExposedPorts: a.Exposed(),
-		Image:        e.meta.Image,
+		Image:        strings.TrimPrefix(e.meta.Image, "~"),
 		Env:          e.Configuration.EnvironmentVariables(),
 		Labels: map[string]string{
 			"Service":       "Pterodactyl",

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/beevik/etree v1.1.0
 	github.com/buger/jsonparser v1.1.0
+	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/cobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249
 	github.com/containerd/containerd v1.4.3 // indirect
 	github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c // indirect

go.sum

@@ -73,7 +73,10 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/buger/jsonparser v1.1.0 h1:EPAGdKZgZCON4ZcMD+h4l/NN4ndr6ijSpj4INh8PbUY=
 github.com/buger/jsonparser v1.1.0/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
+github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=

remote/errors.go

@@ -3,6 +3,8 @@ package remote
 import (
 	"fmt"
 	"net/http"
+
+	"emperror.dev/errors"
 )
 
 type RequestErrors struct {
@@ -16,13 +18,31 @@ type RequestError struct {
 	Detail   string `json:"detail"`
 }
 
+// IsRequestError checks if the given error is of the RequestError type.
 func IsRequestError(err error) bool {
-	_, ok := err.(*RequestError)
+	var rerr *RequestError
-
+	if err == nil {
-	return ok
+		return false
+	}
+	return errors.As(err, &rerr)
 }
 
-// Returns the error response in a string form that can be more easily consumed.
+// AsRequestError transforms the error into a RequestError if it is currently
+// one, checking the wrap status from the other error handlers. If the error
+// is not a RequestError nil is returned.
+func AsRequestError(err error) *RequestError {
+	if err == nil {
+		return nil
+	}
+	var rerr *RequestError
+	if errors.As(err, &rerr) {
+		return rerr
+	}
+	return nil
+}
+
+// Error returns the error response in a string form that can be more easily
+// consumed.
 func (re *RequestError) Error() string {
 	c := 0
 	if re.response != nil {
@@ -32,6 +52,11 @@ func (re *RequestError) Error() string {
 	return fmt.Sprintf("Error response from Panel: %s: %s (HTTP/%d)", re.Code, re.Detail, c)
 }
 
+// StatusCode returns the status code of the response.
+func (re *RequestError) StatusCode() int {
+	return re.response.StatusCode
+}
+
 type SftpInvalidCredentialsError struct {
 }
 

remote/http.go

@@ -8,11 +8,13 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
 	"emperror.dev/errors"
 	"github.com/apex/log"
+	"github.com/cenkalti/backoff/v4"
 	"github.com/pterodactyl/wings/system"
 )
 
@@ -35,7 +37,7 @@ type client struct {
 	baseUrl     string
 	tokenId     string
 	token       string
-	attempts   int
+	maxAttempts int
 }
 
 // New returns a new HTTP request client that is used for making authenticated
@@ -46,7 +48,7 @@ func New(base string, opts ...ClientOption) Client {
 		httpClient: &http.Client{
 			Timeout: time.Second * 15,
 		},
-		attempts: 1,
+		maxAttempts: 0,
 	}
 	for _, opt := range opts {
 		opt(&c)
@@ -71,11 +73,31 @@ func WithHttpClient(httpClient *http.Client) ClientOption {
 	}
 }
 
+// Get executes a HTTP GET request.
+func (c *client) Get(ctx context.Context, path string, query q) (*Response, error) {
+	return c.request(ctx, http.MethodGet, path, nil, func(r *http.Request) {
+		q := r.URL.Query()
+		for k, v := range query {
+			q.Set(k, v)
+		}
+		r.URL.RawQuery = q.Encode()
+	})
+}
+
+// Post executes a HTTP POST request.
+func (c *client) Post(ctx context.Context, path string, data interface{}) (*Response, error) {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	return c.request(ctx, http.MethodPost, path, bytes.NewBuffer(b))
+}
+
 // requestOnce creates a http request and executes it once. Prefer request()
 // over this method when possible. It appends the path to the endpoint of the
 // client and adds the authentication token to the request.
 func (c *client) requestOnce(ctx context.Context, method, path string, body io.Reader, opts ...func(r *http.Request)) (*Response, error) {
-	req, err := http.NewRequest(method, c.baseUrl+path, body)
+	req, err := http.NewRequestWithContext(ctx, method, c.baseUrl+path, body)
 	if err != nil {
 		return nil, err
 	}
@@ -92,45 +114,86 @@ func (c *client) requestOnce(ctx context.Context, method, path string, body io.R
 
 	debugLogRequest(req)
 
-	res, err := c.httpClient.Do(req.WithContext(ctx))
+	res, err := c.httpClient.Do(req)
 	return &Response{res}, err
 }
 
-// request executes a http request and attempts when errors occur.
+// request executes a HTTP request against the Panel API. If there is an error
-// It appends the path to the endpoint of the client and adds the authentication token to the request.
+// encountered with the request it will be retried using an exponential backoff.
-func (c *client) request(ctx context.Context, method, path string, body io.Reader, opts ...func(r *http.Request)) (res *Response, err error) {
+// If the error returned from the Panel is due to API throttling or because there
-	for i := 0; i < c.attempts; i++ {
+// are invalid authentication credentials provided the request will _not_ be
-		res, err = c.requestOnce(ctx, method, path, body, opts...)
+// retried by the backoff.
-		if err == nil &&
+//
-			res.StatusCode < http.StatusInternalServerError &&
+// This function automatically appends the path to the current client endpoint
-			res.StatusCode != http.StatusTooManyRequests {
+// and adds the required authentication headers to the request that is being
-			break
+// created. Errors returned will be of the RequestError type if there was some
-		}
+// type of response from the API that can be parsed.
-	}
+func (c *client) request(ctx context.Context, method, path string, body io.Reader, opts ...func(r *http.Request)) (*Response, error) {
+	var res *Response
+	err := backoff.Retry(func() error {
+		r, err := c.requestOnce(ctx, method, path, body, opts...)
 		if err != nil {
-		return nil, errors.WithStack(err)
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return backoff.Permanent(err)
 			}
-	return
+			return errors.WrapIf(err, "http: request creation failed")
-}
-
-// get executes a http get request.
-func (c *client) get(ctx context.Context, path string, query q) (*Response, error) {
-	return c.request(ctx, http.MethodGet, path, nil, func(r *http.Request) {
-		q := r.URL.Query()
-		for k, v := range query {
-			q.Set(k, v)
 		}
-		r.URL.RawQuery = q.Encode()
+		res = r
-	})
+		if r.HasError() {
-}
+			// Close the request body after returning the error to free up resources.
-
+			defer r.Body.Close()
-// post executes a http post request.
+			// Don't keep spamming the endpoint if we've already made too many requests or
-func (c *client) post(ctx context.Context, path string, data interface{}) (*Response, error) {
+			// if we're not even authenticated correctly. Retrying generally won't fix either
-	b, err := json.Marshal(data)
+			// of these issues.
+			if r.StatusCode == http.StatusForbidden ||
+				r.StatusCode == http.StatusTooManyRequests ||
+				r.StatusCode == http.StatusUnauthorized {
+				return backoff.Permanent(r.Error())
+			}
+			return r.Error()
+		}
+		return nil
+	}, c.backoff(ctx))
 	if err != nil {
+		if v, ok := err.(*backoff.PermanentError); ok {
+			return nil, v.Unwrap()
+		}
 		return nil, err
 	}
-	return c.request(ctx, http.MethodPost, path, bytes.NewBuffer(b))
+	return res, nil
+}
+
+// backoff returns an exponential backoff function for use with remote API
+// requests. This will allow an API call to be executed approximately 10 times
+// before it is finally reported back as an error.
+//
+// This allows for issues with DNS resolution, or rare race conditions due to
+// slower SQL queries on the Panel to potentially self-resolve without just
+// immediately failing the first request. The example below shows the amount of
+// time that has ellapsed between each call to the handler when an error is
+// returned. You can tweak these values as needed to get the effect you desire.
+//
+// If maxAttempts is a value greater than 0 the backoff will be capped at a total
+// number of executions, or the MaxElapsedTime, whichever comes first.
+//
+// call(): 0s
+// call(): 552.330144ms
+// call(): 1.63271196s
+// call(): 2.94284202s
+// call(): 4.525234711s
+// call(): 6.865723375s
+// call(): 11.37194223s
+// call(): 14.593421816s
+// call(): 20.202045293s
+// call(): 27.36567952s <-- Stops here as MaxElapsedTime is 30 seconds
+func (c *client) backoff(ctx context.Context) backoff.BackOffContext {
+	b := backoff.NewExponentialBackOff()
+	b.MaxInterval = time.Second * 12
+	b.MaxElapsedTime = time.Second * 30
+	if c.maxAttempts > 0 {
+		return backoff.WithContext(backoff.WithMaxRetries(b, uint64(c.maxAttempts)), ctx)
+	}
+	return backoff.WithContext(b, ctx)
 }
 
 // Response is a custom response type that allows for commonly used error
@@ -157,15 +220,12 @@ func (r *Response) HasError() bool {
 func (r *Response) Read() ([]byte, error) {
 	var b []byte
 	if r.Response == nil {
-		return nil, errors.New("http: attempting to read missing response")
+		return nil, errors.New("remote: attempting to read missing response")
 	}
-
 	if r.Response.Body != nil {
 		b, _ = ioutil.ReadAll(r.Response.Body)
 	}
-
 	r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(b))
-
 	return b, nil
 }
 
@@ -177,15 +237,16 @@ func (r *Response) BindJSON(v interface{}) error {
 	if err != nil {
 		return err
 	}
-
 	if err := json.Unmarshal(b, &v); err != nil {
-		return errors.Wrap(err, "http: could not unmarshal response")
+		return errors.Wrap(err, "remote: could not unmarshal response")
 	}
 	return nil
 }
 
 // Returns the first error message from the API call as a string. The error
-// message will be formatted similar to the below example:
+// message will be formatted similar to the below example. If there is no error
+// that can be parsed out of the API you'll still get a RequestError returned
+// but the RequestError.Code will be "_MissingResponseCode".
 //
 // HttpNotFoundException: The requested resource does not exist. (HTTP/404)
 func (r *Response) Error() error {
@@ -196,14 +257,18 @@ func (r *Response) Error() error {
 	var errs RequestErrors
 	_ = r.BindJSON(&errs)
 
-	e := &RequestError{}
+	e := &RequestError{
+		Code:   "_MissingResponseCode",
+		Status: strconv.Itoa(r.StatusCode),
+		Detail: "No error response returned from API endpoint.",
+	}
 	if len(errs.Errors) > 0 {
 		e = &errs.Errors[0]
 	}
 
 	e.response = r.Response
 
-	return e
+	return errors.WithStackDepth(e, 1)
 }
 
 // Logs the request into the debug log with all of the important request bits.

remote/http_test.go

@@ -14,8 +14,7 @@ func createTestClient(h http.HandlerFunc) (*client, *httptest.Server) {
 	c := &client{
 		httpClient: s.Client(),
 		baseUrl:    s.URL,
-
+		maxAttempts: 1,
-		attempts: 1,
 		tokenId:  "testid",
 		token:    "testtoken",
 	}
@@ -47,7 +46,7 @@ func TestRequestRetry(t *testing.T) {
 		}
 		i++
 	})
-	c.attempts = 2
+	c.maxAttempts = 2
 	r, err := c.request(context.Background(), "", "", nil)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
@@ -60,12 +59,15 @@ func TestRequestRetry(t *testing.T) {
 		rw.WriteHeader(http.StatusInternalServerError)
 		i++
 	})
-	c.attempts = 2
+	c.maxAttempts = 2
 	r, err = c.request(context.Background(), "get", "", nil)
-	assert.NoError(t, err)
+	assert.Error(t, err)
-	assert.NotNil(t, r)
+	assert.Nil(t, r)
-	assert.Equal(t, http.StatusInternalServerError, r.StatusCode)
+
-	assert.Equal(t, 2, i)
+	v := AsRequestError(err)
+	assert.NotNil(t, v)
+	assert.Equal(t, http.StatusInternalServerError, v.StatusCode())
+	assert.Equal(t, 3, i)
 }
 
 func TestGet(t *testing.T) {
@@ -74,7 +76,7 @@ func TestGet(t *testing.T) {
 		assert.Len(t, r.URL.Query(), 1)
 		assert.Equal(t, "world", r.URL.Query().Get("hello"))
 	})
-	r, err := c.get(context.Background(), "/test", q{"hello": "world"})
+	r, err := c.Get(context.Background(), "/test", q{"hello": "world"})
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
 }
@@ -87,7 +89,7 @@ func TestPost(t *testing.T) {
 		assert.Equal(t, http.MethodPost, r.Method)
 
 	})
-	r, err := c.post(context.Background(), "/test", test)
+	r, err := c.Post(context.Background(), "/test", test)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
 }

remote/servers.go

@@ -58,62 +58,54 @@ func (c *client) GetServers(ctx context.Context, limit int) ([]RawServerData, er
 // things in a bad state within the Panel. This API call is executed once Wings
 // has fully booted all of the servers.
 func (c *client) ResetServersState(ctx context.Context) error {
-	res, err := c.post(ctx, "/servers/reset", nil)
+	res, err := c.Post(ctx, "/servers/reset", nil)
 	if err != nil {
-		return errors.WrapIf(err, "remote/servers: failed to reset server state on Panel")
+		return errors.WrapIf(err, "remote: failed to reset server state on Panel")
 	}
-	res.Body.Close()
+	_ = res.Body.Close()
 	return nil
 }
 
 func (c *client) GetServerConfiguration(ctx context.Context, uuid string) (ServerConfigurationResponse, error) {
 	var config ServerConfigurationResponse
-	res, err := c.get(ctx, fmt.Sprintf("/servers/%s", uuid), nil)
+	res, err := c.Get(ctx, fmt.Sprintf("/servers/%s", uuid), nil)
 	if err != nil {
 		return config, err
 	}
 	defer res.Body.Close()
 
-	if res.HasError() {
-		return config, res.Error()
-	}
-
 	err = res.BindJSON(&config)
 	return config, err
 }
 
 func (c *client) GetInstallationScript(ctx context.Context, uuid string) (InstallationScript, error) {
-	res, err := c.get(ctx, fmt.Sprintf("/servers/%s/install", uuid), nil)
+	res, err := c.Get(ctx, fmt.Sprintf("/servers/%s/install", uuid), nil)
 	if err != nil {
 		return InstallationScript{}, err
 	}
 	defer res.Body.Close()
 
-	if res.HasError() {
-		return InstallationScript{}, res.Error()
-	}
-
 	var config InstallationScript
 	err = res.BindJSON(&config)
 	return config, err
 }
 
 func (c *client) SetInstallationStatus(ctx context.Context, uuid string, successful bool) error {
-	resp, err := c.post(ctx, fmt.Sprintf("/servers/%s/install", uuid), d{"successful": successful})
+	resp, err := c.Post(ctx, fmt.Sprintf("/servers/%s/install", uuid), d{"successful": successful})
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	_ = resp.Body.Close()
-	return resp.Error()
+	return nil
 }
 
 func (c *client) SetArchiveStatus(ctx context.Context, uuid string, successful bool) error {
-	resp, err := c.post(ctx, fmt.Sprintf("/servers/%s/archive", uuid), d{"successful": successful})
+	resp, err := c.Post(ctx, fmt.Sprintf("/servers/%s/archive", uuid), d{"successful": successful})
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	_ = resp.Body.Close()
-	return resp.Error()
+	return nil
 }
 
 func (c *client) SetTransferStatus(ctx context.Context, uuid string, successful bool) error {
@@ -121,12 +113,12 @@ func (c *client) SetTransferStatus(ctx context.Context, uuid string, successful
 	if successful {
 		state = "success"
 	}
-	resp, err := c.get(ctx, fmt.Sprintf("/servers/%s/transfer/%s", uuid, state), nil)
+	resp, err := c.Get(ctx, fmt.Sprintf("/servers/%s/transfer/%s", uuid, state), nil)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	_ = resp.Body.Close()
-	return resp.Error()
+	return nil
 }
 
 // ValidateSftpCredentials makes a request to determine if the username and
@@ -136,66 +128,54 @@ func (c *client) SetTransferStatus(ctx context.Context, uuid string, successful
 // all of the authorization security logic to the Panel.
 func (c *client) ValidateSftpCredentials(ctx context.Context, request SftpAuthRequest) (SftpAuthResponse, error) {
 	var auth SftpAuthResponse
-	res, err := c.post(ctx, "/sftp/auth", request)
+	res, err := c.Post(ctx, "/sftp/auth", request)
 	if err != nil {
+		if err := AsRequestError(err); err != nil && (err.StatusCode() >= 400 && err.StatusCode() < 500) {
+			log.WithFields(log.Fields{"subsystem": "sftp", "username": request.User, "ip": request.IP}).Warn(err.Error())
+			return auth, &SftpInvalidCredentialsError{}
+		}
 		return auth, err
 	}
 	defer res.Body.Close()
 
-	e := res.Error()
+	if err := res.BindJSON(&auth); err != nil {
-	if e != nil {
-		if res.StatusCode >= 400 && res.StatusCode < 500 {
-			log.WithFields(log.Fields{
-				"subsystem": "sftp",
-				"username":  request.User,
-				"ip":        request.IP,
-			}).Warn(e.Error())
-
-			return auth, &SftpInvalidCredentialsError{}
-		}
-
-		return auth, errors.New(e.Error())
-	}
-
-	err = res.BindJSON(&auth)
 		return auth, err
+	}
+	return auth, nil
 }
 
 func (c *client) GetBackupRemoteUploadURLs(ctx context.Context, backup string, size int64) (BackupRemoteUploadResponse, error) {
 	var data BackupRemoteUploadResponse
-	res, err := c.get(ctx, fmt.Sprintf("/backups/%s", backup), q{"size": strconv.FormatInt(size, 10)})
+	res, err := c.Get(ctx, fmt.Sprintf("/backups/%s", backup), q{"size": strconv.FormatInt(size, 10)})
 	if err != nil {
 		return data, err
 	}
 	defer res.Body.Close()
-
+	if err := res.BindJSON(&data); err != nil {
-	if res.HasError() {
-		return data, res.Error()
-	}
-
-	err = res.BindJSON(&data)
 		return data, err
+	}
+	return data, nil
 }
 
 func (c *client) SetBackupStatus(ctx context.Context, backup string, data BackupRequest) error {
-	resp, err := c.post(ctx, fmt.Sprintf("/backups/%s", backup), data)
+	resp, err := c.Post(ctx, fmt.Sprintf("/backups/%s", backup), data)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	_ = resp.Body.Close()
-	return resp.Error()
+	return nil
 }
 
 // SendRestorationStatus triggers a request to the Panel to notify it that a
 // restoration has been completed and the server should be marked as being
 // activated again.
 func (c *client) SendRestorationStatus(ctx context.Context, backup string, successful bool) error {
-	resp, err := c.post(ctx, fmt.Sprintf("/backups/%s/restore", backup), d{"successful": successful})
+	resp, err := c.Post(ctx, fmt.Sprintf("/backups/%s/restore", backup), d{"successful": successful})
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	_ = resp.Body.Close()
-	return resp.Error()
+	return nil
 }
 
 // getServersPaged returns a subset of servers from the Panel API using the
@@ -206,7 +186,7 @@ func (c *client) getServersPaged(ctx context.Context, page, limit int) ([]RawSer
 		Meta Pagination      `json:"meta"`
 	}
 
-	res, err := c.get(ctx, "/servers", q{
+	res, err := c.Get(ctx, "/servers", q{
 		"page":     strconv.Itoa(page),
 		"per_page": strconv.Itoa(limit),
 	})
@@ -214,10 +194,6 @@ func (c *client) getServersPaged(ctx context.Context, page, limit int) ([]RawSer
 		return nil, r.Meta, err
 	}
 	defer res.Body.Close()
-
-	if res.HasError() {
-		return nil, r.Meta, res.Error()
-	}
 	if err := res.BindJSON(&r); err != nil {
 		return nil, r.Meta, err
 	}

server/backup.go

@@ -66,7 +66,7 @@ func (s *Server) Backup(b backup.BackupInterface) error {
 		}
 	}
 
-	ad, err := b.Generate(s.Filesystem().Path(), ignored)
+	ad, err := b.Generate(s.Context(), s.Filesystem().Path(), ignored)
 	if err != nil {
 		if err := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); err != nil {
 			s.Log().WithFields(log.Fields{
@@ -150,7 +150,7 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) (
 	// Attempt to restore the backup to the server by running through each entry
 	// in the file one at a time and writing them to the disk.
 	s.Log().Debug("starting file writing process for backup restoration")
-	err = b.Restore(reader, func(file string, r io.Reader) error {
+	err = b.Restore(s.Context(), reader, func(file string, r io.Reader) error {
 		s.Events().Publish(DaemonMessageEvent, "(restoring): "+file)
 		return s.Filesystem().Writefile(file, r)
 	})

server/backup/backup.go

@@ -1,16 +1,18 @@
 package backup
 
 import (
+	"context"
 	"crypto/sha1"
 	"encoding/hex"
 	"io"
 	"os"
 	"path"
-	"sync"
 
+	"emperror.dev/errors"
 	"github.com/apex/log"
 	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/remote"
+	"golang.org/x/sync/errgroup"
 )
 
 type AdapterType string
@@ -24,20 +26,37 @@ const (
 // and remote backups allowing the files to be restored.
 type RestoreCallback func(file string, r io.Reader) error
 
-type ArchiveDetails struct {
+// noinspection GoNameStartsWithPackageName
-	Checksum     string `json:"checksum"`
+type BackupInterface interface {
-	ChecksumType string `json:"checksum_type"`
+	// SetClient sets the API request client on the backup interface.
-	Size         int64  `json:"size"`
+	SetClient(c remote.Client)
-}
+	// Identifier returns the UUID of this backup as tracked by the panel
-
+	// instance.
-// ToRequest returns a request object.
+	Identifier() string
-func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
+	// WithLogContext attaches additional context to the log output for this
-	return remote.BackupRequest{
+	// backup.
-		Checksum:     ad.Checksum,
+	WithLogContext(map[string]interface{})
-		ChecksumType: ad.ChecksumType,
+	// Generate creates a backup in whatever the configured source for the
-		Size:         ad.Size,
+	// specific implementation is.
-		Successful:   successful,
+	Generate(ctx context.Context, basePath string, ignore string) (*ArchiveDetails, error)
-	}
+	// Ignored returns the ignored files for this backup instance.
+	Ignored() string
+	// Checksum returns a SHA1 checksum for the generated backup.
+	Checksum() ([]byte, error)
+	// Size returns the size of the generated backup.
+	Size() (int64, error)
+	// Path returns the path to the backup on the machine. This is not always
+	// the final storage location of the backup, simply the location we're using
+	// to store it until it is moved to the final spot.
+	Path() string
+	// Details returns details about the archive.
+	Details(ctx context.Context) (*ArchiveDetails, error)
+	// Remove removes a backup file.
+	Remove() error
+	// Restore is called when a backup is ready to be restored to the disk from
+	// the given source. Not every backup implementation will support this nor
+	// will every implementation require a reader be provided.
+	Restore(ctx context.Context, reader io.Reader, callback RestoreCallback) error
 }
 
 type Backup struct {
@@ -54,39 +73,6 @@ type Backup struct {
 	logContext map[string]interface{}
 }
 
-// noinspection GoNameStartsWithPackageName
-type BackupInterface interface {
-	// SetClient sets the API request client on the backup interface.
-	SetClient(c remote.Client)
-	// Identifier returns the UUID of this backup as tracked by the panel
-	// instance.
-	Identifier() string
-	// WithLogContext attaches additional context to the log output for this
-	// backup.
-	WithLogContext(map[string]interface{})
-	// Generate creates a backup in whatever the configured source for the
-	// specific implementation is.
-	Generate(string, string) (*ArchiveDetails, error)
-	// Ignored returns the ignored files for this backup instance.
-	Ignored() string
-	// Checksum returns a SHA1 checksum for the generated backup.
-	Checksum() ([]byte, error)
-	// Size returns the size of the generated backup.
-	Size() (int64, error)
-	// Path returns the path to the backup on the machine. This is not always
-	// the final storage location of the backup, simply the location we're using
-	// to store it until it is moved to the final spot.
-	Path() string
-	// Details returns details about the archive.
-	Details() *ArchiveDetails
-	// Remove removes a backup file.
-	Remove() error
-	// Restore is called when a backup is ready to be restored to the disk from
-	// the given source. Not every backup implementation will support this nor
-	// will every implementation require a reader be provided.
-	Restore(reader io.Reader, callback RestoreCallback) error
-}
-
 func (b *Backup) SetClient(c remote.Client) {
 	b.client = c
 }
@@ -95,12 +81,12 @@ func (b *Backup) Identifier() string {
 	return b.Uuid
 }
 
-// Returns the path for this specific backup.
+// Path returns the path for this specific backup.
 func (b *Backup) Path() string {
 	return path.Join(config.Get().System.BackupDirectory, b.Identifier()+".tar.gz")
 }
 
-// Return the size of the generated backup.
+// Size returns the size of the generated backup.
 func (b *Backup) Size() (int64, error) {
 	st, err := os.Stat(b.Path())
 	if err != nil {
@@ -110,7 +96,7 @@ func (b *Backup) Size() (int64, error) {
 	return st.Size(), nil
 }
 
-// Returns the SHA256 checksum of a backup.
+// Checksum returns the SHA256 checksum of a backup.
 func (b *Backup) Checksum() ([]byte, error) {
 	h := sha1.New()
 
@@ -128,51 +114,34 @@ func (b *Backup) Checksum() ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
-// Returns details of the archive by utilizing two go-routines to get the checksum and
+// Details returns both the checksum and size of the archive currently stored on
-// the size of the archive.
+// the disk to the caller.
-func (b *Backup) Details() *ArchiveDetails {
+func (b *Backup) Details(ctx context.Context) (*ArchiveDetails, error) {
-	wg := sync.WaitGroup{}
+	ad := ArchiveDetails{ChecksumType: "sha1"}
-	wg.Add(2)
+	g, ctx := errgroup.WithContext(ctx)
 
-	l := log.WithField("backup_id", b.Uuid)
+	g.Go(func() error {
-
-	var checksum string
-	// Calculate the checksum for the file.
-	go func() {
-		defer wg.Done()
-
-		l.Info("computing checksum for backup...")
 		resp, err := b.Checksum()
 		if err != nil {
-			log.WithFields(log.Fields{
+			return err
-				"backup": b.Identifier(),
-				"error":  err,
-			}).Error("failed to calculate checksum for backup")
-			return
 		}
+		ad.Checksum = hex.EncodeToString(resp)
+		return nil
+	})
 
-		checksum = hex.EncodeToString(resp)
+	g.Go(func() error {
-		l.WithField("checksum", checksum).Info("computed checksum for backup")
+		s, err := b.Size()
-	}()
+		if err != nil {
-
+			return err
-	var sz int64
-	go func() {
-		defer wg.Done()
-
-		if s, err := b.Size(); err != nil {
-			return
-		} else {
-			sz = s
 		}
-	}()
+		ad.Size = s
+		return nil
+	})
 
-	wg.Wait()
+	if err := g.Wait(); err != nil {
-
+		return nil, errors.WithStackDepth(err, 1)
-	return &ArchiveDetails{
-		Checksum:     checksum,
-		ChecksumType: "sha1",
-		Size:         sz,
 	}
+	return &ad, nil
 }
 
 func (b *Backup) Ignored() string {
@@ -188,3 +157,19 @@ func (b *Backup) log() *log.Entry {
 	}
 	return l
 }
+
+type ArchiveDetails struct {
+	Checksum     string `json:"checksum"`
+	ChecksumType string `json:"checksum_type"`
+	Size         int64  `json:"size"`
+}
+
+// ToRequest returns a request object.
+func (ad *ArchiveDetails) ToRequest(successful bool) remote.BackupRequest {
+	return remote.BackupRequest{
+		Checksum:     ad.Checksum,
+		ChecksumType: ad.ChecksumType,
+		Size:         ad.Size,
+		Successful:   successful,
+	}
+}

server/backup/backup_local.go

@@ -1,10 +1,11 @@
 package backup
 
 import (
-	"errors"
+	"context"
 	"io"
 	"os"
 
+	"emperror.dev/errors"
 	"github.com/pterodactyl/wings/server/filesystem"
 
 	"github.com/mholt/archiver/v3"
@@ -56,28 +57,40 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) {
 
 // Generate generates a backup of the selected files and pushes it to the
 // defined location for this instance.
-func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
+func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) {
 	a := &filesystem.Archive{
 		BasePath: basePath,
 		Ignore:   ignore,
 	}
 
-	b.log().Info("creating backup for server...")
+	b.log().WithField("path", b.Path()).Info("creating backup for server")
 	if err := a.Create(b.Path()); err != nil {
 		return nil, err
 	}
 	b.log().Info("created backup successfully")
 
-	return b.Details(), nil
+	ad, err := b.Details(ctx)
+	if err != nil {
+		return nil, errors.WrapIf(err, "backup: failed to get archive details for local backup")
+	}
+	return ad, nil
 }
 
 // Restore will walk over the archive and call the callback function for each
 // file encountered.
-func (b *LocalBackup) Restore(_ io.Reader, callback RestoreCallback) error {
+func (b *LocalBackup) Restore(ctx context.Context, _ io.Reader, callback RestoreCallback) error {
 	return archiver.Walk(b.Path(), func(f archiver.File) error {
+		select {
+		case <-ctx.Done():
+			// Stop walking if the context is canceled.
+			return archiver.ErrStopWalk
+		default:
+			{
 				if f.IsDir() {
 					return nil
 				}
-		return callback(f.Name(), f)
+				return callback(filesystem.ExtractNameFromArchive(f), f)
+			}
+		}
 	})
 }

server/backup/backup_s3.go

@@ -5,11 +5,15 @@ import (
 	"compress/gzip"
 	"context"
 	"fmt"
-	"github.com/pterodactyl/wings/server/filesystem"
 	"io"
 	"net/http"
 	"os"
 	"strconv"
+	"time"
+
+	"emperror.dev/errors"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/pterodactyl/wings/server/filesystem"
 
 	"github.com/juju/ratelimit"
 	"github.com/pterodactyl/wings/config"
@@ -45,7 +49,7 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) {
 
 // Generate creates a new backup on the disk, moves it into the S3 bucket via
 // the provided presigned URL, and then deletes the backup from the disk.
-func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
+func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) {
 	defer s.Remove()
 
 	a := &filesystem.Archive{
@@ -53,7 +57,7 @@ func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
 		Ignore:   ignore,
 	}
 
-	s.log().Info("creating backup for server...")
+	s.log().WithField("path", s.Path()).Info("creating backup for server")
 	if err := a.Create(s.Path()); err != nil {
 		return nil, err
 	}
@@ -61,29 +65,65 @@ func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) {
 
 	rc, err := os.Open(s.Path())
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "backup: could not read archive from disk")
 	}
 	defer rc.Close()
 
-	if err := s.generateRemoteRequest(rc); err != nil {
+	if err := s.generateRemoteRequest(ctx, rc); err != nil {
 		return nil, err
 	}
-
+	ad, err := s.Details(ctx)
-	return s.Details(), nil
+	if err != nil {
+		return nil, errors.WrapIf(err, "backup: failed to get archive details after upload")
+	}
+	return ad, nil
 }
 
-// Reader provides a wrapper around an existing io.Reader
+// Restore will read from the provided reader assuming that it is a gzipped
-// but implements io.Closer in order to satisfy an io.ReadCloser.
+// tar reader. When a file is encountered in the archive the callback function
-type Reader struct {
+// will be triggered. If the callback returns an error the entire process is
-	io.Reader
+// stopped, otherwise this function will run until all files have been written.
-}
+//
-
+// This restoration uses a workerpool to use up to the number of CPUs available
-func (Reader) Close() error {
+// on the machine when writing files to the disk.
+func (s *S3Backup) Restore(ctx context.Context, r io.Reader, callback RestoreCallback) error {
+	reader := r
+	// Steal the logic we use for making backups which will be applied when restoring
+	// this specific backup. This allows us to prevent overloading the disk unintentionally.
+	if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 {
+		reader = ratelimit.Reader(r, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit))
+	}
+	gr, err := gzip.NewReader(reader)
+	if err != nil {
+		return err
+	}
+	defer gr.Close()
+	tr := tar.NewReader(gr)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil
+		default:
+			// Do nothing, fall through to the next block of code in this loop.
+		}
+		header, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		if header.Typeflag == tar.TypeReg {
+			if err := callback(header.Name, tr); err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }
 
 // Generates the remote S3 request and begins the upload.
-func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
+func (s *S3Backup) generateRemoteRequest(ctx context.Context, rc io.ReadCloser) error {
 	defer rc.Close()
 
 	s.log().Debug("attempting to get size of backup...")
@@ -101,37 +141,7 @@ func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
 	s.log().Debug("got S3 upload urls from the Panel")
 	s.log().WithField("parts", len(urls.Parts)).Info("attempting to upload backup to s3 endpoint...")
 
-	handlePart := func(part string, size int64) (string, error) {
+	uploader := newS3FileUploader(rc)
-		r, err := http.NewRequest(http.MethodPut, part, nil)
-		if err != nil {
-			return "", err
-		}
-
-		r.ContentLength = size
-		r.Header.Add("Content-Length", strconv.Itoa(int(size)))
-		r.Header.Add("Content-Type", "application/x-gzip")
-
-		// Limit the reader to the size of the part.
-		r.Body = Reader{Reader: io.LimitReader(rc, size)}
-
-		// This http request can block forever due to it not having a timeout,
-		// but we are uploading up to 5GB of data, so there is not really
-		// a good way to handle a timeout on this.
-		res, err := http.DefaultClient.Do(r)
-		if err != nil {
-			return "", err
-		}
-		defer res.Body.Close()
-
-		// Handle non-200 status codes.
-		if res.StatusCode != http.StatusOK {
-			return "", fmt.Errorf("failed to put S3 object part, %d:%s", res.StatusCode, res.Status)
-		}
-
-		// Get the ETag from the uploaded part, this should be sent with the CompleteMultipartUpload request.
-		return res.Header.Get("ETag"), nil
-	}
-
 	for i, part := range urls.Parts {
 		// Get the size for the current part.
 		var partSize int64
@@ -144,7 +154,7 @@ func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
 		}
 
 		// Attempt to upload the part.
-		if _, err := handlePart(part, partSize); err != nil {
+		if _, err := uploader.uploadPart(ctx, part, partSize); err != nil {
 			s.log().WithField("part_id", i+1).WithError(err).Warn("failed to upload part")
 			return err
 		}
@@ -157,39 +167,97 @@ func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) error {
 	return nil
 }
 
-// Restore will read from the provided reader assuming that it is a gzipped
+type s3FileUploader struct {
-// tar reader. When a file is encountered in the archive the callback function
+	io.ReadCloser
-// will be triggered. If the callback returns an error the entire process is
+	client *http.Client
-// stopped, otherwise this function will run until all files have been written.
+}
+
+// newS3FileUploader returns a new file uploader instance.
+func newS3FileUploader(file io.ReadCloser) *s3FileUploader {
+	return &s3FileUploader{
+		ReadCloser: file,
+		// We purposefully use a super high timeout on this request since we need to upload
+		// a 5GB file. This assumes at worst a 10Mbps connection for uploading. While technically
+		// you could go slower we're targeting mostly hosted servers that should have 100Mbps
+		// connections anyways.
+		client: &http.Client{Timeout: time.Hour * 2},
+	}
+}
+
+// backoff returns a new expoential backoff implementation using a context that
+// will also stop the backoff if it is canceled.
+func (fu *s3FileUploader) backoff(ctx context.Context) backoff.BackOffContext {
+	b := backoff.NewExponentialBackOff()
+	b.Multiplier = 2
+	b.MaxElapsedTime = time.Minute
+
+	return backoff.WithContext(b, ctx)
+}
+
+// uploadPart attempts to upload a given S3 file part to the S3 system. If a
+// 5xx error is returned from the endpoint this will continue with an exponential
+// backoff to try and successfully upload the part.
 //
-// This restoration uses a workerpool to use up to the number of CPUs available
+// Once uploaded the ETag is returned to the caller.
-// on the machine when writing files to the disk.
+func (fu *s3FileUploader) uploadPart(ctx context.Context, part string, size int64) (string, error) {
-func (s *S3Backup) Restore(r io.Reader, callback RestoreCallback) error {
+	r, err := http.NewRequestWithContext(ctx, http.MethodPut, part, nil)
-	reader := r
-	// Steal the logic we use for making backups which will be applied when restoring
-	// this specific backup. This allows us to prevent overloading the disk unintentionally.
-	if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 {
-		reader = ratelimit.Reader(r, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit))
-	}
-	gr, err := gzip.NewReader(reader)
 	if err != nil {
-		return err
+		return "", errors.Wrap(err, "backup: could not create request for S3")
 	}
-	defer gr.Close()
+
-	tr := tar.NewReader(gr)
+	r.ContentLength = size
-	for {
+	r.Header.Add("Content-Length", strconv.Itoa(int(size)))
-		header, err := tr.Next()
+	r.Header.Add("Content-Type", "application/x-gzip")
+
+	// Limit the reader to the size of the part.
+	r.Body = Reader{Reader: io.LimitReader(fu.ReadCloser, size)}
+
+	var etag string
+	err = backoff.Retry(func() error {
+		res, err := fu.client.Do(r)
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
-				break
+				return backoff.Permanent(err)
 			}
+			// Don't use a permanent error here, if there is a temporary resolution error with
+			// the URL due to DNS issues we want to keep re-trying.
+			return errors.Wrap(err, "backup: S3 HTTP request failed")
+		}
+		_ = res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			err := errors.New(fmt.Sprintf("backup: failed to put S3 object: [HTTP/%d] %s", res.StatusCode, res.Status))
+			// Only attempt a backoff retry if this error is because of a 5xx error from
+			// the S3 endpoint. Any 4xx error should be treated as an error that a retry
+			// would not fix.
+			if res.StatusCode >= http.StatusInternalServerError {
 				return err
 			}
-		if header.Typeflag == tar.TypeReg {
+			return backoff.Permanent(err)
-			if err := callback(header.Name, tr); err != nil {
-				return err
 		}
+
+		// Get the ETag from the uploaded part, this should be sent with the
+		// CompleteMultipartUpload request.
+		etag = res.Header.Get("ETag")
+
+		return nil
+	}, fu.backoff(ctx))
+
+	if err != nil {
+		if v, ok := err.(*backoff.PermanentError); ok {
+			return "", v.Unwrap()
 		}
+		return "", err
 	}
+	return etag, nil
+}
+
+// Reader provides a wrapper around an existing io.Reader
+// but implements io.Closer in order to satisfy an io.ReadCloser.
+type Reader struct {
+	io.Reader
+}
+
+func (Reader) Close() error {
 	return nil
 }

server/filesystem/compress.go

@@ -1,6 +1,9 @@
 package filesystem
 
 import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
 	"fmt"
 	"os"
 	"path"
@@ -121,7 +124,7 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error {
 		if f.IsDir() {
 			return nil
 		}
-		p := filepath.Join(dir, f.Name())
+		p := filepath.Join(dir, ExtractNameFromArchive(f))
 		// If it is ignored, just don't do anything with the file and skip over it.
 		if err := fs.IsIgnored(p); err != nil {
 			return nil
@@ -139,3 +142,35 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error {
 	}
 	return nil
 }
+
+// ExtractNameFromArchive looks at an archive file to try and determine the name
+// for a given element in an archive. Because of... who knows why, each file type
+// uses different methods to determine the file name.
+//
+// If there is a archiver.File#Sys() value present we will try to use the name
+// present in there, otherwise falling back to archiver.File#Name() if all else
+// fails. Without this logic present, some archive types such as zip/tars/etc.
+// will write all of the files to the base directory, rather than the nested
+// directory that is expected.
+//
+// For files like ".rar" types, there is no f.Sys() value present, and the value
+// of archiver.File#Name() will be what you need.
+func ExtractNameFromArchive(f archiver.File) string {
+	sys := f.Sys()
+	// Some archive types won't have a value returned when you call f.Sys() on them,
+	// such as ".rar" archives for example. In those cases the only thing you can do
+	// is hope that "f.Name()" is actually correct for them.
+	if sys == nil {
+		return f.Name()
+	}
+	switch s := sys.(type) {
+	case *tar.Header:
+		return s.Name
+	case *gzip.Header:
+		return s.Name
+	case *zip.FileHeader:
+		return s.Name
+	default:
+		return f.Name()
+	}
+}

server/install.go

@@ -90,13 +90,8 @@ func (s *Server) Reinstall() error {
 func (s *Server) internalInstall() error {
 	script, err := s.client.GetInstallationScript(s.Context(), s.Id())
 	if err != nil {
-		if !remote.IsRequestError(err) {
 		return err
 	}
-
-		return errors.New(err.Error())
-	}
-
 	p, err := NewInstallationProcess(s, &script)
 	if err != nil {
 		return err
@@ -535,19 +530,10 @@ func (ip *InstallationProcess) StreamOutput(ctx context.Context, id string) erro
 	return nil
 }
 
-// Makes a HTTP request to the Panel instance notifying it that the server has
+// SyncInstallState makes a HTTP request to the Panel instance notifying it that
-// completed the installation process, and what the state of the server is. A boolean
+// the server has completed the installation process, and what the state of the
-// value of "true" means everything was successful, "false" means something went
+// server is. A boolean value of "true" means everything was successful, "false"
-// wrong and the server must be deleted and re-created.
+// means something went wrong and the server must be deleted and re-created.
 func (s *Server) SyncInstallState(successful bool) error {
-	err := s.client.SetInstallationStatus(s.Context(), s.Id(), successful)
+	return s.client.SetInstallationStatus(s.Context(), s.Id(), successful)
-	if err != nil {
-		if !remote.IsRequestError(err) {
-			return err
-		}
-
-		return errors.New(err.Error())
-	}
-
-	return nil
 }

server/server.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 	"sync"
@@ -152,17 +153,11 @@ func (s *Server) Log() *log.Entry {
 func (s *Server) Sync() error {
 	cfg, err := s.client.GetServerConfiguration(s.Context(), s.Id())
 	if err != nil {
-		if !remote.IsRequestError(err) {
+		if err := remote.AsRequestError(err); err != nil && err.StatusCode() == http.StatusNotFound {
-			return err
-		}
-
-		if err.(*remote.RequestError).Status == "404" {
 			return &serverDoesNotExist{}
 		}
-
+		return errors.WithStackIf(err)
-		return errors.New(err.Error())
 	}
-
 	return s.SyncWithConfiguration(cfg)
 }