Merge branch 'develop' into dane/backup-restore

api/server_endpoints.go

@@ -55,7 +55,7 @@ type RawServerData struct {
 // be loaded. If so, those requests are spun-up in additional routines and the final resulting
 // slice of all servers will be returned.
 func (r *Request) GetServers() ([]RawServerData, error) {
-	resp, err := r.Get("/servers", Q{"per_page": strconv.Itoa(int(config.Get().RemoteQuery.BootServersPerPage))})
+	resp, err := r.Get("/servers", Q{"per_page": strconv.Itoa(config.Get().RemoteQuery.BootServersPerPage)})
 	if err != nil {
 		return nil, err
 	}

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
 	log2 "log"
 	"net/http"
@@ -10,8 +11,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
-	"emperror.dev/errors"
 	"github.com/NYTimes/logrotate"
 	"github.com/apex/log"
 	"github.com/apex/log/handlers/multi"
@@ -22,6 +23,7 @@ import (
 	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/environment"
 	"github.com/pterodactyl/wings/loggers/cli"
+	"github.com/pterodactyl/wings/remote"
 	"github.com/pterodactyl/wings/router"
 	"github.com/pterodactyl/wings/server"
 	"github.com/pterodactyl/wings/sftp"
@@ -135,7 +137,15 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 		"gid":      config.Get().System.User.Gid,
 	}).Info("configured system user successfully")
 
-	if err := server.LoadDirectory(); err != nil {
+	pclient := remote.CreateClient(
+		config.Get().PanelLocation,
+		config.Get().AuthenticationTokenId,
+		config.Get().AuthenticationToken,
+		remote.WithTimeout(time.Second*time.Duration(config.Get().RemoteQuery.Timeout)),
+	)
+
+	manager, err := server.NewManager(cmd.Context(), pclient)
+	if err != nil {
 		log.WithField("error", err).Fatal("failed to load server configurations")
 	}
 
@@ -148,20 +158,38 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 	}
 
 	// Just for some nice log output.
-	for _, s := range server.GetServers().All() {
+	for _, s := range manager.All() {
-		log.WithField("server", s.Id()).Info("loaded configuration for server")
+		log.WithField("server", s.Id()).Info("finished loading configuration for server")
 	}
 
-	states, err := server.CachedServerStates()
+	states, err := manager.ReadStates()
 	if err != nil {
 		log.WithField("error", err).Error("failed to retrieve locally cached server states from disk, assuming all servers in offline state")
 	}
 
+	ticker := time.NewTicker(time.Minute)
+	// Every minute, write the current server states to the disk to allow for a more
+	// seamless hard-reboot process in which wings will re-sync server states based
+	// on it's last tracked state.
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				if err := manager.PersistStates(); err != nil {
+					log.WithField("error", err).Warn("failed to persist server states to disk")
+				}
+			case <-cmd.Context().Done():
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+
 	// Create a new workerpool that limits us to 4 servers being bootstrapped at a time
 	// on Wings. This allows us to ensure the environment exists, write configurations,
 	// and reboot processes without causing a slow-down due to sequential booting.
 	pool := workerpool.New(4)
-	for _, serv := range server.GetServers().All() {
+	for _, serv := range manager.All() {
 		s := serv
 
 		// For each server we encounter make sure the root data directory exists.
@@ -172,7 +200,6 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 
 		pool.Submit(func() {
 			s.Log().Info("configuring server environment and restoring to previous state")
-
 			var st string
 			if state, exists := states[s.Id()]; exists {
 				st = state
@@ -224,14 +251,14 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 	defer func() {
 		// Cancel the context on all of the running servers at this point, even though the
 		// program is just shutting down.
-		for _, s := range server.GetServers().All() {
+		for _, s := range manager.All() {
 			s.CtxCancel()
 		}
 	}()
 
 	go func() {
 		// Run the SFTP server.
-		if err := sftp.New().Run(); err != nil {
+		if err := sftp.New(manager).Run(); err != nil {
 			log.WithError(err).Fatal("failed to initialize the sftp server")
 			return
 		}
@@ -266,7 +293,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 	// and external clients.
 	s := &http.Server{
 		Addr:      api.Host + ":" + strconv.Itoa(api.Port),
-		Handler:   router.Configure(),
+		Handler:   router.Configure(manager),
 		TLSConfig: config.DefaultTLSConfig,
 	}
 

config/config.go

@@ -99,7 +99,7 @@ type RemoteQueryConfiguration struct {
 	// are taking longer than 30 seconds to complete it is likely a performance issue that
 	// should be resolved on the Panel, and not something that should be resolved by upping this
 	// number.
-	Timeout uint `default:"30" yaml:"timeout"`
+	Timeout int `default:"30" yaml:"timeout"`
 
 	// The number of servers to load in a single request to the Panel API when booting the
 	// Wings instance. A single request is initially made to the Panel to get this number
@@ -110,7 +110,7 @@ type RemoteQueryConfiguration struct {
 	// memory limits on your Panel instance. In the grand scheme of things 4 requests for
 	// 50 servers is likely just as quick as two for 100 or one for 400, and will certainly
 	// be less likely to cause performance issues on the Panel.
-	BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"`
+	BootServersPerPage int `default:"50" yaml:"boot_servers_per_page"`
 }
 
 // SystemConfiguration defines basic system configuration settings.

go.mod

@@ -63,6 +63,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f
 	github.com/sirupsen/logrus v1.7.0 // indirect
 	github.com/spf13/cobra v1.1.1
+	github.com/stretchr/testify v1.6.1
 	github.com/ugorji/go v1.2.2 // indirect
 	github.com/ulikunitz/xz v0.5.9 // indirect
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad

remote/backup.go

@@ -0,0 +1,34 @@
+package remote
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/pterodactyl/wings/api"
+)
+
+func (c *client) GetBackupRemoteUploadURLs(ctx context.Context, backup string, size int64) (api.BackupRemoteUploadResponse, error) {
+	res, err := c.get(ctx, fmt.Sprintf("/backups/%s", backup), q{"size": strconv.FormatInt(size, 10)})
+	if err != nil {
+		return api.BackupRemoteUploadResponse{}, err
+	}
+	defer res.Body.Close()
+
+	if res.HasError() {
+		return api.BackupRemoteUploadResponse{}, res.Error()
+	}
+
+	r := api.BackupRemoteUploadResponse{}
+	err = res.BindJSON(&r)
+	return r, err
+}
+
+func (c *client) SetBackupStatus(ctx context.Context, backup string, data api.BackupRequest) error {
+	resp, err := c.post(ctx, fmt.Sprintf("/backups/%s", backup), data)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return resp.Error()
+}

remote/client.go

@@ -0,0 +1,56 @@
+package remote
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/pterodactyl/wings/api"
+)
+
+type Client interface {
+	GetBackupRemoteUploadURLs(ctx context.Context, backup string, size int64) (api.BackupRemoteUploadResponse, error)
+	GetInstallationScript(ctx context.Context, uuid string) (api.InstallationScript, error)
+	GetServerConfiguration(ctx context.Context, uuid string) (api.ServerConfigurationResponse, error)
+	GetServers(context context.Context, perPage int) ([]api.RawServerData, error)
+	SetArchiveStatus(ctx context.Context, uuid string, successful bool) error
+	SetBackupStatus(ctx context.Context, backup string, data api.BackupRequest) error
+	SetInstallationStatus(ctx context.Context, uuid string, successful bool) error
+	SetTransferStatus(ctx context.Context, uuid string, successful bool) error
+	ValidateSftpCredentials(ctx context.Context, request api.SftpAuthRequest) (api.SftpAuthResponse, error)
+}
+
+type client struct {
+	httpClient *http.Client
+	baseUrl    string
+	tokenId    string
+	token      string
+	retries    int
+}
+
+type ClientOption func(c *client)
+
+func CreateClient(base, tokenId, token string, opts ...ClientOption) Client {
+	httpClient := &http.Client{
+		Timeout: time.Second * 15,
+	}
+	base = strings.TrimSuffix(base, "/")
+	c := &client{
+		baseUrl:    base + "/api/remote",
+		tokenId:    tokenId,
+		token:      token,
+		httpClient: httpClient,
+		retries:    3,
+	}
+	for _, o := range opts {
+		o(c)
+	}
+	return c
+}
+
+func WithTimeout(timeout time.Duration) ClientOption {
+	return func(c *client) {
+		c.httpClient.Timeout = timeout
+	}
+}

remote/client_test.go

@@ -0,0 +1,19 @@
+package remote
+
+import (
+	"net/http"
+	"net/http/httptest"
+)
+
+func createTestClient(h http.HandlerFunc) (*client, *httptest.Server) {
+	s := httptest.NewServer(h)
+	c := &client{
+		httpClient: s.Client(),
+		baseUrl:    s.URL,
+
+		retries: 1,
+		tokenId: "testid",
+		token:   "testtoken",
+	}
+	return c, s
+}

remote/errors.go

@@ -0,0 +1,46 @@
+package remote
+
+import (
+	"fmt"
+	"net/http"
+)
+
+type RequestErrors struct {
+	Errors []RequestError `json:"errors"`
+}
+
+type RequestError struct {
+	response *http.Response
+	Code     string `json:"code"`
+	Status   string `json:"status"`
+	Detail   string `json:"detail"`
+}
+
+func IsRequestError(err error) bool {
+	_, ok := err.(*RequestError)
+
+	return ok
+}
+
+// Returns the error response in a string form that can be more easily consumed.
+func (re *RequestError) Error() string {
+	c := 0
+	if re.response != nil {
+		c = re.response.StatusCode
+	}
+
+	return fmt.Sprintf("Error response from Panel: %s: %s (HTTP/%d)", re.Code, re.Detail, c)
+}
+
+type sftpInvalidCredentialsError struct {
+}
+
+func (ice sftpInvalidCredentialsError) Error() string {
+	return "the credentials provided were invalid"
+}
+
+func IsInvalidCredentialsError(err error) bool {
+	_, ok := err.(*sftpInvalidCredentialsError)
+
+	return ok
+}

remote/http.go

@@ -0,0 +1,157 @@
+package remote
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/pterodactyl/wings/system"
+)
+
+// A custom response type that allows for commonly used error handling and response
+// parsing from the Panel API. This just embeds the normal HTTP response from Go and
+// we attach a few helper functions to it.
+type Response struct {
+	*http.Response
+}
+
+// A generic type allowing for easy binding use when making requests to API endpoints
+// that only expect a singular argument or something that would not benefit from being
+// a typed struct.
+//
+// Inspired by gin.H, same concept.
+type d map[string]interface{}
+
+// Same concept as d, but a map of strings, used for querying GET requests.
+type q map[string]string
+
+// requestOnce creates a http request and executes it once.
+// Prefer request() over this method when possible.
+// It appends the path to the endpoint of the client and adds the authentication token to the request.
+func (c *client) requestOnce(ctx context.Context, method, path string, body io.Reader, opts ...func(r *http.Request)) (*Response, error) {
+	req, err := http.NewRequest(method, c.baseUrl+path, body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("User-Agent", fmt.Sprintf("Pterodactyl Wings/v%s (id:%s)", system.Version, c.tokenId))
+	req.Header.Set("Accept", "application/vnd.pterodactyl.v1+json")
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s.%s", c.tokenId, c.token))
+
+	// Call all opts functions to allow modifying the request
+	for _, o := range opts {
+		o(req)
+	}
+
+	debugLogRequest(req)
+
+	res, err := c.httpClient.Do(req.WithContext(ctx))
+	return &Response{res}, err
+}
+
+// request executes a http request and retries when errors occur.
+// It appends the path to the endpoint of the client and adds the authentication token to the request.
+func (c *client) request(ctx context.Context, method, path string, body io.Reader, opts ...func(r *http.Request)) (*Response, error) {
+	var doErr error
+	var res *Response
+
+	for i := 0; i < c.retries; i++ {
+		res, doErr = c.requestOnce(ctx, method, path, body, opts...)
+
+		if doErr == nil &&
+			res.StatusCode < http.StatusInternalServerError &&
+			res.StatusCode != http.StatusTooManyRequests {
+			break
+		}
+	}
+
+	return res, doErr
+}
+
+// get executes a http get request.
+func (c *client) get(ctx context.Context, path string, query q) (*Response, error) {
+	return c.request(ctx, http.MethodGet, path, nil, func(r *http.Request) {
+		q := r.URL.Query()
+		for k, v := range query {
+			q.Set(k, v)
+		}
+		r.URL.RawQuery = q.Encode()
+	})
+}
+
+// post executes a http post request.
+func (c *client) post(ctx context.Context, path string, data interface{}) (*Response, error) {
+	b, err := json.Marshal(data)
+	if err != nil {
+		return nil, err
+	}
+	return c.request(ctx, http.MethodPost, path, bytes.NewBuffer(b))
+}
+
+// Determines if the API call encountered an error. If no request has been made
+// the response will be false. This function will evaluate to true if the response
+// code is anything 300 or higher.
+func (r *Response) HasError() bool {
+	if r.Response == nil {
+		return false
+	}
+
+	return r.StatusCode >= 300 || r.StatusCode < 200
+}
+
+// Reads the body from the response and returns it, then replaces it on the response
+// so that it can be read again later. This does not close the response body, so any
+// functions calling this should be sure to manually defer a Body.Close() call.
+func (r *Response) Read() ([]byte, error) {
+	var b []byte
+	if r.Response == nil {
+		return nil, errors.New("no response exists on interface")
+	}
+
+	if r.Response.Body != nil {
+		b, _ = ioutil.ReadAll(r.Response.Body)
+	}
+
+	r.Response.Body = ioutil.NopCloser(bytes.NewBuffer(b))
+
+	return b, nil
+}
+
+// Binds a given interface with the data returned in the response. This is a shortcut
+// for calling Read and then manually calling json.Unmarshal on the raw bytes.
+func (r *Response) BindJSON(v interface{}) error {
+	b, err := r.Read()
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(b, &v)
+}
+
+// Returns the first error message from the API call as a string.
+// The error message will be formatted similar to the below example:
+//
+// HttpNotFoundException: The requested resource does not exist. (HTTP/404)
+func (r *Response) Error() error {
+	if !r.HasError() {
+		return nil
+	}
+
+	var errs RequestErrors
+	_ = r.BindJSON(&errs)
+
+	e := &RequestError{}
+	if len(errs.Errors) > 0 {
+		e = &errs.Errors[0]
+	}
+
+	e.response = r.Response
+
+	return e
+}

remote/http_test.go

@@ -0,0 +1,79 @@
+package remote
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRequest(t *testing.T) {
+	c, _ := createTestClient(func(rw http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "application/vnd.pterodactyl.v1+json", r.Header.Get("Accept"))
+		assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
+		assert.Equal(t, "Bearer testid.testtoken", r.Header.Get("Authorization"))
+		assert.Equal(t, "/test", r.URL.Path)
+
+		rw.WriteHeader(http.StatusOK)
+	})
+	r, err := c.requestOnce(context.Background(), "", "/test", nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+}
+
+func TestRequestRetry(t *testing.T) {
+	// Test if the client retries failed requests
+	i := 0
+	c, _ := createTestClient(func(rw http.ResponseWriter, r *http.Request) {
+		if i < 1 {
+			rw.WriteHeader(http.StatusInternalServerError)
+		} else {
+			rw.WriteHeader(http.StatusOK)
+		}
+		i++
+	})
+	c.retries = 2
+	r, err := c.request(context.Background(), "", "", nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+	assert.Equal(t, http.StatusOK, r.StatusCode)
+	assert.Equal(t, 2, i)
+
+	// Test whether the client returns the last request after retry limit is reached
+	i = 0
+	c, _ = createTestClient(func(rw http.ResponseWriter, r *http.Request) {
+		rw.WriteHeader(http.StatusInternalServerError)
+		i++
+	})
+	c.retries = 2
+	r, err = c.request(context.Background(), "get", "", nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+	assert.Equal(t, http.StatusInternalServerError, r.StatusCode)
+	assert.Equal(t, 2, i)
+}
+
+func TestGet(t *testing.T) {
+	c, _ := createTestClient(func(rw http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodGet, r.Method)
+		assert.Len(t, r.URL.Query(), 1)
+		assert.Equal(t, "world", r.URL.Query().Get("hello"))
+	})
+	r, err := c.get(context.Background(), "/test", q{"hello": "world"})
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+}
+
+func TestPost(t *testing.T) {
+	test := map[string]string{
+		"hello": "world",
+	}
+	c, _ := createTestClient(func(rw http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodPost, r.Method)
+
+	})
+	r, err := c.post(context.Background(), "/test", test)
+	assert.NoError(t, err)
+	assert.NotNil(t, r)
+}

remote/servers.go

@@ -0,0 +1,167 @@
+package remote
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/pterodactyl/wings/api"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	ProcessStopCommand    = "command"
+	ProcessStopSignal     = "signal"
+	ProcessStopNativeStop = "stop"
+)
+
+// Holds the server configuration data returned from the Panel. When a server process
+// is started, Wings communicates with the Panel to fetch the latest build information
+// as well as get all of the details needed to parse the given Egg.
+//
+// This means we do not need to hit Wings each time part of the server is updated, and
+// the Panel serves as the source of truth at all times. This also means if a configuration
+// is accidentally wiped on Wings we can self-recover without too much hassle, so long
+// as Wings is aware of what servers should exist on it.
+type ServerConfigurationResponse struct {
+	Settings             json.RawMessage           `json:"settings"`
+	ProcessConfiguration *api.ProcessConfiguration `json:"process_configuration"`
+}
+
+// Defines installation script information for a server process. This is used when
+// a server is installed for the first time, and when a server is marked for re-installation.
+type InstallationScript struct {
+	ContainerImage string `json:"container_image"`
+	Entrypoint     string `json:"entrypoint"`
+	Script         string `json:"script"`
+}
+
+type allServerResponse struct {
+	Data []api.RawServerData `json:"data"`
+	Meta api.Pagination      `json:"meta"`
+}
+
+type RawServerData struct {
+	Uuid                 string          `json:"uuid"`
+	Settings             json.RawMessage `json:"settings"`
+	ProcessConfiguration json.RawMessage `json:"process_configuration"`
+}
+
+func (c *client) GetServersPaged(ctx context.Context, page, limit int) ([]api.RawServerData, api.Pagination, error) {
+	res, err := c.get(ctx, "/servers", q{
+		"page":     strconv.Itoa(page),
+		"per_page": strconv.Itoa(limit),
+	})
+	if err != nil {
+		return nil, api.Pagination{}, err
+	}
+	defer res.Body.Close()
+
+	if res.HasError() {
+		return nil, api.Pagination{}, res.Error()
+	}
+
+	var r allServerResponse
+	if err := res.BindJSON(&r); err != nil {
+		return nil, api.Pagination{}, err
+	}
+
+	return r.Data, r.Meta, nil
+}
+
+func (c *client) GetServers(ctx context.Context, perPage int) ([]api.RawServerData, error) {
+	servers, pageMeta, err := c.GetServersPaged(ctx, 0, perPage)
+	if err != nil {
+		return nil, err
+	}
+
+	// if the amount of servers exceeds the page limit, get the remaining pages in parallel
+	if pageMeta.LastPage > 1 {
+		eg, _ := errgroup.WithContext(ctx)
+		serversMu := sync.Mutex{}
+
+		for page := pageMeta.CurrentPage + 1; page <= pageMeta.LastPage; page++ {
+			eg.Go(func() error {
+				ps, _, err := c.GetServersPaged(ctx, perPage, int(page))
+				if err != nil {
+					return err
+				}
+				serversMu.Lock()
+				servers = append(servers, ps...)
+				serversMu.Unlock()
+				return nil
+			})
+		}
+
+		if err := eg.Wait(); err != nil {
+			return nil, err
+		}
+	}
+
+	return servers, nil
+}
+
+func (c *client) GetServerConfiguration(ctx context.Context, uuid string) (api.ServerConfigurationResponse, error) {
+	res, err := c.get(ctx, fmt.Sprintf("/servers/%s", uuid), nil)
+	if err != nil {
+		return api.ServerConfigurationResponse{}, err
+	}
+	defer res.Body.Close()
+
+	if res.HasError() {
+		return api.ServerConfigurationResponse{}, err
+	}
+
+	config := api.ServerConfigurationResponse{}
+	err = res.BindJSON(&config)
+	return config, err
+}
+
+func (c *client) GetInstallationScript(ctx context.Context, uuid string) (api.InstallationScript, error) {
+	res, err := c.get(ctx, fmt.Sprintf("/servers/%s/install", uuid), nil)
+	if err != nil {
+		return api.InstallationScript{}, err
+	}
+	defer res.Body.Close()
+
+	if res.HasError() {
+		return api.InstallationScript{}, err
+	}
+
+	config := api.InstallationScript{}
+	err = res.BindJSON(&config)
+	return config, err
+}
+
+func (c *client) SetInstallationStatus(ctx context.Context, uuid string, successful bool) error {
+	resp, err := c.post(ctx, fmt.Sprintf("/servers/%s/install", uuid), d{"successful": successful})
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return resp.Error()
+}
+
+func (c *client) SetArchiveStatus(ctx context.Context, uuid string, successful bool) error {
+	resp, err := c.post(ctx, fmt.Sprintf("/servers/%s/archive", uuid), d{"successful": successful})
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return resp.Error()
+}
+
+func (c *client) SetTransferStatus(ctx context.Context, uuid string, successful bool) error {
+	state := "failure"
+	if successful {
+		state = "success"
+	}
+	resp, err := c.get(ctx, fmt.Sprintf("/servers/%s/transfer/%s", uuid, state), nil)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	return resp.Error()
+}

remote/sftp.go

@@ -0,0 +1,50 @@
+package remote
+
+import (
+	"context"
+	"errors"
+	"regexp"
+
+	"github.com/apex/log"
+	"github.com/pterodactyl/wings/api"
+)
+
+// Usernames all follow the same format, so don't even bother hitting the API if the username is not
+// at least in the expected format. This is very basic protection against random bots finding the SFTP
+// server and sending a flood of usernames.
+var validUsernameRegexp = regexp.MustCompile(`^(?i)(.+)\.([a-z0-9]{8})$`)
+
+func (c *client) ValidateSftpCredentials(ctx context.Context, request api.SftpAuthRequest) (api.SftpAuthResponse, error) {
+	if !validUsernameRegexp.MatchString(request.User) {
+		log.WithFields(log.Fields{
+			"subsystem": "sftp",
+			"username":  request.User,
+			"ip":        request.IP,
+		}).Warn("failed to validate user credentials (invalid format)")
+		return api.SftpAuthResponse{}, new(sftpInvalidCredentialsError)
+	}
+
+	res, err := c.post(ctx, "/sftp/auth", request)
+	if err != nil {
+		return api.SftpAuthResponse{}, err
+	}
+
+	e := res.Error()
+	if e != nil {
+		if res.StatusCode >= 400 && res.StatusCode < 500 {
+			log.WithFields(log.Fields{
+				"subsystem": "sftp",
+				"username":  request.User,
+				"ip":        request.IP,
+			}).Warn(e.Error())
+
+			return api.SftpAuthResponse{}, &sftpInvalidCredentialsError{}
+		}
+
+		return api.SftpAuthResponse{}, errors.New(e.Error())
+	}
+
+	r := api.SftpAuthResponse{}
+	err = res.BindJSON(&r)
+	return r, err
+}

remote/util.go

@@ -0,0 +1,30 @@
+package remote
+
+import (
+	"net/http"
+
+	"github.com/apex/log"
+)
+
+// Logs the request into the debug log with all of the important request bits.
+// The authorization key will be cleaned up before being output.
+func debugLogRequest(req *http.Request) {
+	if l, ok := log.Log.(*log.Logger); ok && l.Level != log.DebugLevel {
+		return
+	}
+	headers := make(map[string][]string)
+	for k, v := range req.Header {
+		if k != "Authorization" || len(v) == 0 || len(v[0]) == 0 {
+			headers[k] = v
+			continue
+		}
+
+		headers[k] = []string{"(redacted)"}
+	}
+
+	log.WithFields(log.Fields{
+		"method":   req.Method,
+		"endpoint": req.URL.String(),
+		"headers":  headers,
+	}).Debug("making request to external HTTP endpoint")
+}

router/middleware.go

@@ -6,20 +6,11 @@ import (
 	"github.com/pterodactyl/wings/server"
 )
 
-// GetServer is a helper function to fetch a server out of the servers
-// collection stored in memory. This function should not be used in new
-// controllers, prefer ExtractServer where possible.
-// Deprecated
-func GetServer(uuid string) *server.Server {
-	return server.GetServers().Find(func(s *server.Server) bool {
-		return uuid == s.Id()
-	})
-}
-
 // ExtractServer returns the server instance from the gin context. If there is
-// no server set in the context (e.g. calling from a controller not protected by
+// no server set in the context (e.g. calling from a controller not protected
-// ServerExists) this function will panic.
+// by ServerExists) this function will panic.
-// Deprecated
+//
+// This function is deprecated. Use middleware.ExtractServer.
 func ExtractServer(c *gin.Context) *server.Server {
 	return middleware.ExtractServer(c)
 }

router/middleware/middleware.go

@@ -159,6 +159,15 @@ func AttachRequestID() gin.HandlerFunc {
 	}
 }
 
+// AttachServerManager attaches the server manager to the request context which
+// allows routes to access the underlying server collection.
+func AttachServerManager(m *server.Manager) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("manager", m)
+		c.Next()
+	}
+}
+
 // CaptureAndAbort aborts the request and attaches the provided error to the gin
 // context so it can be reported properly. If the error is missing a stacktrace
 // at the time it is called the stack will be attached.
@@ -239,9 +248,13 @@ func SetAccessControlHeaders() gin.HandlerFunc {
 // the server ID in the fields list.
 func ServerExists() gin.HandlerFunc {
 	return func(c *gin.Context) {
-		s := server.GetServers().Find(func(s *server.Server) bool {
+		var s *server.Server
-			return c.Param("server") == s.Id()
+		if c.Param("server") != "" {
-		})
+			manager := ExtractManager(c)
+			s = manager.Find(func(s *server.Server) bool {
+				return c.Param("server") == s.Id()
+			})
+		}
 		if s == nil {
 			c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": "The requested resource does not exist on this instance."})
 			return
@@ -313,3 +326,11 @@ func ExtractServer(c *gin.Context) *server.Server {
 	}
 	return v.(*server.Server)
 }
+
+// ExtractManager returns the server manager instance set on the request context.
+func ExtractManager(c *gin.Context) *server.Manager {
+	if v, ok := c.Get("manager"); ok {
+		return v.(*server.Manager)
+	}
+	panic("middleware/middleware: cannot extract server manager: not present in context")
+}

router/router.go

@@ -4,21 +4,22 @@ import (
 	"github.com/apex/log"
 	"github.com/gin-gonic/gin"
 	"github.com/pterodactyl/wings/router/middleware"
+	"github.com/pterodactyl/wings/server"
 )
 
 // Configure configures the routing infrastructure for this daemon instance.
-func Configure() *gin.Engine {
+func Configure(m *server.Manager) *gin.Engine {
 	gin.SetMode("release")
 
 	router := gin.New()
 	router.Use(gin.Recovery())
 	router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders())
+	router.Use(middleware.AttachServerManager(m))
 	// @todo log this into a different file so you can setup IP blocking for abusive requests and such.
 	// This should still dump requests in debug mode since it does help with understanding the request
 	// lifecycle and quickly seeing what was called leading to the logs. However, it isn't feasible to mix
 	// this output in production and still get meaningful logs from it since they'll likely just be a huge
 	// spamfest.
-	router.Use()
 	router.Use(gin.LoggerWithFormatter(func(params gin.LogFormatterParams) string {
 		log.WithFields(log.Fields{
 			"client_ip":  params.ClientIP,

router/router_download.go

@@ -8,20 +8,23 @@ import (
 	"strconv"
 
 	"github.com/gin-gonic/gin"
+	"github.com/pterodactyl/wings/router/middleware"
 	"github.com/pterodactyl/wings/router/tokens"
 	"github.com/pterodactyl/wings/server/backup"
 )
 
 // Handle a download request for a server backup.
 func getDownloadBackup(c *gin.Context) {
+	manager := middleware.ExtractManager(c)
+
 	token := tokens.BackupPayload{}
 	if err := tokens.ParseToken([]byte(c.Query("token")), &token); err != nil {
 		NewTrackedError(err).Abort(c)
 		return
 	}
 
-	s := GetServer(token.ServerUuid)
+	s, ok := manager.Get(token.ServerUuid)
-	if s == nil || !token.IsUniqueRequest() {
+	if !ok || !token.IsUniqueRequest() {
 		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
 			"error": "The requested resource was not found on this server.",
 		})
@@ -57,14 +60,15 @@ func getDownloadBackup(c *gin.Context) {
 
 // Handles downloading a specific file for a server.
 func getDownloadFile(c *gin.Context) {
+	manager := middleware.ExtractManager(c)
 	token := tokens.FilePayload{}
 	if err := tokens.ParseToken([]byte(c.Query("token")), &token); err != nil {
 		NewTrackedError(err).Abort(c)
 		return
 	}
 
-	s := GetServer(token.ServerUuid)
+	s, ok := manager.Get(token.ServerUuid)
-	if s == nil || !token.IsUniqueRequest() {
+	if !ok || !token.IsUniqueRequest() {
 		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
 			"error": "The requested resource was not found on this server.",
 		})

router/router_server.go

@@ -11,6 +11,7 @@ import (
 	"github.com/apex/log"
 	"github.com/gin-gonic/gin"
 	"github.com/pterodactyl/wings/router/downloader"
+	"github.com/pterodactyl/wings/router/middleware"
 	"github.com/pterodactyl/wings/router/tokens"
 	"github.com/pterodactyl/wings/server"
 )
@@ -22,7 +23,7 @@ type serverProcData struct {
 
 // Returns a single server from the collection of servers.
 func getServer(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	c.JSON(http.StatusOK, serverProcData{
 		ResourceUsage: s.Proc(),
@@ -32,7 +33,7 @@ func getServer(c *gin.Context) {
 
 // Returns the logs for a given server instance.
 func getServerLogs(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	l, _ := strconv.Atoi(c.DefaultQuery("size", "100"))
 	if l <= 0 {
@@ -59,7 +60,7 @@ func getServerLogs(c *gin.Context) {
 // things are happening, so theres no reason to sit and wait for a request to finish. We'll
 // just see over the socket if something isn't working correctly.
 func postServerPower(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Action server.PowerAction `json:"action"`
@@ -109,7 +110,7 @@ func postServerPower(c *gin.Context) {
 
 // Sends an array of commands to a running server instance.
 func postServerCommands(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	if running, err := s.Environment.IsRunning(); err != nil {
 		NewServerError(err, s).Abort(c)
@@ -140,7 +141,7 @@ func postServerCommands(c *gin.Context) {
 
 // Updates information about a server internally.
 func patchServer(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	buf := bytes.Buffer{}
 	buf.ReadFrom(c.Request.Body)
@@ -157,7 +158,7 @@ func patchServer(c *gin.Context) {
 
 // Performs a server installation in a background thread.
 func postServerInstall(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	go func(serv *server.Server) {
 		if err := serv.Install(true); err != nil {
@@ -170,7 +171,7 @@ func postServerInstall(c *gin.Context) {
 
 // Reinstalls a server.
 func postServerReinstall(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	if s.ExecutingPowerAction() {
 		c.AbortWithStatusJSON(http.StatusConflict, gin.H{
@@ -190,7 +191,7 @@ func postServerReinstall(c *gin.Context) {
 
 // Deletes a server from the wings daemon and dissociate it's objects.
 func deleteServer(c *gin.Context) {
-	s := ExtractServer(c)
+	s := middleware.ExtractServer(c)
 
 	// Immediately suspend the server to prevent a user from attempting
 	// to start it while this process is running.
@@ -234,9 +235,8 @@ func deleteServer(c *gin.Context) {
 		}
 	}(s.Filesystem().Path())
 
-	uuid := s.Id()
+	middleware.ExtractManager(c).Remove(func(server *server.Server) bool {
-	server.GetServers().Remove(func(s2 *server.Server) bool {
+		return server.Id() == s.Id()
-		return s2.Id() == uuid
 	})
 
 	// Deallocate the reference to this server.

router/router_server_files.go

@@ -71,7 +71,7 @@ type renameFile struct {
 
 // Renames (or moves) files for a server.
 func putServerRenameFiles(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Root  string       `json:"root"`
@@ -136,7 +136,7 @@ func putServerRenameFiles(c *gin.Context) {
 
 // Copies a server file.
 func postServerCopyFile(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Location string `json:"location"`
@@ -160,7 +160,7 @@ func postServerCopyFile(c *gin.Context) {
 
 // Deletes files from a server.
 func postServerDeleteFiles(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Root  string   `json:"root"`
@@ -205,7 +205,7 @@ func postServerDeleteFiles(c *gin.Context) {
 
 // Writes the contents of the request to a file on a server.
 func postServerWriteFile(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	f := c.Query("file")
 	f = "/" + strings.TrimLeft(f, "/")
@@ -306,7 +306,7 @@ func deleteServerPullRemoteFile(c *gin.Context) {
 
 // Create a directory on a server.
 func postServerCreateDirectory(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Name string `json:"name"`
@@ -333,7 +333,7 @@ func postServerCreateDirectory(c *gin.Context) {
 }
 
 func postServerCompressFiles(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		RootPath string   `json:"root"`
@@ -423,7 +423,7 @@ type chmodFile struct {
 var errInvalidFileMode = errors.New("invalid file mode")
 
 func postServerChmodFile(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	s := ExtractServer(c)
 
 	var data struct {
 		Root  string      `json:"root"`
@@ -487,14 +487,16 @@ func postServerChmodFile(c *gin.Context) {
 }
 
 func postServerUploadFiles(c *gin.Context) {
+	manager := middleware.ExtractManager(c)
+
 	token := tokens.UploadPayload{}
 	if err := tokens.ParseToken([]byte(c.Query("token")), &token); err != nil {
 		NewTrackedError(err).Abort(c)
 		return
 	}
 
-	s := GetServer(token.ServerUuid)
+	s, ok := manager.Get(token.ServerUuid)
-	if s == nil || !token.IsUniqueRequest() {
+	if !ok || !token.IsUniqueRequest() {
 		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
 			"error": "The requested resource was not found on this server.",
 		})

router/router_server_ws.go

@@ -7,12 +7,14 @@ import (
 
 	"github.com/gin-gonic/gin"
 	ws "github.com/gorilla/websocket"
+	"github.com/pterodactyl/wings/router/middleware"
 	"github.com/pterodactyl/wings/router/websocket"
 )
 
 // Upgrades a connection to a websocket and passes events along between.
 func getServerWebsocket(c *gin.Context) {
-	s := GetServer(c.Param("server"))
+	manager := middleware.ExtractManager(c)
+	s, _ := manager.Get(c.Param("server"))
 	handler, err := websocket.GetHandler(s, c.Writer, c.Request)
 	if err != nil {
 		NewServerError(err, s).Abort(c)

router/router_system.go

@@ -9,7 +9,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/installer"
-	"github.com/pterodactyl/wings/server"
+	"github.com/pterodactyl/wings/router/middleware"
 	"github.com/pterodactyl/wings/system"
 )
 
@@ -28,7 +28,7 @@ func getSystemInformation(c *gin.Context) {
 // Returns all of the servers that are registered and configured correctly on
 // this wings instance.
 func getAllServers(c *gin.Context) {
-	c.JSON(http.StatusOK, server.GetServers().All())
+	c.JSON(http.StatusOK, middleware.ExtractManager(c).All())
 }
 
 // Creates a new server on the wings daemon and begins the installation process
@@ -52,7 +52,8 @@ func postCreateServer(c *gin.Context) {
 
 	// Plop that server instance onto the request so that it can be referenced in
 	// requests from here-on out.
-	server.GetServers().Add(install.Server())
+	manager := middleware.ExtractManager(c)
+	manager.Add(install.Server())
 
 	// Begin the installation process in the background to not block the request
 	// cycle. If there are any errors they will be logged and communicated back

router/router_transfer.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pterodactyl/wings/api"
 	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/installer"
+	"github.com/pterodactyl/wings/router/middleware"
 	"github.com/pterodactyl/wings/router/tokens"
 	"github.com/pterodactyl/wings/server"
 	"github.com/pterodactyl/wings/system"
@@ -323,18 +324,19 @@ func postTransfer(c *gin.Context) {
 			i.Server().Events().Publish(server.TransferLogsEvent, output)
 		}
 
+		manager := middleware.ExtractManager(c)
 		// Mark the server as transferring to prevent problems later on during the process and
 		// then push the server into the global server collection for this instance.
 		i.Server().SetTransferring(true)
-		server.GetServers().Add(i.Server())
+		manager.Add(i.Server())
 		defer func(s *server.Server) {
 			// In the event that this transfer call fails, remove the server from the global
 			// server tracking so that we don't have a dangling instance.
 			if err := data.sendTransferStatus(!hasError); hasError || err != nil {
 				sendTransferLog("Server transfer failed, check Wings logs for additional information.")
 				s.Events().Publish(server.TransferStatusEvent, "failure")
-				server.GetServers().Remove(func(s2 *server.Server) bool {
+				manager.Remove(func(match *server.Server) bool {
-					return s.Id() == s2.Id()
+					return match.Id() == s.Id()
 				})
 
 				// If the transfer status was successful but the request failed, act like the transfer failed.

server/collection.go

@@ -1,77 +0,0 @@
-package server
-
-import "sync"
-
-type Collection struct {
-	items []*Server
-	sync.RWMutex
-}
-
-// Create a new collection from a slice of servers.
-func NewCollection(servers []*Server) *Collection {
-	return &Collection{
-		items: servers,
-	}
-}
-
-// Return all of the items in the collection.
-func (c *Collection) All() []*Server {
-	c.RLock()
-	defer c.RUnlock()
-
-	return c.items
-}
-
-// Adds an item to the collection store.
-func (c *Collection) Add(s *Server) {
-	c.Lock()
-	c.items = append(c.items, s)
-	c.Unlock()
-}
-
-// Returns only those items matching the filter criteria.
-func (c *Collection) Filter(filter func(*Server) bool) []*Server {
-	c.RLock()
-	defer c.RUnlock()
-
-	r := make([]*Server, 0)
-	for _, v := range c.items {
-		if filter(v) {
-			r = append(r, v)
-		}
-	}
-
-	return r
-}
-
-// Returns a single element from the collection matching the filter. If nothing is
-// found a nil result is returned.
-func (c *Collection) Find(filter func(*Server) bool) *Server {
-	c.RLock()
-	defer c.RUnlock()
-
-	for _, v := range c.items {
-		if filter(v) {
-			return v
-		}
-	}
-
-	return nil
-}
-
-// Removes all items from the collection that match the filter function.
-//
-// TODO: cancel the context?
-func (c *Collection) Remove(filter func(*Server) bool) {
-	c.Lock()
-	defer c.Unlock()
-
-	r := make([]*Server, 0)
-	for _, v := range c.items {
-		if !filter(v) {
-			r = append(r, v)
-		}
-	}
-
-	c.items = r
-}

server/loader.go

@@ -1,134 +0,0 @@
-package server
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"time"
-
-	"emperror.dev/errors"
-	"github.com/apex/log"
-	"github.com/gammazero/workerpool"
-	"github.com/pterodactyl/wings/api"
-	"github.com/pterodactyl/wings/config"
-	"github.com/pterodactyl/wings/environment"
-	"github.com/pterodactyl/wings/environment/docker"
-	"github.com/pterodactyl/wings/server/filesystem"
-)
-
-var servers = NewCollection(nil)
-
-func GetServers() *Collection {
-	return servers
-}
-
-// Iterates over a given directory and loads all of the servers listed before returning
-// them to the calling function.
-func LoadDirectory() error {
-	if len(servers.items) != 0 {
-		return errors.New("cannot call LoadDirectory with a non-nil collection")
-	}
-
-	log.Info("fetching list of servers from API")
-	configs, err := api.New().GetServers()
-	if err != nil {
-		if !api.IsRequestError(err) {
-			return err
-		}
-
-		return errors.New(err.Error())
-	}
-
-	start := time.Now()
-	log.WithField("total_configs", len(configs)).Info("processing servers returned by the API")
-
-	pool := workerpool.New(runtime.NumCPU())
-	log.Debugf("using %d workerpools to instantiate server instances", runtime.NumCPU())
-	for _, data := range configs {
-		data := data
-
-		pool.Submit(func() {
-			// Parse the json.RawMessage into an expected struct value. We do this here so that a single broken
-			// server does not cause the entire boot process to hang, and allows us to show more useful error
-			// messaging in the output.
-			d := api.ServerConfigurationResponse{
-				Settings: data.Settings,
-			}
-
-			log.WithField("server", data.Uuid).Info("creating new server object from API response")
-			if err := json.Unmarshal(data.ProcessConfiguration, &d.ProcessConfiguration); err != nil {
-				log.WithField("server", data.Uuid).WithField("error", err).Error("failed to parse server configuration from API response, skipping...")
-				return
-			}
-
-			s, err := FromConfiguration(d)
-			if err != nil {
-				log.WithField("server", data.Uuid).WithField("error", err).Error("failed to load server, skipping...")
-				return
-			}
-
-			servers.Add(s)
-		})
-	}
-
-	// Wait until we've processed all of the configuration files in the directory
-	// before continuing.
-	pool.StopWait()
-
-	diff := time.Now().Sub(start)
-	log.WithField("duration", fmt.Sprintf("%s", diff)).Info("finished processing server configurations")
-
-	return nil
-}
-
-// Initializes a server using a data byte array. This will be marshaled into the
-// given struct using a YAML marshaler. This will also configure the given environment
-// for a server.
-func FromConfiguration(data api.ServerConfigurationResponse) (*Server, error) {
-	s, err := New()
-	if err != nil {
-		return nil, errors.WithMessage(err, "loader: failed to instantiate empty server struct")
-	}
-	if err := s.UpdateDataStructure(data.Settings); err != nil {
-		return nil, err
-	}
-
-	s.Archiver = Archiver{Server: s}
-	s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist)
-
-	// Right now we only support a Docker based environment, so I'm going to hard code
-	// this logic in. When we're ready to support other environment we'll need to make
-	// some modifications here obviously.
-	settings := environment.Settings{
-		Mounts:      s.Mounts(),
-		Allocations: s.cfg.Allocations,
-		Limits:      s.cfg.Build,
-	}
-
-	envCfg := environment.NewConfiguration(settings, s.GetEnvironmentVariables())
-	meta := docker.Metadata{
-		Image: s.Config().Container.Image,
-	}
-
-	if env, err := docker.New(s.Id(), &meta, envCfg); err != nil {
-		return nil, err
-	} else {
-		s.Environment = env
-		s.StartEventListeners()
-		s.Throttler().StartTimer(s.Context())
-	}
-
-	// Forces the configuration to be synced with the panel.
-	if err := s.SyncWithConfiguration(data); err != nil {
-		return nil, err
-	}
-
-	// If the server's data directory exists, force disk usage calculation.
-	if _, err := os.Stat(s.Filesystem().Path()); err == nil {
-		s.Filesystem().HasSpaceAvailable(true)
-	}
-
-	return s, nil
-}

server/manager.go

@@ -0,0 +1,204 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"emperror.dev/errors"
+	"github.com/apex/log"
+	"github.com/gammazero/workerpool"
+	"github.com/pterodactyl/wings/api"
+	"github.com/pterodactyl/wings/config"
+	"github.com/pterodactyl/wings/remote"
+)
+
+type Manager struct {
+	mu      sync.RWMutex
+	servers []*Server
+}
+
+// NewManager returns a new server manager instance. This will boot up all of
+// the servers that are currently present on the filesystem and set them into
+// the manager.
+func NewManager(ctx context.Context, client remote.Client) (*Manager, error) {
+	c := NewEmptyManager()
+	if err := c.initializeFromRemoteSource(ctx, client); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+// NewEmptyManager returns a new empty manager collection without actually
+// loading any of the servers from the disk. This allows the caller to set their
+// own servers into the collection as needed.
+func NewEmptyManager() *Manager {
+	return &Manager{}
+}
+
+// initializeFromRemoteSource iterates over a given directory and loads all of
+// the servers listed before returning them to the calling function.
+func (m *Manager) initializeFromRemoteSource(ctx context.Context, client remote.Client) error {
+	log.Info("fetching list of servers from API")
+	servers, err := client.GetServers(ctx, config.Get().RemoteQuery.BootServersPerPage)
+	if err != nil {
+		if !remote.IsRequestError(err) {
+			return errors.WithStackIf(err)
+		}
+		return errors.New(err.Error())
+	}
+
+	start := time.Now()
+	log.WithField("total_configs", len(servers)).Info("processing servers returned by the API")
+
+	pool := workerpool.New(runtime.NumCPU())
+	log.Debugf("using %d workerpools to instantiate server instances", runtime.NumCPU())
+	for _, data := range servers {
+		data := data
+		pool.Submit(func() {
+			// Parse the json.RawMessage into an expected struct value. We do this here so that a single broken
+			// server does not cause the entire boot process to hang, and allows us to show more useful error
+			// messaging in the output.
+			d := api.ServerConfigurationResponse{
+				Settings: data.Settings,
+			}
+			log.WithField("server", data.Uuid).Info("creating new server object from API response")
+			if err := json.Unmarshal(data.ProcessConfiguration, &d.ProcessConfiguration); err != nil {
+				log.WithField("server", data.Uuid).WithField("error", err).Error("failed to parse server configuration from API response, skipping...")
+				return
+			}
+			s, err := FromConfiguration(d)
+			if err != nil {
+				log.WithField("server", data.Uuid).WithField("error", err).Error("failed to load server, skipping...")
+				return
+			}
+			m.Add(s)
+		})
+	}
+
+	// Wait until we've processed all of the configuration files in the directory
+	// before continuing.
+	pool.StopWait()
+
+	diff := time.Now().Sub(start)
+	log.WithField("duration", fmt.Sprintf("%s", diff)).Info("finished processing server configurations")
+
+	return nil
+}
+
+// Put replaces all of the current values in the collection with the value that
+// is passed through.
+func (m *Manager) Put(s []*Server) {
+	m.mu.Lock()
+	m.servers = s
+	m.mu.Unlock()
+}
+
+// All returns all of the items in the collection.
+func (m *Manager) All() []*Server {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return m.servers
+}
+
+// Add adds an item to the collection store.
+func (m *Manager) Add(s *Server) {
+	m.mu.Lock()
+	m.servers = append(m.servers, s)
+	m.mu.Unlock()
+}
+
+// Get returns a single server instance and a boolean value indicating if it was
+// found in the global collection or not.
+func (m *Manager) Get(uuid string) (*Server, bool) {
+	match := m.Find(func(server *Server) bool {
+		return server.Id() == uuid
+	})
+	return match, match != nil
+}
+
+// Filter returns only those items matching the filter criteria.
+func (m *Manager) Filter(filter func(match *Server) bool) []*Server {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	r := make([]*Server, 0)
+	for _, v := range m.servers {
+		if filter(v) {
+			r = append(r, v)
+		}
+	}
+	return r
+}
+
+// Find returns a single element from the collection matching the filter. If
+// nothing is found a nil result is returned.
+func (m *Manager) Find(filter func(match *Server) bool) *Server {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	for _, v := range m.servers {
+		if filter(v) {
+			return v
+		}
+	}
+	return nil
+}
+
+// Remove removes all items from the collection that match the filter function.
+func (m *Manager) Remove(filter func(match *Server) bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	r := make([]*Server, 0)
+	for _, v := range m.servers {
+		if !filter(v) {
+			r = append(r, v)
+		}
+	}
+	m.servers = r
+}
+
+// PersistStates writes the current environment states to the disk for each
+// server. This is generally called at a specific interval defined in the root
+// runner command to avoid hammering disk I/O when tons of server switch states
+// at once. It is fine if this file falls slightly out of sync, it is just here
+// to make recovering from an unexpected system reboot a little easier.
+func (m *Manager) PersistStates() error {
+	states := map[string]string{}
+	for _, s := range m.All() {
+		states[s.Id()] = s.Environment.State()
+	}
+	data, err := json.Marshal(states)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	if err := ioutil.WriteFile(config.Get().System.GetStatesPath(), data, 0644); err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+// ReadStates returns the state of the servers.
+func (m *Manager) ReadStates() (map[string]string, error) {
+	f, err := os.OpenFile(config.Get().System.GetStatesPath(), os.O_RDONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	defer f.Close()
+	var states map[string]string
+	if err := json.NewDecoder(f).Decode(&states); err != nil && err != io.EOF {
+		return nil, errors.WithStack(err)
+	}
+	out := make(map[string]string, 0)
+	// Only return states for servers that we're currently tracking in the system.
+	for id, state := range states {
+		if _, ok := m.Get(id); ok {
+			out[id] = state
+		}
+	}
+	return out, nil
+}

server/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 
@@ -20,7 +21,8 @@ import (
 	"golang.org/x/sync/semaphore"
 )
 
-// High level definition for a server instance being controlled by Wings.
+// Server is the high level definition for a server instance being controlled
+// by Wings.
 type Server struct {
 	// Internal mutex used to block actions that need to occur sequentially, such as
 	// writing the configuration to the disk.
@@ -246,3 +248,108 @@ func (s *Server) EnsureDataDirectoryExists() error {
 	}
 	return nil
 }
+
+// Sets the state of the server internally. This function handles crash detection as
+// well as reporting to event listeners for the server.
+func (s *Server) OnStateChange() {
+	prevState := s.resources.State.Load()
+
+	st := s.Environment.State()
+	// Update the currently tracked state for the server.
+	s.resources.State.Store(st)
+
+	// Emit the event to any listeners that are currently registered.
+	if prevState != s.Environment.State() {
+		s.Log().WithField("status", st).Debug("saw server status change event")
+		s.Events().Publish(StatusEvent, st)
+	}
+
+	// Reset the resource usage to 0 when the process fully stops so that all of the UI
+	// views in the Panel correctly display 0.
+	if st == environment.ProcessOfflineState {
+		s.resources.Reset()
+		s.emitProcUsage()
+	}
+
+	// If server was in an online state, and is now in an offline state we should handle
+	// that as a crash event. In that scenario, check the last crash time, and the crash
+	// counter.
+	//
+	// In the event that we have passed the thresholds, don't do anything, otherwise
+	// automatically attempt to start the process back up for the user. This is done in a
+	// separate thread as to not block any actions currently taking place in the flow
+	// that called this function.
+	if (prevState == environment.ProcessStartingState || prevState == environment.ProcessRunningState) && s.Environment.State() == environment.ProcessOfflineState {
+		s.Log().Info("detected server as entering a crashed state; running crash handler")
+
+		go func(server *Server) {
+			if err := server.handleServerCrash(); err != nil {
+				if IsTooFrequentCrashError(err) {
+					server.Log().Info("did not restart server after crash; occurred too soon after the last")
+				} else {
+					s.PublishConsoleOutputFromDaemon("Server crash was detected but an error occurred while handling it.")
+					server.Log().WithField("error", err).Error("failed to handle server crash")
+				}
+			}
+		}(s)
+	}
+}
+
+// Determines if the server state is running or not. This is different than the
+// environment state, it is simply the tracked state from this daemon instance, and
+// not the response from Docker.
+func (s *Server) IsRunning() bool {
+	st := s.Environment.State()
+
+	return st == environment.ProcessRunningState || st == environment.ProcessStartingState
+}
+
+// FromConfiguration initializes a server using a data byte array. This will be
+// marshaled into the given struct using a YAML marshaler. This will also
+// configure the given environment for a server.
+func FromConfiguration(data api.ServerConfigurationResponse) (*Server, error) {
+	s, err := New()
+	if err != nil {
+		return nil, errors.WithMessage(err, "loader: failed to instantiate empty server struct")
+	}
+	if err := s.UpdateDataStructure(data.Settings); err != nil {
+		return nil, err
+	}
+
+	s.Archiver = Archiver{Server: s}
+	s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist)
+
+	// Right now we only support a Docker based environment, so I'm going to hard code
+	// this logic in. When we're ready to support other environment we'll need to make
+	// some modifications here obviously.
+	settings := environment.Settings{
+		Mounts:      s.Mounts(),
+		Allocations: s.cfg.Allocations,
+		Limits:      s.cfg.Build,
+	}
+
+	envCfg := environment.NewConfiguration(settings, s.GetEnvironmentVariables())
+	meta := docker.Metadata{
+		Image: s.Config().Container.Image,
+	}
+
+	if env, err := docker.New(s.Id(), &meta, envCfg); err != nil {
+		return nil, err
+	} else {
+		s.Environment = env
+		s.StartEventListeners()
+		s.Throttler().StartTimer(s.Context())
+	}
+
+	// Forces the configuration to be synced with the panel.
+	if err := s.SyncWithConfiguration(data); err != nil {
+		return nil, err
+	}
+
+	// If the server's data directory exists, force disk usage calculation.
+	if _, err := os.Stat(s.Filesystem().Path()); err == nil {
+		s.Filesystem().HasSpaceAvailable(true)
+	}
+
+	return s, nil
+}

server/state.go

@@ -1,137 +0,0 @@
-package server
-
-import (
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"os"
-	"sync"
-
-	"github.com/pterodactyl/wings/config"
-	"github.com/pterodactyl/wings/environment"
-)
-
-var stateMutex sync.Mutex
-
-// Returns the state of the servers.
-func CachedServerStates() (map[string]string, error) {
-	// Request a lock after we check if the file exists.
-	stateMutex.Lock()
-	defer stateMutex.Unlock()
-
-	// Open the states file.
-	f, err := os.OpenFile(config.Get().System.GetStatesPath(), os.O_RDONLY|os.O_CREATE, 0644)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	// Convert the json object to a map.
-	states := map[string]string{}
-	if err := json.NewDecoder(f).Decode(&states); err != nil && err != io.EOF {
-		return nil, err
-	}
-
-	return states, nil
-}
-
-// saveServerStates .
-func saveServerStates() error {
-	// Get the states of all servers on the daemon.
-	states := map[string]string{}
-	for _, s := range GetServers().All() {
-		states[s.Id()] = s.Environment.State()
-	}
-
-	// Convert the map to a json object.
-	data, err := json.Marshal(states)
-	if err != nil {
-		return err
-	}
-
-	stateMutex.Lock()
-	defer stateMutex.Unlock()
-
-	// Write the data to the file
-	if err := ioutil.WriteFile(config.Get().System.GetStatesPath(), data, 0644); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Sets the state of the server internally. This function handles crash detection as
-// well as reporting to event listeners for the server.
-func (s *Server) OnStateChange() {
-	prevState := s.resources.State.Load()
-
-	st := s.Environment.State()
-	// Update the currently tracked state for the server.
-	s.resources.State.Store(st)
-
-	// Emit the event to any listeners that are currently registered.
-	if prevState != s.Environment.State() {
-		s.Log().WithField("status", st).Debug("saw server status change event")
-		s.Events().Publish(StatusEvent, st)
-	}
-
-	// Persist this change to the disk immediately so that should the Daemon be stopped or
-	// crash we can immediately restore the server state.
-	//
-	// This really only makes a difference if all of the Docker containers are also stopped,
-	// but this was a highly requested feature and isn't hard to work with, so lets do it.
-	//
-	// We also get the benefit of server status changes always propagating corrected configurations
-	// to the disk should we forget to do it elsewhere.
-	go func() {
-		if err := saveServerStates(); err != nil {
-			s.Log().WithField("error", err).Warn("failed to write server states to disk")
-		}
-	}()
-
-	// Reset the resource usage to 0 when the process fully stops so that all of the UI
-	// views in the Panel correctly display 0.
-	if st == environment.ProcessOfflineState {
-		s.resources.Reset()
-		s.emitProcUsage()
-	}
-
-	// If server was in an online state, and is now in an offline state we should handle
-	// that as a crash event. In that scenario, check the last crash time, and the crash
-	// counter.
-	//
-	// In the event that we have passed the thresholds, don't do anything, otherwise
-	// automatically attempt to start the process back up for the user. This is done in a
-	// separate thread as to not block any actions currently taking place in the flow
-	// that called this function.
-	if (prevState == environment.ProcessStartingState || prevState == environment.ProcessRunningState) && s.Environment.State() == environment.ProcessOfflineState {
-		s.Log().Info("detected server as entering a crashed state; running crash handler")
-
-		go func(server *Server) {
-			if err := server.handleServerCrash(); err != nil {
-				if IsTooFrequentCrashError(err) {
-					server.Log().Info("did not restart server after crash; occurred too soon after the last")
-				} else {
-					s.PublishConsoleOutputFromDaemon("Server crash was detected but an error occurred while handling it.")
-					server.Log().WithField("error", err).Error("failed to handle server crash")
-				}
-			}
-		}(s)
-	}
-}
-
-// Returns the current state of the server in a race-safe manner.
-// Deprecated
-// use Environment.State()
-func (s *Server) GetState() string {
-	return s.Environment.State()
-}
-
-// Determines if the server state is running or not. This is different than the
-// environment state, it is simply the tracked state from this daemon instance, and
-// not the response from Docker.
-func (s *Server) IsRunning() bool {
-	st := s.Environment.State()
-
-	return st == environment.ProcessRunningState || st == environment.ProcessStartingState
-}

sftp/server.go

@@ -24,14 +24,16 @@ import (
 
 //goland:noinspection GoNameStartsWithPackageName
 type SFTPServer struct {
+	manager  *server.Manager
 	BasePath string
 	ReadOnly bool
 	Listen   string
 }
 
-func New() *SFTPServer {
+func New(m *server.Manager) *SFTPServer {
 	cfg := config.Get().System
 	return &SFTPServer{
+		manager:  m,
 		BasePath: cfg.Data,
 		ReadOnly: cfg.Sftp.ReadOnly,
 		Listen:   cfg.Sftp.Address + ":" + strconv.Itoa(cfg.Sftp.Port),
@@ -81,7 +83,7 @@ func (c *SFTPServer) Run() error {
 
 // Handles an inbound connection to the instance and determines if we should serve the
 // request or not.
-func (c SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) {
+func (c *SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) {
 	// Before beginning a handshake must be performed on the incoming net.Conn
 	sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
 	if err != nil {
@@ -119,7 +121,7 @@ func (c SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) {
 		// This will also attempt to match a specific server out of the global server
 		// store and return nil if there is no match.
 		uuid := sconn.Permissions.Extensions["uuid"]
-		srv := server.GetServers().Find(func(s *server.Server) bool {
+		srv := c.manager.Find(func(s *server.Server) bool {
 			if uuid == "" {
 				return false
 			}