Use a paginated loop to fetch servers from the Panel rather than a single massive request

api/api.go

@@ -37,9 +37,19 @@ type Response struct {
 	*http.Response
 }
 
+// A pagination struct matching the expected pagination response from the Panel API.
+type Pagination struct {
+	CurrentPage uint `json:"current_page"`
+	From        uint `json:"from"`
+	LastPage    uint `json:"last_page"`
+	PerPage     uint `json:"per_page"`
+	To          uint `json:"to"`
+	Total       uint `json:"total"`
+}
+
 // Builds the base request instance that can be used with the HTTP client.
 func (r *Request) Client() *http.Client {
-	return &http.Client{Timeout: time.Second * 30}
+	return &http.Client{Timeout: time.Second * time.Duration(config.Get().RemoteQuery.Timeout)}
 }
 
 // Returns the given endpoint formatted as a URL to the Panel API.

api/server_endpoints.go

@@ -1,9 +1,14 @@
 package api
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/apex/log"
 	"github.com/pkg/errors"
+	"github.com/pterodactyl/wings/config"
+	"golang.org/x/sync/errgroup"
+	"sync"
 )
 
 const (
@@ -33,9 +38,17 @@ type InstallationScript struct {
 	Script         string `json:"script"`
 }
 
-// GetAllServerConfigurations fetches configurations for all servers assigned to this node.
-func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, error) {
-	resp, err := r.Get("/servers", nil)
+type allServerResponse struct {
+	Data []json.RawMessage `json:"data"`
+	Meta Pagination        `json:"meta"`
+}
+
+// Fetches all of the server configurations from the Panel API. This will initially load the
+// first 50 servers, and then check the pagination response to determine if more pages should
+// be loaded. If so, those requests are spun-up in additional routines and the final resulting
+// slice of all servers will be returned.
+func (r *Request) GetServers() ([]json.RawMessage, error) {
+	resp, err := r.Get("/servers", D{"per_page": config.Get().RemoteQuery.BootServersPerPage})
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -45,12 +58,63 @@ func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, erro
 		return nil, resp.Error()
 	}
 
-	var res map[string]json.RawMessage
+	var res allServerResponse
 	if err := resp.Bind(&res); err != nil {
 		return nil, errors.WithStack(err)
 	}
 
-	return res, nil
+	var mu sync.Mutex
+	ret := res.Data
+
+	// Check for pagination, and if it exists we'll need to then make a request to the API
+	// for each page that would exist and get all of the resulting servers.
+	if res.Meta.LastPage > 1 {
+		pp := res.Meta.PerPage
+		log.WithField("per_page", pp).
+			WithField("total_pages", res.Meta.LastPage).
+			Debug("detected multiple pages of server configurations, fetching remaining...")
+
+		g, ctx := errgroup.WithContext(context.Background())
+		for i := res.Meta.CurrentPage; i <= res.Meta.LastPage; i++ {
+			page := i
+
+			g.Go(func() error {
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				default:
+					{
+						resp, err := r.Get("/servers", D{"page": page, "per_page": pp})
+						if err != nil {
+							return err
+						}
+						defer resp.Body.Close()
+
+						if resp.Error() != nil {
+							return resp.Error()
+						}
+
+						var servers allServerResponse
+						if err := resp.Bind(&servers); err != nil {
+							return err
+						}
+
+						mu.Lock()
+						defer mu.Unlock()
+						ret = append(ret, servers.Data...)
+
+						return nil
+					}
+				}
+			})
+		}
+
+		if err := g.Wait(); err != nil {
+			return nil, errors.WithStack(err)
+		}
+	}
+
+	return ret, nil
 }
 
 // Fetches the server configuration and returns the struct for it.
@@ -91,7 +155,6 @@ func (r *Request) GetInstallationScript(uuid string) (InstallationScript, error)
 		return is, errors.WithStack(err)
 	}
 
-
 	return is, nil
 }
 

config/config.go

@@ -59,7 +59,8 @@ type Configuration struct {
 
 	// The location where the panel is running that this daemon should connect to
 	// to collect data and send events.
-	PanelLocation string `json:"remote" yaml:"remote"`
+	PanelLocation string                   `json:"remote" yaml:"remote"`
+	RemoteQuery   RemoteQueryConfiguration `json:"remote_query" yaml:"remote_query"`
 
 	// AllowedMounts is a list of allowed host-system mount points.
 	// This is required to have the "Server Mounts" feature work properly.
@@ -101,6 +102,27 @@ type ApiConfiguration struct {
 	UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
 }
 
+// Defines the configuration settings for remote requests from Wings to the Panel.
+type RemoteQueryConfiguration struct {
+	// The amount of time in seconds that Wings should allow for a request to the Panel API
+	// to complete. If this time passes the request will be marked as failed. If your requests
+	// are taking longer than 30 seconds to complete it is likely a performance issue that
+	// should be resolved on the Panel, and not something that should be resolved by upping this
+	// number.
+	Timeout uint `default:"30" yaml:"timeout"`
+
+	// The number of servers to load in a single request to the Panel API when booting the
+	// Wings instance. A single request is initially made to the Panel to get this number
+	// of servers, and then the pagination status is checked and additional requests are
+	// fired off in parallel to request the remaining pages.
+	//
+	// It is not recommended to change this from the default as you will likely encounter
+	// memory limits on your Panel instance. In the grand scheme of things 4 requests for
+	// 50 servers is likely just as quick as two for 100 or one for 400, and will certainly
+	// be less likely to cause performance issues on the Panel.
+	BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"`
+}
+
 // Reads the configuration from the provided file and returns the configuration
 // object that can then be used.
 func ReadConfiguration(path string) (*Configuration, error) {

server/loader.go

@@ -32,7 +32,7 @@ func LoadDirectory() error {
 	}
 
 	log.Info("fetching list of servers from API")
-	configs, err := api.New().GetAllServerConfigurations()
+	configs, err := api.New().GetServers()
 	if err != nil {
 		if !api.IsRequestError(err) {
 			return errors.WithStack(err)