diff --git a/api/api.go b/api/api.go index 3367445..7dabba4 100644 --- a/api/api.go +++ b/api/api.go @@ -37,9 +37,19 @@ type Response struct { *http.Response } +// A pagination struct matching the expected pagination response from the Panel API. +type Pagination struct { + CurrentPage uint `json:"current_page"` + From uint `json:"from"` + LastPage uint `json:"last_page"` + PerPage uint `json:"per_page"` + To uint `json:"to"` + Total uint `json:"total"` +} + // Builds the base request instance that can be used with the HTTP client. func (r *Request) Client() *http.Client { - return &http.Client{Timeout: time.Second * 30} + return &http.Client{Timeout: time.Second * time.Duration(config.Get().RemoteQuery.Timeout)} } // Returns the given endpoint formatted as a URL to the Panel API. diff --git a/api/server_endpoints.go b/api/server_endpoints.go index b44f287..ade271d 100644 --- a/api/server_endpoints.go +++ b/api/server_endpoints.go @@ -1,9 +1,14 @@ package api import ( + "context" "encoding/json" "fmt" + "github.com/apex/log" "github.com/pkg/errors" + "github.com/pterodactyl/wings/config" + "golang.org/x/sync/errgroup" + "sync" ) const ( @@ -33,9 +38,17 @@ type InstallationScript struct { Script string `json:"script"` } -// GetAllServerConfigurations fetches configurations for all servers assigned to this node. -func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, error) { - resp, err := r.Get("/servers", nil) +type allServerResponse struct { + Data []json.RawMessage `json:"data"` + Meta Pagination `json:"meta"` +} + +// Fetches all of the server configurations from the Panel API. This will initially load the +// first 50 servers, and then check the pagination response to determine if more pages should +// be loaded. If so, those requests are spun-up in additional routines and the final resulting +// slice of all servers will be returned. +func (r *Request) GetServers() ([]json.RawMessage, error) { + resp, err := r.Get("/servers", D{"per_page": config.Get().RemoteQuery.BootServersPerPage}) if err != nil { return nil, errors.WithStack(err) } @@ -45,12 +58,63 @@ func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, erro return nil, resp.Error() } - var res map[string]json.RawMessage + var res allServerResponse if err := resp.Bind(&res); err != nil { return nil, errors.WithStack(err) } - return res, nil + var mu sync.Mutex + ret := res.Data + + // Check for pagination, and if it exists we'll need to then make a request to the API + // for each page that would exist and get all of the resulting servers. + if res.Meta.LastPage > 1 { + pp := res.Meta.PerPage + log.WithField("per_page", pp). + WithField("total_pages", res.Meta.LastPage). + Debug("detected multiple pages of server configurations, fetching remaining...") + + g, ctx := errgroup.WithContext(context.Background()) + for i := res.Meta.CurrentPage; i <= res.Meta.LastPage; i++ { + page := i + + g.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + { + resp, err := r.Get("/servers", D{"page": page, "per_page": pp}) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.Error() != nil { + return resp.Error() + } + + var servers allServerResponse + if err := resp.Bind(&servers); err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + ret = append(ret, servers.Data...) + + return nil + } + } + }) + } + + if err := g.Wait(); err != nil { + return nil, errors.WithStack(err) + } + } + + return ret, nil } // Fetches the server configuration and returns the struct for it. @@ -91,7 +155,6 @@ func (r *Request) GetInstallationScript(uuid string) (InstallationScript, error) return is, errors.WithStack(err) } - return is, nil } diff --git a/config/config.go b/config/config.go index e0027f1..26b6e28 100644 --- a/config/config.go +++ b/config/config.go @@ -59,7 +59,8 @@ type Configuration struct { // The location where the panel is running that this daemon should connect to // to collect data and send events. - PanelLocation string `json:"remote" yaml:"remote"` + PanelLocation string `json:"remote" yaml:"remote"` + RemoteQuery RemoteQueryConfiguration `json:"remote_query" yaml:"remote_query"` // AllowedMounts is a list of allowed host-system mount points. // This is required to have the "Server Mounts" feature work properly. @@ -101,6 +102,27 @@ type ApiConfiguration struct { UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"` } +// Defines the configuration settings for remote requests from Wings to the Panel. +type RemoteQueryConfiguration struct { + // The amount of time in seconds that Wings should allow for a request to the Panel API + // to complete. If this time passes the request will be marked as failed. If your requests + // are taking longer than 30 seconds to complete it is likely a performance issue that + // should be resolved on the Panel, and not something that should be resolved by upping this + // number. + Timeout uint `default:"30" yaml:"timeout"` + + // The number of servers to load in a single request to the Panel API when booting the + // Wings instance. A single request is initially made to the Panel to get this number + // of servers, and then the pagination status is checked and additional requests are + // fired off in parallel to request the remaining pages. + // + // It is not recommended to change this from the default as you will likely encounter + // memory limits on your Panel instance. In the grand scheme of things 4 requests for + // 50 servers is likely just as quick as two for 100 or one for 400, and will certainly + // be less likely to cause performance issues on the Panel. + BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"` +} + // Reads the configuration from the provided file and returns the configuration // object that can then be used. func ReadConfiguration(path string) (*Configuration, error) { diff --git a/server/loader.go b/server/loader.go index fef3904..f46ff81 100644 --- a/server/loader.go +++ b/server/loader.go @@ -32,7 +32,7 @@ func LoadDirectory() error { } log.Info("fetching list of servers from API") - configs, err := api.New().GetAllServerConfigurations() + configs, err := api.New().GetServers() if err != nil { if !api.IsRequestError(err) { return errors.WithStack(err)