Use a paginated loop to fetch servers from the Panel rather than a single massive request

This commit is contained in:
Dane Everitt 2020-10-31 11:05:53 -07:00
parent 334b3e8d10
commit 41a67933eb
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
4 changed files with 104 additions and 9 deletions

View File

@ -37,9 +37,19 @@ type Response struct {
*http.Response
}
// A pagination struct matching the expected pagination response from the Panel API.
type Pagination struct {
CurrentPage uint `json:"current_page"`
From uint `json:"from"`
LastPage uint `json:"last_page"`
PerPage uint `json:"per_page"`
To uint `json:"to"`
Total uint `json:"total"`
}
// Builds the base request instance that can be used with the HTTP client.
func (r *Request) Client() *http.Client {
return &http.Client{Timeout: time.Second * 30}
return &http.Client{Timeout: time.Second * time.Duration(config.Get().RemoteQuery.Timeout)}
}
// Returns the given endpoint formatted as a URL to the Panel API.

View File

@ -1,9 +1,14 @@
package api
import (
"context"
"encoding/json"
"fmt"
"github.com/apex/log"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/config"
"golang.org/x/sync/errgroup"
"sync"
)
const (
@ -33,9 +38,17 @@ type InstallationScript struct {
Script string `json:"script"`
}
// GetAllServerConfigurations fetches configurations for all servers assigned to this node.
func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, error) {
resp, err := r.Get("/servers", nil)
type allServerResponse struct {
Data []json.RawMessage `json:"data"`
Meta Pagination `json:"meta"`
}
// Fetches all of the server configurations from the Panel API. This will initially load the
// first 50 servers, and then check the pagination response to determine if more pages should
// be loaded. If so, those requests are spun-up in additional routines and the final resulting
// slice of all servers will be returned.
func (r *Request) GetServers() ([]json.RawMessage, error) {
resp, err := r.Get("/servers", D{"per_page": config.Get().RemoteQuery.BootServersPerPage})
if err != nil {
return nil, errors.WithStack(err)
}
@ -45,12 +58,63 @@ func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, erro
return nil, resp.Error()
}
var res map[string]json.RawMessage
var res allServerResponse
if err := resp.Bind(&res); err != nil {
return nil, errors.WithStack(err)
}
return res, nil
var mu sync.Mutex
ret := res.Data
// Check for pagination, and if it exists we'll need to then make a request to the API
// for each page that would exist and get all of the resulting servers.
if res.Meta.LastPage > 1 {
pp := res.Meta.PerPage
log.WithField("per_page", pp).
WithField("total_pages", res.Meta.LastPage).
Debug("detected multiple pages of server configurations, fetching remaining...")
g, ctx := errgroup.WithContext(context.Background())
for i := res.Meta.CurrentPage; i <= res.Meta.LastPage; i++ {
page := i
g.Go(func() error {
select {
case <-ctx.Done():
return ctx.Err()
default:
{
resp, err := r.Get("/servers", D{"page": page, "per_page": pp})
if err != nil {
return err
}
defer resp.Body.Close()
if resp.Error() != nil {
return resp.Error()
}
var servers allServerResponse
if err := resp.Bind(&servers); err != nil {
return err
}
mu.Lock()
defer mu.Unlock()
ret = append(ret, servers.Data...)
return nil
}
}
})
}
if err := g.Wait(); err != nil {
return nil, errors.WithStack(err)
}
}
return ret, nil
}
// Fetches the server configuration and returns the struct for it.
@ -91,7 +155,6 @@ func (r *Request) GetInstallationScript(uuid string) (InstallationScript, error)
return is, errors.WithStack(err)
}
return is, nil
}

View File

@ -59,7 +59,8 @@ type Configuration struct {
// The location where the panel is running that this daemon should connect to
// to collect data and send events.
PanelLocation string `json:"remote" yaml:"remote"`
PanelLocation string `json:"remote" yaml:"remote"`
RemoteQuery RemoteQueryConfiguration `json:"remote_query" yaml:"remote_query"`
// AllowedMounts is a list of allowed host-system mount points.
// This is required to have the "Server Mounts" feature work properly.
@ -101,6 +102,27 @@ type ApiConfiguration struct {
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
}
// Defines the configuration settings for remote requests from Wings to the Panel.
type RemoteQueryConfiguration struct {
// The amount of time in seconds that Wings should allow for a request to the Panel API
// to complete. If this time passes the request will be marked as failed. If your requests
// are taking longer than 30 seconds to complete it is likely a performance issue that
// should be resolved on the Panel, and not something that should be resolved by upping this
// number.
Timeout uint `default:"30" yaml:"timeout"`
// The number of servers to load in a single request to the Panel API when booting the
// Wings instance. A single request is initially made to the Panel to get this number
// of servers, and then the pagination status is checked and additional requests are
// fired off in parallel to request the remaining pages.
//
// It is not recommended to change this from the default as you will likely encounter
// memory limits on your Panel instance. In the grand scheme of things 4 requests for
// 50 servers is likely just as quick as two for 100 or one for 400, and will certainly
// be less likely to cause performance issues on the Panel.
BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"`
}
// Reads the configuration from the provided file and returns the configuration
// object that can then be used.
func ReadConfiguration(path string) (*Configuration, error) {

View File

@ -32,7 +32,7 @@ func LoadDirectory() error {
}
log.Info("fetching list of servers from API")
configs, err := api.New().GetAllServerConfigurations()
configs, err := api.New().GetServers()
if err != nil {
if !api.IsRequestError(err) {
return errors.WithStack(err)