Use a paginated loop to fetch servers from the Panel rather than a single massive request
This commit is contained in:
parent
334b3e8d10
commit
41a67933eb
12
api/api.go
12
api/api.go
|
@ -37,9 +37,19 @@ type Response struct {
|
|||
*http.Response
|
||||
}
|
||||
|
||||
// A pagination struct matching the expected pagination response from the Panel API.
|
||||
type Pagination struct {
|
||||
CurrentPage uint `json:"current_page"`
|
||||
From uint `json:"from"`
|
||||
LastPage uint `json:"last_page"`
|
||||
PerPage uint `json:"per_page"`
|
||||
To uint `json:"to"`
|
||||
Total uint `json:"total"`
|
||||
}
|
||||
|
||||
// Builds the base request instance that can be used with the HTTP client.
|
||||
func (r *Request) Client() *http.Client {
|
||||
return &http.Client{Timeout: time.Second * 30}
|
||||
return &http.Client{Timeout: time.Second * time.Duration(config.Get().RemoteQuery.Timeout)}
|
||||
}
|
||||
|
||||
// Returns the given endpoint formatted as a URL to the Panel API.
|
||||
|
|
|
@ -1,9 +1,14 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/apex/log"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pterodactyl/wings/config"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -33,9 +38,17 @@ type InstallationScript struct {
|
|||
Script string `json:"script"`
|
||||
}
|
||||
|
||||
// GetAllServerConfigurations fetches configurations for all servers assigned to this node.
|
||||
func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, error) {
|
||||
resp, err := r.Get("/servers", nil)
|
||||
type allServerResponse struct {
|
||||
Data []json.RawMessage `json:"data"`
|
||||
Meta Pagination `json:"meta"`
|
||||
}
|
||||
|
||||
// Fetches all of the server configurations from the Panel API. This will initially load the
|
||||
// first 50 servers, and then check the pagination response to determine if more pages should
|
||||
// be loaded. If so, those requests are spun-up in additional routines and the final resulting
|
||||
// slice of all servers will be returned.
|
||||
func (r *Request) GetServers() ([]json.RawMessage, error) {
|
||||
resp, err := r.Get("/servers", D{"per_page": config.Get().RemoteQuery.BootServersPerPage})
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
@ -45,12 +58,63 @@ func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, erro
|
|||
return nil, resp.Error()
|
||||
}
|
||||
|
||||
var res map[string]json.RawMessage
|
||||
var res allServerResponse
|
||||
if err := resp.Bind(&res); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
var mu sync.Mutex
|
||||
ret := res.Data
|
||||
|
||||
// Check for pagination, and if it exists we'll need to then make a request to the API
|
||||
// for each page that would exist and get all of the resulting servers.
|
||||
if res.Meta.LastPage > 1 {
|
||||
pp := res.Meta.PerPage
|
||||
log.WithField("per_page", pp).
|
||||
WithField("total_pages", res.Meta.LastPage).
|
||||
Debug("detected multiple pages of server configurations, fetching remaining...")
|
||||
|
||||
g, ctx := errgroup.WithContext(context.Background())
|
||||
for i := res.Meta.CurrentPage; i <= res.Meta.LastPage; i++ {
|
||||
page := i
|
||||
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
{
|
||||
resp, err := r.Get("/servers", D{"page": page, "per_page": pp})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.Error() != nil {
|
||||
return resp.Error()
|
||||
}
|
||||
|
||||
var servers allServerResponse
|
||||
if err := resp.Bind(&servers); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
ret = append(ret, servers.Data...)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// Fetches the server configuration and returns the struct for it.
|
||||
|
@ -91,7 +155,6 @@ func (r *Request) GetInstallationScript(uuid string) (InstallationScript, error)
|
|||
return is, errors.WithStack(err)
|
||||
}
|
||||
|
||||
|
||||
return is, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -60,6 +60,7 @@ type Configuration struct {
|
|||
// The location where the panel is running that this daemon should connect to
|
||||
// to collect data and send events.
|
||||
PanelLocation string `json:"remote" yaml:"remote"`
|
||||
RemoteQuery RemoteQueryConfiguration `json:"remote_query" yaml:"remote_query"`
|
||||
|
||||
// AllowedMounts is a list of allowed host-system mount points.
|
||||
// This is required to have the "Server Mounts" feature work properly.
|
||||
|
@ -101,6 +102,27 @@ type ApiConfiguration struct {
|
|||
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
|
||||
}
|
||||
|
||||
// Defines the configuration settings for remote requests from Wings to the Panel.
|
||||
type RemoteQueryConfiguration struct {
|
||||
// The amount of time in seconds that Wings should allow for a request to the Panel API
|
||||
// to complete. If this time passes the request will be marked as failed. If your requests
|
||||
// are taking longer than 30 seconds to complete it is likely a performance issue that
|
||||
// should be resolved on the Panel, and not something that should be resolved by upping this
|
||||
// number.
|
||||
Timeout uint `default:"30" yaml:"timeout"`
|
||||
|
||||
// The number of servers to load in a single request to the Panel API when booting the
|
||||
// Wings instance. A single request is initially made to the Panel to get this number
|
||||
// of servers, and then the pagination status is checked and additional requests are
|
||||
// fired off in parallel to request the remaining pages.
|
||||
//
|
||||
// It is not recommended to change this from the default as you will likely encounter
|
||||
// memory limits on your Panel instance. In the grand scheme of things 4 requests for
|
||||
// 50 servers is likely just as quick as two for 100 or one for 400, and will certainly
|
||||
// be less likely to cause performance issues on the Panel.
|
||||
BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"`
|
||||
}
|
||||
|
||||
// Reads the configuration from the provided file and returns the configuration
|
||||
// object that can then be used.
|
||||
func ReadConfiguration(path string) (*Configuration, error) {
|
||||
|
|
|
@ -32,7 +32,7 @@ func LoadDirectory() error {
|
|||
}
|
||||
|
||||
log.Info("fetching list of servers from API")
|
||||
configs, err := api.New().GetAllServerConfigurations()
|
||||
configs, err := api.New().GetServers()
|
||||
if err != nil {
|
||||
if !api.IsRequestError(err) {
|
||||
return errors.WithStack(err)
|
||||
|
|
Loading…
Reference in New Issue
Block a user