Use a paginated loop to fetch servers from the Panel rather than a single massive request
This commit is contained in:
parent
334b3e8d10
commit
41a67933eb
12
api/api.go
12
api/api.go
|
@ -37,9 +37,19 @@ type Response struct {
|
||||||
*http.Response
|
*http.Response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A pagination struct matching the expected pagination response from the Panel API.
|
||||||
|
type Pagination struct {
|
||||||
|
CurrentPage uint `json:"current_page"`
|
||||||
|
From uint `json:"from"`
|
||||||
|
LastPage uint `json:"last_page"`
|
||||||
|
PerPage uint `json:"per_page"`
|
||||||
|
To uint `json:"to"`
|
||||||
|
Total uint `json:"total"`
|
||||||
|
}
|
||||||
|
|
||||||
// Builds the base request instance that can be used with the HTTP client.
|
// Builds the base request instance that can be used with the HTTP client.
|
||||||
func (r *Request) Client() *http.Client {
|
func (r *Request) Client() *http.Client {
|
||||||
return &http.Client{Timeout: time.Second * 30}
|
return &http.Client{Timeout: time.Second * time.Duration(config.Get().RemoteQuery.Timeout)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the given endpoint formatted as a URL to the Panel API.
|
// Returns the given endpoint formatted as a URL to the Panel API.
|
||||||
|
|
|
@ -1,9 +1,14 @@
|
||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/apex/log"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
"github.com/pterodactyl/wings/config"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -33,9 +38,17 @@ type InstallationScript struct {
|
||||||
Script string `json:"script"`
|
Script string `json:"script"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAllServerConfigurations fetches configurations for all servers assigned to this node.
|
type allServerResponse struct {
|
||||||
func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, error) {
|
Data []json.RawMessage `json:"data"`
|
||||||
resp, err := r.Get("/servers", nil)
|
Meta Pagination `json:"meta"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetches all of the server configurations from the Panel API. This will initially load the
|
||||||
|
// first 50 servers, and then check the pagination response to determine if more pages should
|
||||||
|
// be loaded. If so, those requests are spun-up in additional routines and the final resulting
|
||||||
|
// slice of all servers will be returned.
|
||||||
|
func (r *Request) GetServers() ([]json.RawMessage, error) {
|
||||||
|
resp, err := r.Get("/servers", D{"per_page": config.Get().RemoteQuery.BootServersPerPage})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.WithStack(err)
|
return nil, errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
@ -45,12 +58,63 @@ func (r *Request) GetAllServerConfigurations() (map[string]json.RawMessage, erro
|
||||||
return nil, resp.Error()
|
return nil, resp.Error()
|
||||||
}
|
}
|
||||||
|
|
||||||
var res map[string]json.RawMessage
|
var res allServerResponse
|
||||||
if err := resp.Bind(&res); err != nil {
|
if err := resp.Bind(&res); err != nil {
|
||||||
return nil, errors.WithStack(err)
|
return nil, errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return res, nil
|
var mu sync.Mutex
|
||||||
|
ret := res.Data
|
||||||
|
|
||||||
|
// Check for pagination, and if it exists we'll need to then make a request to the API
|
||||||
|
// for each page that would exist and get all of the resulting servers.
|
||||||
|
if res.Meta.LastPage > 1 {
|
||||||
|
pp := res.Meta.PerPage
|
||||||
|
log.WithField("per_page", pp).
|
||||||
|
WithField("total_pages", res.Meta.LastPage).
|
||||||
|
Debug("detected multiple pages of server configurations, fetching remaining...")
|
||||||
|
|
||||||
|
g, ctx := errgroup.WithContext(context.Background())
|
||||||
|
for i := res.Meta.CurrentPage; i <= res.Meta.LastPage; i++ {
|
||||||
|
page := i
|
||||||
|
|
||||||
|
g.Go(func() error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
{
|
||||||
|
resp, err := r.Get("/servers", D{"page": page, "per_page": pp})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.Error() != nil {
|
||||||
|
return resp.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
var servers allServerResponse
|
||||||
|
if err := resp.Bind(&servers); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
ret = append(ret, servers.Data...)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := g.Wait(); err != nil {
|
||||||
|
return nil, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetches the server configuration and returns the struct for it.
|
// Fetches the server configuration and returns the struct for it.
|
||||||
|
@ -91,7 +155,6 @@ func (r *Request) GetInstallationScript(uuid string) (InstallationScript, error)
|
||||||
return is, errors.WithStack(err)
|
return is, errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return is, nil
|
return is, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -60,6 +60,7 @@ type Configuration struct {
|
||||||
// The location where the panel is running that this daemon should connect to
|
// The location where the panel is running that this daemon should connect to
|
||||||
// to collect data and send events.
|
// to collect data and send events.
|
||||||
PanelLocation string `json:"remote" yaml:"remote"`
|
PanelLocation string `json:"remote" yaml:"remote"`
|
||||||
|
RemoteQuery RemoteQueryConfiguration `json:"remote_query" yaml:"remote_query"`
|
||||||
|
|
||||||
// AllowedMounts is a list of allowed host-system mount points.
|
// AllowedMounts is a list of allowed host-system mount points.
|
||||||
// This is required to have the "Server Mounts" feature work properly.
|
// This is required to have the "Server Mounts" feature work properly.
|
||||||
|
@ -101,6 +102,27 @@ type ApiConfiguration struct {
|
||||||
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
|
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Defines the configuration settings for remote requests from Wings to the Panel.
|
||||||
|
type RemoteQueryConfiguration struct {
|
||||||
|
// The amount of time in seconds that Wings should allow for a request to the Panel API
|
||||||
|
// to complete. If this time passes the request will be marked as failed. If your requests
|
||||||
|
// are taking longer than 30 seconds to complete it is likely a performance issue that
|
||||||
|
// should be resolved on the Panel, and not something that should be resolved by upping this
|
||||||
|
// number.
|
||||||
|
Timeout uint `default:"30" yaml:"timeout"`
|
||||||
|
|
||||||
|
// The number of servers to load in a single request to the Panel API when booting the
|
||||||
|
// Wings instance. A single request is initially made to the Panel to get this number
|
||||||
|
// of servers, and then the pagination status is checked and additional requests are
|
||||||
|
// fired off in parallel to request the remaining pages.
|
||||||
|
//
|
||||||
|
// It is not recommended to change this from the default as you will likely encounter
|
||||||
|
// memory limits on your Panel instance. In the grand scheme of things 4 requests for
|
||||||
|
// 50 servers is likely just as quick as two for 100 or one for 400, and will certainly
|
||||||
|
// be less likely to cause performance issues on the Panel.
|
||||||
|
BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"`
|
||||||
|
}
|
||||||
|
|
||||||
// Reads the configuration from the provided file and returns the configuration
|
// Reads the configuration from the provided file and returns the configuration
|
||||||
// object that can then be used.
|
// object that can then be used.
|
||||||
func ReadConfiguration(path string) (*Configuration, error) {
|
func ReadConfiguration(path string) (*Configuration, error) {
|
||||||
|
|
|
@ -32,7 +32,7 @@ func LoadDirectory() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("fetching list of servers from API")
|
log.Info("fetching list of servers from API")
|
||||||
configs, err := api.New().GetAllServerConfigurations()
|
configs, err := api.New().GetServers()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !api.IsRequestError(err) {
|
if !api.IsRequestError(err) {
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user