metrics: initial commit
This commit is contained in:
parent
08a7ccd175
commit
dafbbab2ed
11
cmd/root.go
11
cmd/root.go
|
@ -4,6 +4,7 @@ import (
|
||||||
"crypto/tls"
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/pterodactyl/wings/metrics"
|
||||||
log2 "log"
|
log2 "log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
@ -137,6 +138,9 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
|
||||||
"gid": config.Get().System.User.Gid,
|
"gid": config.Get().System.User.Gid,
|
||||||
}).Info("configured system user successfully")
|
}).Info("configured system user successfully")
|
||||||
|
|
||||||
|
done := make(chan bool)
|
||||||
|
go metrics.Initialize(done)
|
||||||
|
|
||||||
pclient := remote.New(
|
pclient := remote.New(
|
||||||
config.Get().PanelLocation,
|
config.Get().PanelLocation,
|
||||||
remote.WithCredentials(config.Get().AuthenticationTokenId, config.Get().AuthenticationToken),
|
remote.WithCredentials(config.Get().AuthenticationTokenId, config.Get().AuthenticationToken),
|
||||||
|
@ -199,6 +203,12 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if states[s.Id()] == environment.ProcessRunningState {
|
||||||
|
metrics.ServerStatus.WithLabelValues(s.Id()).Set(1)
|
||||||
|
} else {
|
||||||
|
metrics.ServerStatus.WithLabelValues(s.Id()).Set(0)
|
||||||
|
}
|
||||||
|
|
||||||
pool.Submit(func() {
|
pool.Submit(func() {
|
||||||
s.Log().Info("configuring server environment and restoring to previous state")
|
s.Log().Info("configuring server environment and restoring to previous state")
|
||||||
var st string
|
var st string
|
||||||
|
@ -346,6 +356,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
|
||||||
if err := s.ListenAndServe(); err != nil {
|
if err := s.ListenAndServe(); err != nil {
|
||||||
log.WithField("error", err).Fatal("failed to configure HTTP server")
|
log.WithField("error", err).Fatal("failed to configure HTTP server")
|
||||||
}
|
}
|
||||||
|
<-done
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reads the configuration from the disk and then sets up the global singleton
|
// Reads the configuration from the disk and then sets up the global singleton
|
||||||
|
|
|
@ -91,6 +91,12 @@ type ApiConfiguration struct {
|
||||||
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
|
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MetricsConfiguration .
|
||||||
|
type MetricsConfiguration struct {
|
||||||
|
// Bind .
|
||||||
|
Bind string `default:":9000" yaml:"bind"`
|
||||||
|
}
|
||||||
|
|
||||||
// RemoteQueryConfiguration defines the configuration settings for remote requests
|
// RemoteQueryConfiguration defines the configuration settings for remote requests
|
||||||
// from Wings to the Panel.
|
// from Wings to the Panel.
|
||||||
type RemoteQueryConfiguration struct {
|
type RemoteQueryConfiguration struct {
|
||||||
|
@ -263,6 +269,7 @@ type Configuration struct {
|
||||||
Api ApiConfiguration `json:"api" yaml:"api"`
|
Api ApiConfiguration `json:"api" yaml:"api"`
|
||||||
System SystemConfiguration `json:"system" yaml:"system"`
|
System SystemConfiguration `json:"system" yaml:"system"`
|
||||||
Docker DockerConfiguration `json:"docker" yaml:"docker"`
|
Docker DockerConfiguration `json:"docker" yaml:"docker"`
|
||||||
|
Metrics MetricsConfiguration `json:"metrics" yaml:"metrics"`
|
||||||
|
|
||||||
// Defines internal throttling configurations for server processes to prevent
|
// Defines internal throttling configurations for server processes to prevent
|
||||||
// someone from running an endless loop that spams data to logs.
|
// someone from running an endless loop that spams data to logs.
|
||||||
|
|
|
@ -3,6 +3,7 @@ package docker
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/pterodactyl/wings/metrics"
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
@ -212,5 +213,15 @@ func (e *Environment) SetState(state string) {
|
||||||
// If the state changed make sure we update the internal tracking to note that.
|
// If the state changed make sure we update the internal tracking to note that.
|
||||||
e.st.Store(state)
|
e.st.Store(state)
|
||||||
e.Events().Publish(environment.StateChangeEvent, state)
|
e.Events().Publish(environment.StateChangeEvent, state)
|
||||||
|
|
||||||
|
if state == environment.ProcessRunningState || state == environment.ProcessOfflineState {
|
||||||
|
val := 0
|
||||||
|
if state == environment.ProcessRunningState {
|
||||||
|
val = 1
|
||||||
|
} else {
|
||||||
|
metrics.ResetServer(e.Id)
|
||||||
|
}
|
||||||
|
metrics.ServerStatus.WithLabelValues(e.Id).Set(float64(val))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/pterodactyl/wings/environment"
|
"github.com/pterodactyl/wings/environment"
|
||||||
|
"github.com/pterodactyl/wings/metrics"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
)
|
)
|
||||||
|
@ -60,6 +61,11 @@ func (e *Environment) pollResources(ctx context.Context) error {
|
||||||
st.Network.TxBytes += nw.TxBytes
|
st.Network.TxBytes += nw.TxBytes
|
||||||
}
|
}
|
||||||
|
|
||||||
|
metrics.ServerCPU.WithLabelValues(e.Id).Set(st.CpuAbsolute)
|
||||||
|
metrics.ServerMemory.WithLabelValues(e.Id).Set(float64(st.Memory))
|
||||||
|
metrics.ServerNetworkRx.WithLabelValues(e.Id).Set(float64(st.Network.RxBytes))
|
||||||
|
metrics.ServerNetworkTx.WithLabelValues(e.Id).Set(float64(st.Network.TxBytes))
|
||||||
|
|
||||||
if b, err := json.Marshal(st); err != nil {
|
if b, err := json.Marshal(st); err != nil {
|
||||||
e.log().WithField("error", err).Warn("error while marshaling stats object for environment")
|
e.log().WithField("error", err).Warn("error while marshaling stats object for environment")
|
||||||
} else {
|
} else {
|
||||||
|
|
107
metrics/metrics.go
Normal file
107
metrics/metrics.go
Normal file
|
@ -0,0 +1,107 @@
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/apex/log"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
"github.com/pterodactyl/wings/config"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Metrics struct {
|
||||||
|
handler http.Handler
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
namespace = "pterodactyl"
|
||||||
|
subsystem = "wings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
bootTimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "boot_time_seconds",
|
||||||
|
Help: "Boot time of this instance since epoch (1970)",
|
||||||
|
})
|
||||||
|
timeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "time_seconds",
|
||||||
|
Help: "System time in seconds since epoch (1970)",
|
||||||
|
})
|
||||||
|
|
||||||
|
ServerStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "server_status",
|
||||||
|
}, []string{"server_id"})
|
||||||
|
ServerCPU = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "server_cpu",
|
||||||
|
}, []string{"server_id"})
|
||||||
|
ServerMemory = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "server_memory",
|
||||||
|
}, []string{"server_id"})
|
||||||
|
ServerNetworkRx = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "server_network_rx",
|
||||||
|
}, []string{"server_id"})
|
||||||
|
ServerNetworkTx = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "server_network_tx",
|
||||||
|
}, []string{"server_id"})
|
||||||
|
|
||||||
|
HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Namespace: namespace,
|
||||||
|
Subsystem: subsystem,
|
||||||
|
Name: "http_requests_total",
|
||||||
|
}, []string{"method", "route_path", "raw_path", "raw_query", "code"})
|
||||||
|
)
|
||||||
|
|
||||||
|
func Initialize(done chan bool) {
|
||||||
|
bootTimeSeconds.Set(float64(time.Now().UnixNano()) / 1e9)
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
go func() {
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
// Received a "signal" on the done channel.
|
||||||
|
log.Debug("metrics: done")
|
||||||
|
return
|
||||||
|
case t := <-ticker.C:
|
||||||
|
// Update the current time.
|
||||||
|
timeSeconds.Set(float64(t.UnixNano()) / 1e9)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := http.ListenAndServe(config.Get().Metrics.Bind, promhttp.Handler()); err != nil && err != http.ErrServerClosed {
|
||||||
|
log.WithField("error", err).Error("failed to start metrics server")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteServer will remove any existing labels from being scraped by Prometheus.
|
||||||
|
// Any previously scraped data will still be persisted by Prometheus.
|
||||||
|
func DeleteServer(sID string) {
|
||||||
|
ServerStatus.DeleteLabelValues(sID)
|
||||||
|
ServerCPU.DeleteLabelValues(sID)
|
||||||
|
ServerMemory.DeleteLabelValues(sID)
|
||||||
|
ServerNetworkRx.DeleteLabelValues(sID)
|
||||||
|
ServerNetworkTx.DeleteLabelValues(sID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetServer will reset a server's metrics to their default values except the status.
|
||||||
|
func ResetServer(sID string) {
|
||||||
|
ServerCPU.WithLabelValues(sID).Set(0)
|
||||||
|
ServerMemory.WithLabelValues(sID).Set(0)
|
||||||
|
ServerNetworkRx.WithLabelValues(sID).Set(0)
|
||||||
|
ServerNetworkTx.WithLabelValues(sID).Set(0)
|
||||||
|
}
|
|
@ -3,9 +3,11 @@ package middleware
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/subtle"
|
"crypto/subtle"
|
||||||
|
"github.com/pterodactyl/wings/metrics"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"emperror.dev/errors"
|
"emperror.dev/errors"
|
||||||
|
@ -352,3 +354,19 @@ func ExtractManager(c *gin.Context) *server.Manager {
|
||||||
}
|
}
|
||||||
panic("middleware/middleware: cannot extract server manager: not present in context")
|
panic("middleware/middleware: cannot extract server manager: not present in context")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Metrics() gin.HandlerFunc {
|
||||||
|
return func(c *gin.Context) {
|
||||||
|
path := c.Request.URL.Path
|
||||||
|
rawQuery := c.Request.URL.RawQuery
|
||||||
|
|
||||||
|
c.Next()
|
||||||
|
|
||||||
|
// Skip over the server websocket endpoint.
|
||||||
|
if strings.HasSuffix(c.FullPath(), "/ws") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics.HTTPRequestsTotal.WithLabelValues(c.Request.Method, c.FullPath(), path, rawQuery, strconv.Itoa(c.Writer.Status())).Inc()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ func Configure(m *server.Manager, client remote.Client) *gin.Engine {
|
||||||
|
|
||||||
router := gin.New()
|
router := gin.New()
|
||||||
router.Use(gin.Recovery())
|
router.Use(gin.Recovery())
|
||||||
|
router.Use(middleware.Metrics())
|
||||||
router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders())
|
router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders())
|
||||||
router.Use(middleware.AttachServerManager(m), middleware.AttachApiClient(client))
|
router.Use(middleware.AttachServerManager(m), middleware.AttachApiClient(client))
|
||||||
// @todo log this into a different file so you can setup IP blocking for abusive requests and such.
|
// @todo log this into a different file so you can setup IP blocking for abusive requests and such.
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/pterodactyl/wings/metrics"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
|
@ -72,6 +73,9 @@ func (m *Manager) Add(s *Server) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
m.servers = append(m.servers, s)
|
m.servers = append(m.servers, s)
|
||||||
m.mu.Unlock()
|
m.mu.Unlock()
|
||||||
|
|
||||||
|
// Add the server to the metrics with a offline status.
|
||||||
|
metrics.ServerStatus.WithLabelValues(s.Id()).Set(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns a single server instance and a boolean value indicating if it was
|
// Get returns a single server instance and a boolean value indicating if it was
|
||||||
|
@ -117,6 +121,9 @@ func (m *Manager) Remove(filter func(match *Server) bool) {
|
||||||
for _, v := range m.servers {
|
for _, v := range m.servers {
|
||||||
if !filter(v) {
|
if !filter(v) {
|
||||||
r = append(r, v)
|
r = append(r, v)
|
||||||
|
} else {
|
||||||
|
// Delete the server from the metric.
|
||||||
|
metrics.DeleteServer(v.Id())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
m.servers = r
|
m.servers = r
|
||||||
|
|
Loading…
Reference in New Issue
Block a user