metrics: initial commit

This commit is contained in:
Matthew Penner 2021-03-09 08:45:54 -07:00
parent 08a7ccd175
commit dafbbab2ed
8 changed files with 171 additions and 3 deletions

View File

@ -4,6 +4,7 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/pterodactyl/wings/metrics"
log2 "log" log2 "log"
"net/http" "net/http"
"os" "os"
@ -137,6 +138,9 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
"gid": config.Get().System.User.Gid, "gid": config.Get().System.User.Gid,
}).Info("configured system user successfully") }).Info("configured system user successfully")
done := make(chan bool)
go metrics.Initialize(done)
pclient := remote.New( pclient := remote.New(
config.Get().PanelLocation, config.Get().PanelLocation,
remote.WithCredentials(config.Get().AuthenticationTokenId, config.Get().AuthenticationToken), remote.WithCredentials(config.Get().AuthenticationTokenId, config.Get().AuthenticationToken),
@ -199,6 +203,12 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
continue continue
} }
if states[s.Id()] == environment.ProcessRunningState {
metrics.ServerStatus.WithLabelValues(s.Id()).Set(1)
} else {
metrics.ServerStatus.WithLabelValues(s.Id()).Set(0)
}
pool.Submit(func() { pool.Submit(func() {
s.Log().Info("configuring server environment and restoring to previous state") s.Log().Info("configuring server environment and restoring to previous state")
var st string var st string
@ -346,6 +356,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
if err := s.ListenAndServe(); err != nil { if err := s.ListenAndServe(); err != nil {
log.WithField("error", err).Fatal("failed to configure HTTP server") log.WithField("error", err).Fatal("failed to configure HTTP server")
} }
<-done
} }
// Reads the configuration from the disk and then sets up the global singleton // Reads the configuration from the disk and then sets up the global singleton

View File

@ -91,6 +91,12 @@ type ApiConfiguration struct {
UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"` UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"`
} }
// MetricsConfiguration .
type MetricsConfiguration struct {
// Bind .
Bind string `default:":9000" yaml:"bind"`
}
// RemoteQueryConfiguration defines the configuration settings for remote requests // RemoteQueryConfiguration defines the configuration settings for remote requests
// from Wings to the Panel. // from Wings to the Panel.
type RemoteQueryConfiguration struct { type RemoteQueryConfiguration struct {
@ -263,6 +269,7 @@ type Configuration struct {
Api ApiConfiguration `json:"api" yaml:"api"` Api ApiConfiguration `json:"api" yaml:"api"`
System SystemConfiguration `json:"system" yaml:"system"` System SystemConfiguration `json:"system" yaml:"system"`
Docker DockerConfiguration `json:"docker" yaml:"docker"` Docker DockerConfiguration `json:"docker" yaml:"docker"`
Metrics MetricsConfiguration `json:"metrics" yaml:"metrics"`
// Defines internal throttling configurations for server processes to prevent // Defines internal throttling configurations for server processes to prevent
// someone from running an endless loop that spams data to logs. // someone from running an endless loop that spams data to logs.

View File

@ -3,6 +3,7 @@ package docker
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/pterodactyl/wings/metrics"
"io" "io"
"sync" "sync"
@ -212,5 +213,15 @@ func (e *Environment) SetState(state string) {
// If the state changed make sure we update the internal tracking to note that. // If the state changed make sure we update the internal tracking to note that.
e.st.Store(state) e.st.Store(state)
e.Events().Publish(environment.StateChangeEvent, state) e.Events().Publish(environment.StateChangeEvent, state)
if state == environment.ProcessRunningState || state == environment.ProcessOfflineState {
val := 0
if state == environment.ProcessRunningState {
val = 1
} else {
metrics.ResetServer(e.Id)
}
metrics.ServerStatus.WithLabelValues(e.Id).Set(float64(val))
}
} }
} }

View File

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
"github.com/pterodactyl/wings/metrics"
"io" "io"
"math" "math"
) )
@ -60,6 +61,11 @@ func (e *Environment) pollResources(ctx context.Context) error {
st.Network.TxBytes += nw.TxBytes st.Network.TxBytes += nw.TxBytes
} }
metrics.ServerCPU.WithLabelValues(e.Id).Set(st.CpuAbsolute)
metrics.ServerMemory.WithLabelValues(e.Id).Set(float64(st.Memory))
metrics.ServerNetworkRx.WithLabelValues(e.Id).Set(float64(st.Network.RxBytes))
metrics.ServerNetworkTx.WithLabelValues(e.Id).Set(float64(st.Network.TxBytes))
if b, err := json.Marshal(st); err != nil { if b, err := json.Marshal(st); err != nil {
e.log().WithField("error", err).Warn("error while marshaling stats object for environment") e.log().WithField("error", err).Warn("error while marshaling stats object for environment")
} else { } else {

107
metrics/metrics.go Normal file
View File

@ -0,0 +1,107 @@
package metrics
import (
"github.com/apex/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/pterodactyl/wings/config"
"net/http"
"time"
)
type Metrics struct {
handler http.Handler
}
const (
namespace = "pterodactyl"
subsystem = "wings"
)
var (
bootTimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "boot_time_seconds",
Help: "Boot time of this instance since epoch (1970)",
})
timeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "time_seconds",
Help: "System time in seconds since epoch (1970)",
})
ServerStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "server_status",
}, []string{"server_id"})
ServerCPU = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "server_cpu",
}, []string{"server_id"})
ServerMemory = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "server_memory",
}, []string{"server_id"})
ServerNetworkRx = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "server_network_rx",
}, []string{"server_id"})
ServerNetworkTx = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "server_network_tx",
}, []string{"server_id"})
HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "http_requests_total",
}, []string{"method", "route_path", "raw_path", "raw_query", "code"})
)
func Initialize(done chan bool) {
bootTimeSeconds.Set(float64(time.Now().UnixNano()) / 1e9)
ticker := time.NewTicker(time.Second)
go func() {
defer ticker.Stop()
for {
select {
case <-done:
// Received a "signal" on the done channel.
log.Debug("metrics: done")
return
case t := <-ticker.C:
// Update the current time.
timeSeconds.Set(float64(t.UnixNano()) / 1e9)
}
}
}()
if err := http.ListenAndServe(config.Get().Metrics.Bind, promhttp.Handler()); err != nil && err != http.ErrServerClosed {
log.WithField("error", err).Error("failed to start metrics server")
}
}
// DeleteServer will remove any existing labels from being scraped by Prometheus.
// Any previously scraped data will still be persisted by Prometheus.
func DeleteServer(sID string) {
ServerStatus.DeleteLabelValues(sID)
ServerCPU.DeleteLabelValues(sID)
ServerMemory.DeleteLabelValues(sID)
ServerNetworkRx.DeleteLabelValues(sID)
ServerNetworkTx.DeleteLabelValues(sID)
}
// ResetServer will reset a server's metrics to their default values except the status.
func ResetServer(sID string) {
ServerCPU.WithLabelValues(sID).Set(0)
ServerMemory.WithLabelValues(sID).Set(0)
ServerNetworkRx.WithLabelValues(sID).Set(0)
ServerNetworkTx.WithLabelValues(sID).Set(0)
}

View File

@ -3,9 +3,11 @@ package middleware
import ( import (
"context" "context"
"crypto/subtle" "crypto/subtle"
"github.com/pterodactyl/wings/metrics"
"io" "io"
"net/http" "net/http"
"os" "os"
"strconv"
"strings" "strings"
"emperror.dev/errors" "emperror.dev/errors"
@ -352,3 +354,19 @@ func ExtractManager(c *gin.Context) *server.Manager {
} }
panic("middleware/middleware: cannot extract server manager: not present in context") panic("middleware/middleware: cannot extract server manager: not present in context")
} }
func Metrics() gin.HandlerFunc {
return func(c *gin.Context) {
path := c.Request.URL.Path
rawQuery := c.Request.URL.RawQuery
c.Next()
// Skip over the server websocket endpoint.
if strings.HasSuffix(c.FullPath(), "/ws") {
return
}
metrics.HTTPRequestsTotal.WithLabelValues(c.Request.Method, c.FullPath(), path, rawQuery, strconv.Itoa(c.Writer.Status())).Inc()
}
}

View File

@ -14,6 +14,7 @@ func Configure(m *server.Manager, client remote.Client) *gin.Engine {
router := gin.New() router := gin.New()
router.Use(gin.Recovery()) router.Use(gin.Recovery())
router.Use(middleware.Metrics())
router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders()) router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders())
router.Use(middleware.AttachServerManager(m), middleware.AttachApiClient(client)) router.Use(middleware.AttachServerManager(m), middleware.AttachApiClient(client))
// @todo log this into a different file so you can setup IP blocking for abusive requests and such. // @todo log this into a different file so you can setup IP blocking for abusive requests and such.

View File

@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/pterodactyl/wings/metrics"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
@ -72,6 +73,9 @@ func (m *Manager) Add(s *Server) {
m.mu.Lock() m.mu.Lock()
m.servers = append(m.servers, s) m.servers = append(m.servers, s)
m.mu.Unlock() m.mu.Unlock()
// Add the server to the metrics with a offline status.
metrics.ServerStatus.WithLabelValues(s.Id()).Set(0)
} }
// Get returns a single server instance and a boolean value indicating if it was // Get returns a single server instance and a boolean value indicating if it was
@ -117,6 +121,9 @@ func (m *Manager) Remove(filter func(match *Server) bool) {
for _, v := range m.servers { for _, v := range m.servers {
if !filter(v) { if !filter(v) {
r = append(r, v) r = append(r, v)
} else {
// Delete the server from the metric.
metrics.DeleteServer(v.Id())
} }
} }
m.servers = r m.servers = r