MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correctly send stats & proc information back for servers

Browse Source
This commit is contained in:
Dane Everitt
2020-08-18 21:38:42 -07:00
parent 956e87eb93
commit 5b241fdf36
7 changed files with 225 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
server/listeners.go
View File

@@ -1,7 +1,9 @@
package server
import (
"encoding/json"
"github.com/apex/log"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/environment"
"github.com/pterodactyl/wings/events"
@@ -10,17 +12,19 @@ import (
// Adds all of the internal event listeners we want to use for a server.
func (s *Server) StartEventListeners() {
consoleChannel := make(chan events.Event)
stateChannel := make(chan events.Event)
console := make(chan events.Event)
state := make(chan events.Event)
stats := make(chan events.Event)
s.Environment.Events().Subscribe(environment.ConsoleOutputEvent, consoleChannel)
s.Environment.Events().Subscribe(environment.StateChangeEvent, stateChannel)
s.Environment.Events().Subscribe(environment.ConsoleOutputEvent, console)
s.Environment.Events().Subscribe(environment.StateChangeEvent, state)
s.Environment.Events().Subscribe(environment.ResourceEvent, stats)
// TODO: this is leaky I imagine since the routines aren't destroyed when the server is?
go func() {
for {
select {
case data := <-consoleChannel:
case data := <-console:
// Immediately emit this event back over the server event stream since it is
// being called from the environment event stream and things probably aren't
// listening to that event.
@@ -28,9 +32,28 @@ func (s *Server) StartEventListeners() {
// Also pass the data along to the console output channel.
s.onConsoleOutput(data.Data)
case data := <-stateChannel:
case data := <-state:
s.SetState(data.Data)
case data := <-stats:
st := new(environment.Stats)
if err := json.Unmarshal([]byte(data.Data), st); err != nil {
s.Log().WithField("error", errors.WithStack(err)).Warn("failed to unmarshal server environment stats")
continue
}
// Update the server resource tracking object with the resources we got here.
s.resources.mu.Lock()
s.resources.Stats = *st
s.resources.mu.Unlock()
// TODO: we'll need to handle this better since calling it in rapid succession will
// cause it to block until the first call is done calculating disk usage, which will
// case stat events to pile up for the server.
s.Filesystem.HasSpaceAvailable()
// Emit the event to the websocket.
b, _ := json.Marshal(s.Proc())
s.Events().Publish(StatsEvent, string(b))
}
}
}()

104
server/resources.go
View File

@@ -1,8 +1,7 @@
package server
import (
"github.com/docker/docker/api/types"
"math"
"github.com/pterodactyl/wings/environment"
"sync"
"sync/atomic"
)
@@ -13,33 +12,15 @@ import (
type ResourceUsage struct {
mu sync.RWMutex
// Embed the current environment stats into this server specific resource usage struct.
environment.Stats
// The current server status.
State string `json:"state" default:"offline"`
// The total amount of memory, in bytes, that this server instance is consuming. This is
// calculated slightly differently than just using the raw Memory field that the stats
// return from the container, so please check the code setting this value for how that
// is calculated.
Memory uint64 `json:"memory_bytes"`
// The total amount of memory this container or resource can use. Inside Docker this is
// going to be higher than you'd expect because we're automatically allocating overhead
// abilities for the container, so its not going to be a perfect match.
MemoryLimit uint64 `json:"memory_limit_bytes"`
// The absolute CPU usage is the amount of CPU used in relation to the entire system and
// does not take into account any limits on the server process itself.
CpuAbsolute float64 `json:"cpu_absolute"`
// The current disk space being used by the server. This is cached to prevent slow lookup
// issues on frequent refreshes.
Disk int64 `json:"disk_bytes"`
// Current network transmit in & out for a container.
Network struct {
RxBytes uint64 `json:"rx_bytes"`
TxBytes uint64 `json:"tx_bytes"`
} `json:"network"`
}
// Returns the resource usage stats for the server instance. If the server is not running, only the
@@ -69,81 +50,6 @@ func (ru *ResourceUsage) setInternalState(state string) {
ru.mu.Unlock()
}
// Resets the usages values to zero, used when a server is stopped to ensure we don't hold
// onto any values incorrectly.
func (ru *ResourceUsage) Empty() {
ru.mu.Lock()
defer ru.mu.Unlock()
ru.Memory = 0
ru.CpuAbsolute = 0
ru.Network.TxBytes = 0
ru.Network.RxBytes = 0
}
func (ru *ResourceUsage) SetDisk(i int64) {
ru.mu.Lock()
defer ru.mu.Unlock()
ru.Disk = i
}
func (ru *ResourceUsage) UpdateFromDocker(v *types.StatsJSON) {
ru.mu.Lock()
defer ru.mu.Unlock()
ru.CpuAbsolute = ru.calculateDockerAbsoluteCpu(&v.PreCPUStats, &v.CPUStats)
ru.Memory = ru.calculateDockerMemory(v.MemoryStats)
ru.MemoryLimit = v.MemoryStats.Limit
}
func (ru *ResourceUsage) UpdateNetworkBytes(nw *types.NetworkStats) {
atomic.AddUint64(&ru.Network.RxBytes, nw.RxBytes)
atomic.AddUint64(&ru.Network.TxBytes, nw.TxBytes)
}
// The "docker stats" CLI call does not return the same value as the types.MemoryStats.Usage
// value which can be rather confusing to people trying to compare panel usage to
// their stats output.
//
// This math is straight up lifted from their CLI repository in order to show the same
// values to avoid people bothering me about it. It should also reflect a slightly more
// correct memory value anyways.
//
// @see https://github.com/docker/cli/blob/96e1d1d6/cli/command/container/stats_helpers.go#L227-L249
func (ru *ResourceUsage) calculateDockerMemory(stats types.MemoryStats) uint64 {
if v, ok := stats.Stats["total_inactive_file"]; ok && v < stats.Usage {
return stats.Usage - v
}
if v := stats.Stats["inactive_file"]; v < stats.Usage {
return stats.Usage - v
}
return stats.Usage
}
// Calculates the absolute CPU usage used by the server process on the system, not constrained
// by the defined CPU limits on the container.
//
// @see https://github.com/docker/cli/blob/aa097cf1aa19099da70930460250797c8920b709/cli/command/container/stats_helpers.go#L166
func (ru *ResourceUsage) calculateDockerAbsoluteCpu(pStats *types.CPUStats, stats *types.CPUStats) float64 {
// Calculate the change in CPU usage between the current and previous reading.
cpuDelta := float64(stats.CPUUsage.TotalUsage) - float64(pStats.CPUUsage.TotalUsage)
// Calculate the change for the entire system's CPU usage between current and previous reading.
systemDelta := float64(stats.SystemUsage) - float64(pStats.SystemUsage)
// Calculate the total number of CPU cores being used.
cpus := float64(stats.OnlineCPUs)
if cpus == 0.0 {
cpus = float64(len(stats.CPUUsage.PercpuUsage))
}
percent := 0.0
if systemDelta > 0.0 && cpuDelta > 0.0 {
percent = (cpuDelta / systemDelta) * cpus * 100.0
}
return math.Round(percent*1000) / 1000
atomic.SwapInt64(&ru.Disk, i)
}

10
server/state.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/system"
"io"
"io/ioutil"
"os"
@@ -99,6 +100,15 @@ func (s *Server) SetState(state string) error {
}
}()
// Reset the resource usage to 0 when the process fully stops so that all of the UI
// views in the Panel correctly display 0.
if state == system.ProcessOfflineState {
s.resources.Empty()
b, _ := json.Marshal(s.Proc())
s.Events().Publish(StatsEvent, string(b))
}
// If server was in an online state, and is now in an offline state we should handle
// that as a crash event. In that scenario, check the last crash time, and the crash
// counter.