Add support for streaming usage stats back to the calling process
This commit is contained in:
parent
a1987f3aef
commit
c8e6e29abc
|
@ -51,4 +51,12 @@ type Environment interface {
|
|||
// Reads the log file for the process from the end backwards until the provided
|
||||
// number of bytes is met.
|
||||
Readlog(int64) ([]string, error)
|
||||
|
||||
// Polls the given environment for resource usage of the server when the process
|
||||
// is running.
|
||||
EnableResourcePolling() error
|
||||
|
||||
// Disables the polling operation for resource usage and sets the required values
|
||||
// to 0 in the server resource usage struct.
|
||||
DisableResourcePolling() error
|
||||
}
|
||||
|
|
|
@ -43,6 +43,10 @@ type DockerEnvironment struct {
|
|||
// Controls the hijacked response stream which exists only when we're attached to
|
||||
// the running container instance.
|
||||
stream types.HijackedResponse
|
||||
|
||||
// Holds the stats stream used by the polling commands so that we can easily close
|
||||
// it out.
|
||||
stats io.ReadCloser
|
||||
}
|
||||
|
||||
// Creates a new base Docker environment. A server must still be attached to it.
|
||||
|
@ -209,6 +213,7 @@ func (d *DockerEnvironment) Attach() error {
|
|||
Server: d.Server,
|
||||
}
|
||||
|
||||
d.EnableResourcePolling()
|
||||
d.attached = true
|
||||
|
||||
go func() {
|
||||
|
@ -261,6 +266,69 @@ func (d *DockerEnvironment) FollowConsoleOutput() error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Enables resource polling on the docker instance. Except we aren't actually polling Docker for this
|
||||
// information, instead just sit there with an async process that lets Docker stream all of this data
|
||||
// to us automatically.
|
||||
func (d *DockerEnvironment) EnableResourcePolling() error {
|
||||
fmt.Println("called")
|
||||
if d.Server.State == ProcessOfflineState {
|
||||
fmt.Println("not running")
|
||||
return errors.New("cannot enable resource polling on a server that is not running")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
stats, err := d.Client.ContainerStats(ctx, d.Server.Uuid, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.stats = stats.Body
|
||||
|
||||
dec := json.NewDecoder(d.stats)
|
||||
go func(s *Server) {
|
||||
pCpu := 0.0
|
||||
pSystem := 0.0
|
||||
|
||||
for {
|
||||
var v *types.StatsJSON
|
||||
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
zap.S().Warnw("encountered error processing server stats; stopping collection", zap.Error(err))
|
||||
d.DisableResourcePolling()
|
||||
return
|
||||
}
|
||||
|
||||
// Disable collection if the server is in an offline state and this process is
|
||||
// still running.
|
||||
if s.State == ProcessOfflineState {
|
||||
d.DisableResourcePolling()
|
||||
return
|
||||
}
|
||||
|
||||
s.Resources.CpuAbsolute = s.Resources.CalculateAbsoluteCpu(pCpu, pSystem, &v.CPUStats)
|
||||
s.Resources.Memory = v.MemoryStats.Usage
|
||||
s.Resources.MemoryLimit = v.MemoryStats.Limit
|
||||
s.Resources.Disk = 0
|
||||
|
||||
for _, nw := range v.Networks {
|
||||
s.Resources.Network.RxBytes += nw.RxBytes
|
||||
s.Resources.Network.TxBytes += nw.TxBytes
|
||||
}
|
||||
}
|
||||
}(d.Server)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Closes the stats stream for a server process.
|
||||
func (d *DockerEnvironment) DisableResourcePolling() error {
|
||||
if d.stats == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return d.stats.Close()
|
||||
}
|
||||
|
||||
// Creates a new container for the server using all of the data that is currently
|
||||
// available for it. If the container already exists it will be returned.
|
||||
func (d *DockerEnvironment) Create() error {
|
||||
|
@ -529,4 +597,4 @@ func (d *DockerEnvironment) exposedPorts() nat.PortSet {
|
|||
}
|
||||
|
||||
return out
|
||||
}
|
||||
}
|
51
server/resources.go
Normal file
51
server/resources.go
Normal file
|
@ -0,0 +1,51 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"github.com/docker/docker/api/types"
|
||||
"math"
|
||||
)
|
||||
|
||||
// Defines the current resource usage for a given server instance. If a server is offline you
|
||||
// should obviously expect memory and CPU usage to be 0. However, disk will always be returned
|
||||
// since that is not dependent on the server being running to collect that data.
|
||||
type ResourceUsage struct {
|
||||
// The total amount of memory, in bytes, that this server instance is consuming.
|
||||
Memory uint64 `json:"memory_bytes"`
|
||||
// The total amount of memory this container or resource can use. Inside Docker this is
|
||||
// going to be higher than you'd expect because we're automatically allocating overhead
|
||||
// abilities for the container, so its not going to be a perfect match.
|
||||
MemoryLimit uint64 `json:"memory_limit_bytes"`
|
||||
// The absolute CPU usage is the amount of CPU used in relation to the entire system and
|
||||
// does not take into account any limits on the server process itself.
|
||||
CpuAbsolute float64 `json:"cpu_absolute"`
|
||||
// The current disk space being used by the server. This is cached to prevent slow lookup
|
||||
// issues on frequent refreshes.
|
||||
Disk uint64 `json:"disk_bytes"`
|
||||
// Current network transmit in & out for a container.
|
||||
Network struct {
|
||||
RxBytes uint64 `json:"rx_bytes"`
|
||||
TxBytes uint64 `json:"tx_bytes"`
|
||||
} `json:"network"`
|
||||
}
|
||||
|
||||
// Calculates the absolute CPU usage used by the server process on the system, not constrained
|
||||
// by the defined CPU limits on the container.
|
||||
//
|
||||
// @see https://github.com/docker/cli/blob/aa097cf1aa19099da70930460250797c8920b709/cli/command/container/stats_helpers.go#L166
|
||||
func (ru *ResourceUsage) CalculateAbsoluteCpu(previousCpu, previousSystem float64, stats *types.CPUStats) float64 {
|
||||
// Calculate the change in CPU usage between the current and previous reading.
|
||||
cpuDelta := float64(stats.CPUUsage.TotalUsage) - float64(previousCpu)
|
||||
|
||||
// Calculate the change for the entire system's CPU usage between current and previous reading.
|
||||
systemDelta := float64(stats.SystemUsage) - float64(previousSystem)
|
||||
|
||||
// Calculate the total number of CPU cores being used.
|
||||
cpus := float64(len(stats.CPUUsage.PercpuUsage))
|
||||
|
||||
percent := 0.0
|
||||
if systemDelta > 0.0 && cpuDelta > 0.0 {
|
||||
percent = (cpuDelta / systemDelta) * cpus * 100.0
|
||||
}
|
||||
|
||||
return math.Round(percent * 1000) / 1000
|
||||
}
|
|
@ -48,6 +48,8 @@ type Server struct {
|
|||
|
||||
Filesystem *Filesystem `json:"-"`
|
||||
|
||||
Resources *ResourceUsage `json:"resources"`
|
||||
|
||||
// Server cache used to store frequently requested information in memory and make
|
||||
// certain long operations return faster. For example, FS disk space usage.
|
||||
Cache *cache.Cache `json:"-"`
|
||||
|
@ -203,6 +205,7 @@ func FromConfiguration(data []byte, cfg *config.SystemConfiguration) (*Server, e
|
|||
Configuration: cfg,
|
||||
Server: s,
|
||||
}
|
||||
s.Resources = &ResourceUsage{}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
|
4
wings.go
4
wings.go
|
@ -72,10 +72,10 @@ func main() {
|
|||
zap.S().Errorw("error checking server environment status", zap.String("server", s.Uuid), zap.Error(err))
|
||||
} else if r {
|
||||
zap.S().Infow("detected server is running, re-attaching to process", zap.String("server", s.Uuid))
|
||||
s.SetState(server.ProcessRunningState)
|
||||
if err := s.Environment.Attach(); err != nil {
|
||||
zap.S().Errorw("error attaching to server environment", zap.String("server", s.Uuid), zap.Error(err))
|
||||
} else {
|
||||
s.SetState(server.ProcessRunningState)
|
||||
s.SetState(server.ProcessOfflineState)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user