Improve server state logical handling; allow setting state directly on the environment

This commit is contained in:
Dane Everitt 2020-11-06 21:53:00 -08:00
parent 3fce1b98d5
commit 944d381778
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
13 changed files with 73 additions and 105 deletions

View File

@ -229,7 +229,7 @@ func rootCmdRun(*cobra.Command, []string) {
// is that it was running, but we see that the container process is not currently running. // is that it was running, but we see that the container process is not currently running.
s.Log().Info("detected server is running, re-attaching to process...") s.Log().Info("detected server is running, re-attaching to process...")
s.SetState(environment.ProcessRunningState) s.Environment.SetState(environment.ProcessRunningState)
if err := s.Environment.Attach(); err != nil { if err := s.Environment.Attach(); err != nil {
s.Log().WithField("error", errors.WithStack(err)).Warn("failed to attach to running server environment") s.Log().WithField("error", errors.WithStack(err)).Warn("failed to attach to running server environment")
} }
@ -239,7 +239,7 @@ func rootCmdRun(*cobra.Command, []string) {
// Addresses potentially invalid data in the stored file that can cause Wings to lose // Addresses potentially invalid data in the stored file that can cause Wings to lose
// track of what the actual server state is. // track of what the actual server state is.
_ = s.SetState(environment.ProcessOfflineState) s.Environment.SetState(environment.ProcessOfflineState)
}) })
} }
@ -277,34 +277,20 @@ func rootCmdRun(*cobra.Command, []string) {
Handler: r, Handler: r,
TLSConfig: &tls.Config{ TLSConfig: &tls.Config{
NextProtos: []string{ NextProtos: []string{"h2", "http/1.1"},
"h2", // enable HTTP/2 // @see https://blog.cloudflare.com/exposing-go-on-the-internet
"http/1.1",
},
// https://blog.cloudflare.com/exposing-go-on-the-internet
CipherSuites: []uint16{ CipherSuites: []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
}, },
PreferServerCipherSuites: true, PreferServerCipherSuites: true,
MinVersion: tls.VersionTLS12, MinVersion: tls.VersionTLS12,
MaxVersion: tls.VersionTLS13, MaxVersion: tls.VersionTLS13,
CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256},
CurvePreferences: []tls.CurveID{
tls.X25519,
tls.CurveP256,
},
// END https://blog.cloudflare.com/exposing-go-on-the-internet
}, },
} }

View File

@ -60,7 +60,7 @@ func (e *Environment) Attach() error {
defer cancel() defer cancel()
defer e.stream.Close() defer e.stream.Close()
defer func() { defer func() {
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
e.SetStream(nil) e.SetStream(nil)
}() }()
@ -245,7 +245,7 @@ func (e *Environment) convertMounts() []mount.Mount {
// it will be forcibly stopped by Docker. // it will be forcibly stopped by Docker.
func (e *Environment) Destroy() error { func (e *Environment) Destroy() error {
// We set it to stopping than offline to prevent crash detection from being triggered. // We set it to stopping than offline to prevent crash detection from being triggered.
e.setState(environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
err := e.client.ContainerRemove(context.Background(), e.Id, types.ContainerRemoveOptions{ err := e.client.ContainerRemove(context.Background(), e.Id, types.ContainerRemoveOptions{
RemoveVolumes: true, RemoveVolumes: true,
@ -261,7 +261,7 @@ func (e *Environment) Destroy() error {
return nil return nil
} }
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
return err return err
} }

View File

@ -48,7 +48,7 @@ type Environment struct {
emitter *events.EventBus emitter *events.EventBus
// Tracks the environment state. // Tracks the environment state.
State system.AtomicString st system.AtomicString
} }
// Creates a new base Docker environment. The ID passed through will be the ID that is used to // Creates a new base Docker environment. The ID passed through will be the ID that is used to
@ -67,7 +67,7 @@ func New(id string, m *Metadata, c *environment.Configuration) (*Environment, er
client: cli, client: cli,
} }
e.State.Store(environment.ProcessOfflineState) e.st.Store(environment.ProcessOfflineState)
return e, nil return e, nil
} }

View File

@ -57,8 +57,8 @@ func (e *Environment) Start() error {
// If we don't set it to stopping first, you'll trigger crash detection which // If we don't set it to stopping first, you'll trigger crash detection which
// we don't want to do at this point since it'll just immediately try to do the // we don't want to do at this point since it'll just immediately try to do the
// exact same action that lead to it crashing in the first place... // exact same action that lead to it crashing in the first place...
e.setState(environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
} }
}() }()
@ -74,7 +74,7 @@ func (e *Environment) Start() error {
} else { } else {
// If the server is running update our internal state and continue on with the attach. // If the server is running update our internal state and continue on with the attach.
if c.State.Running { if c.State.Running {
e.setState(environment.ProcessRunningState) e.SetState(environment.ProcessRunningState)
return e.Attach() return e.Attach()
} }
@ -89,7 +89,7 @@ func (e *Environment) Start() error {
} }
} }
e.setState(environment.ProcessStartingState) e.SetState(environment.ProcessStartingState)
// Set this to true for now, we will set it to false once we reach the // Set this to true for now, we will set it to false once we reach the
// end of this chain. // end of this chain.
@ -136,8 +136,8 @@ func (e *Environment) Stop() error {
// If the process is already offline don't switch it back to stopping. Just leave it how // If the process is already offline don't switch it back to stopping. Just leave it how
// it is and continue through to the stop handling for the process. // it is and continue through to the stop handling for the process.
if e.State.Load() != environment.ProcessOfflineState { if e.st.Load() != environment.ProcessOfflineState {
e.setState(environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
} }
// Only attempt to send the stop command to the instance if we are actually attached to // Only attempt to send the stop command to the instance if we are actually attached to
@ -153,7 +153,7 @@ func (e *Environment) Stop() error {
// an error. // an error.
if client.IsErrNotFound(err) { if client.IsErrNotFound(err) {
e.SetStream(nil) e.SetStream(nil)
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
return nil return nil
} }
@ -217,16 +217,16 @@ func (e *Environment) Terminate(signal os.Signal) error {
// If the container is not running but we're not already in a stopped state go ahead // If the container is not running but we're not already in a stopped state go ahead
// and update things to indicate we should be completely stopped now. Set to stopping // and update things to indicate we should be completely stopped now. Set to stopping
// first so crash detection is not triggered. // first so crash detection is not triggered.
if e.State.Load() != environment.ProcessOfflineState { if e.st.Load() != environment.ProcessOfflineState {
e.setState(environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
} }
return nil return nil
} }
// We set it to stopping than offline to prevent crash detection from being triggered. // We set it to stopping than offline to prevent crash detection from being triggered.
e.setState(environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
sig := strings.TrimSuffix(strings.TrimPrefix(signal.String(), "signal "), "ed") sig := strings.TrimSuffix(strings.TrimPrefix(signal.String(), "signal "), "ed")
@ -234,7 +234,7 @@ func (e *Environment) Terminate(signal os.Signal) error {
return err return err
} }
e.setState(environment.ProcessOfflineState) e.SetState(environment.ProcessOfflineState)
return nil return nil
} }

View File

@ -6,25 +6,24 @@ import (
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
) )
func (e *Environment) State() string {
return e.st.Load()
}
// Sets the state of the environment. This emits an event that server's can hook into to // Sets the state of the environment. This emits an event that server's can hook into to
// take their own actions and track their own state based on the environment. // take their own actions and track their own state based on the environment.
func (e *Environment) setState(state string) error { func (e *Environment) SetState(state string) {
if state != environment.ProcessOfflineState && if state != environment.ProcessOfflineState &&
state != environment.ProcessStartingState && state != environment.ProcessStartingState &&
state != environment.ProcessRunningState && state != environment.ProcessRunningState &&
state != environment.ProcessStoppingState { state != environment.ProcessStoppingState {
return errors.New(fmt.Sprintf("invalid server state received: %s", state)) panic(errors.New(fmt.Sprintf("invalid server state received: %s", state)))
} }
// Get the current state of the environment before changing it.
prevState := e.State.Load()
// Emit the event to any listeners that are currently registered. // Emit the event to any listeners that are currently registered.
if prevState != state { if e.State() != state {
// If the state changed make sure we update the internal tracking to note that. // If the state changed make sure we update the internal tracking to note that.
e.State.Store(state) e.st.Store(state)
e.Events().Publish(environment.StateChangeEvent, state) e.Events().Publish(environment.StateChangeEvent, state)
} }
return nil
} }

View File

@ -20,7 +20,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
l.Debug("starting resource polling for container") l.Debug("starting resource polling for container")
defer l.Debug("stopped resource polling for container") defer l.Debug("stopped resource polling for container")
if e.State.Load() == environment.ProcessOfflineState { if e.st.Load() == environment.ProcessOfflineState {
return errors.New("cannot enable resource polling on a stopped server") return errors.New("cannot enable resource polling on a stopped server")
} }
@ -50,7 +50,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
} }
// Disable collection if the server is in an offline state and this process is still running. // Disable collection if the server is in an offline state and this process is still running.
if e.State.Load() == environment.ProcessOfflineState { if e.st.Load() == environment.ProcessOfflineState {
l.Debug("process in offline state while resource polling is still active; stopping poll") l.Debug("process in offline state while resource polling is still active; stopping poll")
return nil return nil
} }

View File

@ -37,7 +37,7 @@ func (e *Environment) SendCommand(c string) error {
// the server as entering the stopping state otherwise the process will stop and Wings will think // the server as entering the stopping state otherwise the process will stop and Wings will think
// it has crashed and attempt to restart it. // it has crashed and attempt to restart it.
if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value { if e.meta.Stop.Type == "command" && c == e.meta.Stop.Value {
e.Events().Publish(environment.StateChangeEvent, environment.ProcessStoppingState) e.SetState(environment.ProcessStoppingState)
} }
_, err := e.stream.Conn.Write([]byte(c + "\n")) _, err := e.stream.Conn.Write([]byte(c + "\n"))

View File

@ -94,4 +94,12 @@ type ProcessEnvironment interface {
// Reads the log file for the process from the end backwards until the provided // Reads the log file for the process from the end backwards until the provided
// number of lines is met. // number of lines is met.
Readlog(int) ([]string, error) Readlog(int) ([]string, error)
// Returns the current state of the environment.
State() string
// Sets the current state of the environment. In general you should let the environment
// handle this itself, but there are some scenarios where it is helpful for the server
// to update the state externally (e.g. starting -> started).
SetState(string)
} }

View File

@ -63,7 +63,7 @@ func (s *Server) Install(sync bool) error {
// Ensure that the server is marked as offline at this point, otherwise you end up // Ensure that the server is marked as offline at this point, otherwise you end up
// with a blank value which is a bit confusing. // with a blank value which is a bit confusing.
s.SetState(environment.ProcessOfflineState) s.Environment.SetState(environment.ProcessOfflineState)
// Push an event to the websocket so we can auto-refresh the information in the panel once // Push an event to the websocket so we can auto-refresh the information in the panel once
// the install is completed. // the install is completed.

View File

@ -63,8 +63,8 @@ func (s *Server) StartEventListeners() {
if err != nil { if err != nil {
// If the process is already stopping, just let it continue with that action rather than attempting // If the process is already stopping, just let it continue with that action rather than attempting
// to terminate again. // to terminate again.
if s.GetState() != environment.ProcessStoppingState { if s.Environment.State() != environment.ProcessStoppingState {
s.SetState(environment.ProcessStoppingState) s.Environment.SetState(environment.ProcessStoppingState)
go func() { go func() {
s.Log().Warn("stopping server instance, violating throttle limits") s.Log().Warn("stopping server instance, violating throttle limits")
s.PublishConsoleOutputFromDaemon("Your server is being stopped for outputting too much data in a short period of time.") s.PublishConsoleOutputFromDaemon("Your server is being stopped for outputting too much data in a short period of time.")
@ -73,8 +73,8 @@ func (s *Server) StartEventListeners() {
if err := s.Environment.WaitForStop(config.Get().Throttles.StopGracePeriod, true); err != nil { if err := s.Environment.WaitForStop(config.Get().Throttles.StopGracePeriod, true); err != nil {
// If there is an error set the process back to running so that this throttler is called // If there is an error set the process back to running so that this throttler is called
// again and hopefully kills the server. // again and hopefully kills the server.
if s.GetState() != environment.ProcessOfflineState { if s.Environment.State() != environment.ProcessOfflineState {
s.SetState(environment.ProcessRunningState) s.Environment.SetState(environment.ProcessRunningState)
} }
s.Log().WithField("error", errors.WithStack(err)).Error("failed to terminate environment after triggering throttle") s.Log().WithField("error", errors.WithStack(err)).Error("failed to terminate environment after triggering throttle")
@ -100,7 +100,7 @@ func (s *Server) StartEventListeners() {
s.Throttler().Reset() s.Throttler().Reset()
} }
s.SetState(e.Data) s.OnStateChange()
} }
stats := func(e events.Event) { stats := func(e events.Event) {
@ -173,7 +173,7 @@ func (s *Server) onConsoleOutput(data string) {
// If the specific line of output is one that would mark the server as started, // If the specific line of output is one that would mark the server as started,
// set the server to that state. Only do this if the server is not currently stopped // set the server to that state. Only do this if the server is not currently stopped
// or stopping. // or stopping.
_ = s.SetState(environment.ProcessRunningState) s.Environment.SetState(environment.ProcessRunningState)
break break
} }
} }
@ -185,7 +185,7 @@ func (s *Server) onConsoleOutput(data string) {
stop := processConfiguration.Stop stop := processConfiguration.Stop
if stop.Type == api.ProcessStopCommand && data == stop.Value { if stop.Type == api.ProcessStopCommand && data == stop.Value {
_ = s.SetState(environment.ProcessOfflineState) s.Environment.SetState(environment.ProcessOfflineState)
} }
} }
} }

View File

@ -104,6 +104,7 @@ func FromConfiguration(data api.ServerConfigurationResponse) (*Server, error) {
s.resources = ResourceUsage{} s.resources = ResourceUsage{}
defaults.Set(&s.resources) defaults.Set(&s.resources)
s.resources.State.Store(environment.ProcessOfflineState)
s.Archiver = Archiver{Server: s} s.Archiver = Archiver{Server: s}
s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace()) s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace())

View File

@ -3,7 +3,9 @@ package server
import ( import (
"encoding/json" "encoding/json"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
"github.com/pterodactyl/wings/system"
"sync" "sync"
"sync/atomic"
) )
// Defines the current resource usage for a given server instance. If a server is offline you // Defines the current resource usage for a given server instance. If a server is offline you
@ -16,7 +18,7 @@ type ResourceUsage struct {
environment.Stats environment.Stats
// The current server status. // The current server status.
State string `json:"state" default:"offline"` State system.AtomicString `json:"state"`
// The current disk space being used by the server. This value is not guaranteed to be accurate // The current disk space being used by the server. This value is not guaranteed to be accurate
// at all times. It is "manually" set whenever server.Proc() is called. This is kind of just a // at all times. It is "manually" set whenever server.Proc() is called. This is kind of just a
@ -24,16 +26,16 @@ type ResourceUsage struct {
Disk int64 `json:"disk_bytes"` Disk int64 `json:"disk_bytes"`
} }
// Alias the resource usage so that we don't infinitely recurse when marshaling the struct.
type IResourceUsage ResourceUsage
// Custom marshaler to ensure that the object is locked when we're converting it to JSON in // Custom marshaler to ensure that the object is locked when we're converting it to JSON in
// order to avoid race conditions. // order to avoid race conditions.
func (ru *ResourceUsage) MarshalJSON() ([]byte, error) { func (ru *ResourceUsage) MarshalJSON() ([]byte, error) {
ru.mu.Lock() ru.mu.Lock()
defer ru.mu.Unlock() defer ru.mu.Unlock()
return json.Marshal(IResourceUsage(*ru)) // Alias the resource usage so that we don't infinitely recurse when marshaling the struct.
type alias ResourceUsage
return json.Marshal(alias(*ru))
} }
// Returns the resource usage stats for the server instance. If the server is not running, only the // Returns the resource usage stats for the server instance. If the server is not running, only the
@ -42,10 +44,10 @@ func (ru *ResourceUsage) MarshalJSON() ([]byte, error) {
// //
// When a process is stopped all of the stats are zeroed out except for the disk. // When a process is stopped all of the stats are zeroed out except for the disk.
func (s *Server) Proc() *ResourceUsage { func (s *Server) Proc() *ResourceUsage {
s.resources.SetDisk(s.Filesystem().CachedUsage()) // Store the updated disk usage when requesting process usage.
atomic.StoreInt64(&s.resources.Disk, s.Filesystem().CachedUsage())
// Get a read lock on the resources at this point. Don't do this before setting // Acquire a lock before attempting to return the value of resources.
// the disk, otherwise you'll cause a deadlock.
s.resources.mu.RLock() s.resources.mu.RLock()
defer s.resources.mu.RUnlock() defer s.resources.mu.RUnlock()
@ -57,24 +59,3 @@ func (s *Server) emitProcUsage() {
s.Log().WithField("error", err).Warn("error while emitting server resource usage to listeners") s.Log().WithField("error", err).Warn("error while emitting server resource usage to listeners")
} }
} }
// Returns the servers current state.
func (ru *ResourceUsage) getInternalState() string {
ru.mu.RLock()
defer ru.mu.RUnlock()
return ru.State
}
// Sets the new state for the server.
func (ru *ResourceUsage) setInternalState(state string) {
ru.mu.Lock()
ru.State = state
ru.mu.Unlock()
}
func (ru *ResourceUsage) SetDisk(i int64) {
ru.mu.Lock()
ru.Disk = i
ru.mu.Unlock()
}

View File

@ -2,7 +2,6 @@ package server
import ( import (
"encoding/json" "encoding/json"
"fmt"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
@ -63,23 +62,17 @@ func saveServerStates() error {
// Sets the state of the server internally. This function handles crash detection as // Sets the state of the server internally. This function handles crash detection as
// well as reporting to event listeners for the server. // well as reporting to event listeners for the server.
func (s *Server) SetState(state string) error { func (s *Server) OnStateChange() {
if state != environment.ProcessOfflineState && prevState := s.Proc().State.Load()
state != environment.ProcessStartingState &&
state != environment.ProcessRunningState &&
state != environment.ProcessStoppingState {
return errors.New(fmt.Sprintf("invalid server state received: %s", state))
}
prevState := s.GetState()
st := s.Environment.State()
// Update the currently tracked state for the server. // Update the currently tracked state for the server.
s.Proc().setInternalState(state) s.Proc().State.Store(st)
// Emit the event to any listeners that are currently registered. // Emit the event to any listeners that are currently registered.
if prevState != state { if prevState != s.Environment.State() {
s.Log().WithField("status", s.Proc().getInternalState()).Debug("saw server status change event") s.Log().WithField("status", st).Debug("saw server status change event")
s.Events().Publish(StatusEvent, s.Proc().getInternalState()) s.Events().Publish(StatusEvent, st)
} }
// Persist this change to the disk immediately so that should the Daemon be stopped or // Persist this change to the disk immediately so that should the Daemon be stopped or
@ -98,7 +91,7 @@ func (s *Server) SetState(state string) error {
// Reset the resource usage to 0 when the process fully stops so that all of the UI // Reset the resource usage to 0 when the process fully stops so that all of the UI
// views in the Panel correctly display 0. // views in the Panel correctly display 0.
if state == environment.ProcessOfflineState { if st == environment.ProcessOfflineState {
s.resources.mu.Lock() s.resources.mu.Lock()
s.resources.Empty() s.resources.Empty()
s.resources.mu.Unlock() s.resources.mu.Unlock()
@ -127,13 +120,13 @@ func (s *Server) SetState(state string) error {
} }
}(s) }(s)
} }
return nil
} }
// Returns the current state of the server in a race-safe manner. // Returns the current state of the server in a race-safe manner.
// Deprecated
// use Environment.State()
func (s *Server) GetState() string { func (s *Server) GetState() string {
return s.Proc().getInternalState() return s.Environment.State()
} }
// Determines if the server state is running or not. This is different than the // Determines if the server state is running or not. This is different than the