Better handling of stop & restart without releasing process locks too soon

This commit is contained in:
Dane Everitt 2020-08-13 21:10:33 -07:00
parent 5fcec86e98
commit 63c09adaa1
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
3 changed files with 37 additions and 20 deletions

View File

@ -129,20 +129,34 @@ func (e *Environment) Start() error {
}
// Restarts the server process by waiting for the process to gracefully stop and then triggering a
// start commane. This will return an error if there is already a restart process executing for the
// server. The lock is released when the process is stopped and a start has begun.
func (e *Environment) Restart() error {
err := e.WaitForStop(60, false)
if err != nil {
return err
// start command. This works identially to how
func (e *Environment) Restart(seconds uint, terminate bool) error {
// Only try to wait for stop if the process is currently running, otherwise just skip right to the
// start event.
if r, _ := e.IsRunning(); !r {
if err := e.WaitForStop(seconds, terminate); err != nil {
// Even timeout errors should be bubbled back up the stack. If the process didn't stop
// nicely, but the terminate argument was passed then the server is stopped without an
// error being returned.
//
// However, if terminate is not passed you'll get a context deadline error. We could
// probably handle that nicely here, but I'd rather just pass it back up the stack for now.
// Either way, any type of error indicates we should not attempt to start the server back
// up.
return err
}
}
// Start the process.
return e.Start()
}
// Stops the container that the server is running in. This will allow up to 10
// seconds to pass before a failure occurs.
// Stops the container that the server is running in. This will allow up to 30 seconds to pass
// before the container is forcefully terminated if we are trying to stop it without using a command
// sent into the instance.
//
// You most likely want to be using WaitForStop() rather than this function, since this will return
// as soon as the command is sent, rather than waiting for the process to be completed stopped.
func (e *Environment) Stop() error {
e.mu.RLock()
s := e.meta.Stop
@ -164,8 +178,7 @@ func (e *Environment) Stop() error {
return e.SendCommand(s.Value)
}
t := time.Second * 10
t := time.Second * 30
err := e.client.ContainerStop(context.Background(), e.Id, &t)
if err != nil {
// If the container does not exist just mark the process as stopped and return without
@ -183,10 +196,10 @@ func (e *Environment) Stop() error {
return nil
}
// Attempts to gracefully stop a server using the defined stop commane. If the server
// Attempts to gracefully stop a server using the defined stop command. If the server
// does not stop after seconds have passed, an error will be returned, or the instance
// will be terminated forcefully depending on the value of the second argument.
func (e *Environment) WaitForStop(seconds int, terminate bool) error {
func (e *Environment) WaitForStop(seconds uint, terminate bool) error {
if err := e.Stop(); err != nil {
return errors.WithStack(err)
}

View File

@ -47,16 +47,16 @@ type ProcessEnvironment interface {
// not be returned.
Stop() error
// Restart a server instance. If already stopped the process will be started. This function
// will return an error if the server is already performing a restart process as to avoid
// unnecessary double/triple/quad looping issues if multiple people press restart or spam the
// button to restart.
Restart() error
// Restart a server instance. If already stopped the process will be started. Pass the number
// of seconds to wait for the server to stop, and wether or not the process should be terminated
// if not stopped within that time period. If you pass "false" as the terminate option the server
// will not be started but the lock on the process will be released.
Restart(seconds uint, terminate bool) error
// Waits for a server instance to stop gracefully. If the server is still detected
// as running after seconds, an error will be returned, or the server will be terminated
// depending on the value of the second argument.
WaitForStop(seconds int, terminate bool) error
WaitForStop(seconds uint, terminate bool) error
// Terminates a running server instance using the provided signal. If the server
// is not running no error should be returned.

View File

@ -106,9 +106,13 @@ func (s *Server) HandlePowerAction(action PowerAction, waitSeconds ...int) error
case PowerActionStart:
return s.Environment.Start()
case PowerActionStop:
return s.Environment.Stop()
// We're specificially waiting for the process to be stopped here, otherwise the lock is released
// too soon, and you can rack up all sorts of issues.
return s.Environment.WaitForStop(10 * 60, true)
case PowerActionRestart:
return s.Environment.Restart()
// Same as stopping, give the process up to 10 minutes to stop before just forcibly terminating
// the process and moving on with things.
return s.Environment.Restart(10 * 60, true)
case PowerActionTerminate:
return s.Environment.Terminate(os.Kill)
}