Better race condition prevention

server/state.go

@@ -2,7 +2,9 @@ package server
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/pkg/errors"
+	"go.uber.org/zap"
 	"io/ioutil"
 	"os"
 	"sync"
@@ -61,12 +63,12 @@ func getServerStates() (map[string]string, error) {
 	return states, nil
 }
 
-// SaveServerStates .
-func SaveServerStates() error {
+// saveServerStates .
+func saveServerStates() error {
 	// Get the states of all servers on the daemon.
 	states := map[string]string{}
 	for _, s := range GetServers().All() {
-		states[s.Uuid] = s.State
+		states[s.Uuid] = s.GetState()
 	}
 
 	// Convert the map to a json object.
@@ -85,3 +87,84 @@ func SaveServerStates() error {
 
 	return nil
 }
+
+const (
+	ProcessOfflineState  = "offline"
+	ProcessStartingState = "starting"
+	ProcessRunningState  = "running"
+	ProcessStoppingState = "stopping"
+)
+
+// Sets the state of the server internally. This function handles crash detection as
+// well as reporting to event listeners for the server.
+func (s *Server) SetState(state string) error {
+	if state != ProcessOfflineState && state != ProcessStartingState && state != ProcessRunningState && state != ProcessStoppingState {
+		return errors.New(fmt.Sprintf("invalid server state received: %s", state))
+	}
+
+	prevState := s.GetState()
+
+	// Obtain a mutex lock and update the current state of the server.
+	s.Lock()
+	s.State = state
+
+	// Emit the event to any listeners that are currently registered.
+	zap.S().Debugw("saw server status change event", zap.String("server", s.Uuid), zap.String("status", s.State))
+	s.Events().Publish(StatusEvent, s.State)
+
+	// Release the lock as it is no longer needed for the following actions.
+	s.Unlock()
+
+	// Persist this change to the disk immediately so that should the Daemon be stopped or
+	// crash we can immediately restore the server state.
+	//
+	// This really only makes a difference if all of the Docker containers are also stopped,
+	// but this was a highly requested feature and isn't hard to work with, so lets do it.
+	//
+	// We also get the benefit of server status changes always propagating corrected configurations
+	// to the disk should we forget to do it elsewhere.
+	go func() {
+		if err := saveServerStates(); err != nil {
+			zap.S().Warnw("failed to write server states to disk", zap.Error(err))
+		}
+	}()
+
+	// If server was in an online state, and is now in an offline state we should handle
+	// that as a crash event. In that scenario, check the last crash time, and the crash
+	// counter.
+	//
+	// In the event that we have passed the thresholds, don't do anything, otherwise
+	// automatically attempt to start the process back up for the user. This is done in a
+	// separate thread as to not block any actions currently taking place in the flow
+	// that called this function.
+	if (prevState == ProcessStartingState || prevState == ProcessRunningState) && s.GetState() == ProcessOfflineState {
+		zap.S().Infow("detected server as entering a potentially crashed state; running handler", zap.String("server", s.Uuid))
+
+		go func(server *Server) {
+			if err := server.handleServerCrash(); err != nil {
+				if IsTooFrequentCrashError(err) {
+					zap.S().Infow("did not restart server after crash; occurred too soon after last", zap.String("server", server.Uuid))
+				} else {
+					zap.S().Errorw("failed to handle server crash state", zap.String("server", server.Uuid), zap.Error(err))
+				}
+			}
+		}(s)
+	}
+
+	return nil
+}
+
+// Returns the current state of the server in a race-safe manner.
+func (s *Server) GetState() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.State
+}
+
+// Determines if the server state is running or not. This is different than the
+// environment state, it is simply the tracked state from this daemon instance, and
+// not the response from Docker.
+func (s *Server) IsRunning() bool {
+	return s.GetState() == ProcessRunningState || s.GetState() == ProcessStartingState
+}