Speed up wings boot when restoring from cleared docker environment

Doesn't pull images for any server that is not supposed to be running at boot time

cmd/root.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/NYTimes/logrotate"
 	"github.com/apex/log/handlers/multi"
+	"github.com/docker/docker/client"
 	"github.com/gammazero/workerpool"
 	"golang.org/x/crypto/acme"
 	"net/http"
@@ -172,6 +173,11 @@ func rootCmdRun(*cobra.Command, []string) {
 		log.WithField("server", s.Id()).Info("loaded configuration for server")
 	}
 
+	states, err := server.CachedServerStates()
+	if err != nil {
+		log.WithField("error", errors.WithStack(err)).Error("failed to retrieve locally cached server states from disk, assuming all servers in offline state")
+	}
+
 	// Create a new workerpool that limits us to 4 servers being bootstrapped at a time
 	// on Wings. This allows us to ensure the environment exists, write configurations,
 	// and reboot processes without causing a slow-down due to sequential booting.
@@ -181,25 +187,41 @@ func rootCmdRun(*cobra.Command, []string) {
 		s := serv
 
 		pool.Submit(func() {
-			s.Log().Info("ensuring server environment exists")
+			s.Log().Info("configuring server environment and restoring to previous state")
-			// Create a server environment if none exists currently. This allows us to recover from Docker
+
-			// being reinstalled on the host system for example.
+			var st string
-			if err := s.Environment.Create(); err != nil {
+			if state, exists := states[s.Id()]; exists {
-				s.Log().WithField("error", err).Error("failed to process environment")
+				st = state
 			}
 
 			r, err := s.Environment.IsRunning()
-			if err != nil {
+			// We ignore missing containers because we don't want to actually block booting of wings at this
+			// point. If we didn't do this and you pruned all of the images and then started wings you could
+			// end up waiting a long period of time for all of the images to be re-pulled on Wings boot rather
+			// than when the server itself is started.
+			if err != nil && !client.IsErrNotFound(err) {
 				s.Log().WithField("error", err).Error("error checking server environment status")
 			}
 
+			fmt.Println(s.Id(), st, r, s.IsRunning(), s.GetState())
+			// Check if the server was previously running. If so, attempt to start the server now so that Wings
+			// can pick up where it left off. If the environment does not exist at all, just create it and then allow
+			// the normal flow to execute.
+			//
+			// This does mean that booting wings after a catastrophic machine crash and wiping out the Docker images
+			// as a result will result in a slow boot.
+			if !r && (st == environment.ProcessRunningState || st == environment.ProcessStartingState) {
+				fmt.Println("starting server, not running and should be")
+				if err := s.HandlePowerAction(server.PowerActionStart); err != nil {
+					s.Log().WithField("error", errors.WithStack(err)).Warn("failed to return server to running state")
+				}
+			} else if r || (!r && s.IsRunning()) {
 				// If the server is currently running on Docker, mark the process as being in that state.
 				// We never want to stop an instance that is currently running external from Wings since
 				// that is a good way of keeping things running even if Wings gets in a very corrupted state.
 				//
 				// This will also validate that a server process is running if the last tracked state we have
 				// is that it was running, but we see that the container process is not currently running.
-			if r || (!r && s.IsRunning()) {
 				s.Log().Info("detected server is running, re-attaching to process...")
 
 				s.SetState(environment.ProcessRunningState)

server/loader.go

@@ -37,12 +37,6 @@ func LoadDirectory() error {
 		return errors.New(rerr.String())
 	}
 
-	log.Debug("retrieving cached server states from disk")
-	states, err := getServerStates()
-	if err != nil {
-		log.WithField("error", errors.WithStack(err)).Error("failed to retrieve locally cached server states from disk, assuming all servers in offline state")
-	}
-
 	start := time.Now()
 	log.WithField("total_configs", len(configs)).Info("processing servers returned by the API")
 
@@ -59,11 +53,6 @@ func LoadDirectory() error {
 				return
 			}
 
-			if state, exists := states[s.Id()]; exists {
-				s.Log().WithField("state", state).Debug("found existing server state in cache file; re-instantiating server state")
-				s.SetState(state)
-			}
-
 			servers.Add(s)
 		})
 	}
@@ -97,6 +86,9 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) {
 		return nil, err
 	}
 
+	s.resources = ResourceUsage{}
+	defaults.Set(&s.resources)
+
 	s.Archiver = Archiver{Server: s}
 	s.Filesystem = Filesystem{Server: s}
 

server/state.go

@@ -15,7 +15,7 @@ import (
 var stateMutex sync.Mutex
 
 // Returns the state of the servers.
-func getServerStates() (map[string]string, error) {
+func CachedServerStates() (map[string]string, error) {
 	// Request a lock after we check if the file exists.
 	stateMutex.Lock()
 	defer stateMutex.Unlock()