From 029560394311dd1543f9137a508b7e6a0f0c3ae5 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 12 Sep 2020 22:08:50 -0700 Subject: [PATCH] Speed up wings boot when restoring from cleared docker environment Doesn't pull images for any server that is not supposed to be running at boot time --- cmd/root.go | 46 ++++++++++++++++++++++++++++++++++------------ server/loader.go | 14 +++----------- server/state.go | 2 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 6fe555b..8abd5d1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/NYTimes/logrotate" "github.com/apex/log/handlers/multi" + "github.com/docker/docker/client" "github.com/gammazero/workerpool" "golang.org/x/crypto/acme" "net/http" @@ -172,6 +173,11 @@ func rootCmdRun(*cobra.Command, []string) { log.WithField("server", s.Id()).Info("loaded configuration for server") } + states, err := server.CachedServerStates() + if err != nil { + log.WithField("error", errors.WithStack(err)).Error("failed to retrieve locally cached server states from disk, assuming all servers in offline state") + } + // Create a new workerpool that limits us to 4 servers being bootstrapped at a time // on Wings. This allows us to ensure the environment exists, write configurations, // and reboot processes without causing a slow-down due to sequential booting. @@ -181,25 +187,41 @@ func rootCmdRun(*cobra.Command, []string) { s := serv pool.Submit(func() { - s.Log().Info("ensuring server environment exists") - // Create a server environment if none exists currently. This allows us to recover from Docker - // being reinstalled on the host system for example. - if err := s.Environment.Create(); err != nil { - s.Log().WithField("error", err).Error("failed to process environment") + s.Log().Info("configuring server environment and restoring to previous state") + + var st string + if state, exists := states[s.Id()]; exists { + st = state } r, err := s.Environment.IsRunning() - if err != nil { + // We ignore missing containers because we don't want to actually block booting of wings at this + // point. If we didn't do this and you pruned all of the images and then started wings you could + // end up waiting a long period of time for all of the images to be re-pulled on Wings boot rather + // than when the server itself is started. + if err != nil && !client.IsErrNotFound(err) { s.Log().WithField("error", err).Error("error checking server environment status") } - // If the server is currently running on Docker, mark the process as being in that state. - // We never want to stop an instance that is currently running external from Wings since - // that is a good way of keeping things running even if Wings gets in a very corrupted state. + fmt.Println(s.Id(), st, r, s.IsRunning(), s.GetState()) + // Check if the server was previously running. If so, attempt to start the server now so that Wings + // can pick up where it left off. If the environment does not exist at all, just create it and then allow + // the normal flow to execute. // - // This will also validate that a server process is running if the last tracked state we have - // is that it was running, but we see that the container process is not currently running. - if r || (!r && s.IsRunning()) { + // This does mean that booting wings after a catastrophic machine crash and wiping out the Docker images + // as a result will result in a slow boot. + if !r && (st == environment.ProcessRunningState || st == environment.ProcessStartingState) { + fmt.Println("starting server, not running and should be") + if err := s.HandlePowerAction(server.PowerActionStart); err != nil { + s.Log().WithField("error", errors.WithStack(err)).Warn("failed to return server to running state") + } + } else if r || (!r && s.IsRunning()) { + // If the server is currently running on Docker, mark the process as being in that state. + // We never want to stop an instance that is currently running external from Wings since + // that is a good way of keeping things running even if Wings gets in a very corrupted state. + // + // This will also validate that a server process is running if the last tracked state we have + // is that it was running, but we see that the container process is not currently running. s.Log().Info("detected server is running, re-attaching to process...") s.SetState(environment.ProcessRunningState) diff --git a/server/loader.go b/server/loader.go index 45de2ab..75095f8 100644 --- a/server/loader.go +++ b/server/loader.go @@ -37,12 +37,6 @@ func LoadDirectory() error { return errors.New(rerr.String()) } - log.Debug("retrieving cached server states from disk") - states, err := getServerStates() - if err != nil { - log.WithField("error", errors.WithStack(err)).Error("failed to retrieve locally cached server states from disk, assuming all servers in offline state") - } - start := time.Now() log.WithField("total_configs", len(configs)).Info("processing servers returned by the API") @@ -59,11 +53,6 @@ func LoadDirectory() error { return } - if state, exists := states[s.Id()]; exists { - s.Log().WithField("state", state).Debug("found existing server state in cache file; re-instantiating server state") - s.SetState(state) - } - servers.Add(s) }) } @@ -97,6 +86,9 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) { return nil, err } + s.resources = ResourceUsage{} + defaults.Set(&s.resources) + s.Archiver = Archiver{Server: s} s.Filesystem = Filesystem{Server: s} diff --git a/server/state.go b/server/state.go index b2e1a00..bef2d91 100644 --- a/server/state.go +++ b/server/state.go @@ -15,7 +15,7 @@ import ( var stateMutex sync.Mutex // Returns the state of the servers. -func getServerStates() (map[string]string, error) { +func CachedServerStates() (map[string]string, error) { // Request a lock after we check if the file exists. stateMutex.Lock() defer stateMutex.Unlock()