diff --git a/router/router_server.go b/router/router_server.go index 4946223..a516005 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -164,6 +164,11 @@ func deleteServer(c *gin.Context) { // to start it while this process is running. s.Suspended = true + // If the server is currently installing, abort it. + if s.IsInstalling() { + s.AbortInstallation() + } + // Delete the server's archive if it exists. We intentionally don't return // here, if the archive fails to delete, the server can still be removed. if err := s.Archiver.DeleteIfExists(); err != nil { diff --git a/server/install.go b/server/install.go index b89ffae..e5e8ad9 100644 --- a/server/install.go +++ b/server/install.go @@ -12,12 +12,14 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" + "golang.org/x/sync/semaphore" "io" "io/ioutil" "os" "path" "path/filepath" "sync" + "time" ) // Executes the installation stack for a server process. Bubbles any errors up to the calling @@ -84,8 +86,8 @@ type InstallationProcess struct { Server *Server Script *api.InstallationScript - client *client.Client - mutex *sync.Mutex + client *client.Client + context context.Context } // Generates a new installation process struct that will be used to create containers, @@ -94,21 +96,70 @@ func NewInstallationProcess(s *Server, script *api.InstallationScript) (*Install proc := &InstallationProcess{ Script: script, Server: s, - mutex: &sync.Mutex{}, } + ctx, cancel := context.WithCancel(context.Background()) + s.installer.cancel = &cancel + if c, err := client.NewClientWithOpts(client.FromEnv); err != nil { return nil, errors.WithStack(err) } else { proc.client = c + proc.context = ctx } return proc, nil } +// Try to obtain an exclusive lock on the installation process for the server. Waits up to 10 +// seconds before aborting with a context timeout. +func (s *Server) acquireInstallationLock() error { + if s.installer.sem == nil { + s.installer.sem = semaphore.NewWeighted(1) + } + + ctx, _ := context.WithTimeout(context.Background(), time.Second*10) + + return s.installer.sem.Acquire(ctx, 1) +} + +// Determines if the server is actively running the installation process by checking the status +// of the semaphore lock. +func (s *Server) IsInstalling() bool { + if s.installer.sem == nil { + return false + } + + if s.installer.sem.TryAcquire(1) { + // If we made it into this block it means we were able to obtain an exclusive lock + // on the semaphore. In that case, go ahead and release that lock immediately, and + // return false. + s.installer.sem.Release(1) + + return false + } + + return true +} + +// Aborts the server installation process by calling the cancel function on the installer +// context. +func (s *Server) AbortInstallation() { + if !s.IsInstalling() { + return + } + + if s.installer.cancel != nil { + cancel := *s.installer.cancel + + s.Log().Warn("aborting running installation process") + cancel() + } +} + // Removes the installer container for the server. func (ip *InstallationProcess) RemoveContainer() { - err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{ + err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, }) @@ -124,6 +175,20 @@ func (ip *InstallationProcess) RemoveContainer() { // Once the container finishes installing the results will be stored in an installation // log in the server's configuration directory. func (ip *InstallationProcess) Run() error { + ip.Server.Log().Debug("acquiring installation process lock") + if err := ip.Server.acquireInstallationLock(); err != nil { + return err + } + + // We now have an exclusive lock on this installation process. Ensure that whenever this + // process is finished that the semaphore is released so that other processes and be executed + // without encounting a wait timeout. + defer func() { + ip.Server.Log().Debug("releasing installation process lock") + ip.Server.installer.sem.Release(1) + ip.Server.installer.cancel = nil + }() + installPath, err := ip.BeforeExecute() if err != nil { return err @@ -183,7 +248,7 @@ func (ip *InstallationProcess) writeScriptToDisk() (string, error) { // Pulls the docker image to be used for the installation container. func (ip *InstallationProcess) pullInstallationImage() error { - r, err := ip.client.ImagePull(context.Background(), ip.Script.ContainerImage, types.ImagePullOptions{}) + r, err := ip.client.ImagePull(ip.context, ip.Script.ContainerImage, types.ImagePullOptions{}) if err != nil { return errors.WithStack(err) } @@ -237,7 +302,7 @@ func (ip *InstallationProcess) BeforeExecute() (string, error) { Force: true, } - if err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", opts); err != nil { + if err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", opts); err != nil { if !client.IsErrNotFound(err) { e = append(e, err) } @@ -264,11 +329,10 @@ func (ip *InstallationProcess) GetLogPath() string { // process to store in the server configuration directory, and then destroys the associated // installation container. func (ip *InstallationProcess) AfterExecute(containerId string) error { - ctx := context.Background() defer ip.RemoveContainer() ip.Server.Log().WithField("container_id", containerId).Debug("pulling installation logs for server") - reader, err := ip.client.ContainerLogs(ctx, containerId, types.ContainerLogsOptions{ + reader, err := ip.client.ContainerLogs(ip.context, containerId, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, Follow: false, @@ -295,8 +359,6 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { // Executes the installation process inside a specially created docker container. func (ip *InstallationProcess) Execute(installPath string) (string, error) { - ctx := context.Background() - conf := &container.Config{ Hostname: "installer", AttachStdout: true, @@ -345,13 +407,13 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { } ip.Server.Log().WithField("install_script", installPath+"/install.sh").Info("creating install container for server process") - r, err := ip.client.ContainerCreate(ctx, conf, hostConf, nil, ip.Server.Uuid+"_installer") + r, err := ip.client.ContainerCreate(ip.context, conf, hostConf, nil, ip.Server.Uuid+"_installer") if err != nil { return "", errors.WithStack(err) } ip.Server.Log().WithField("container_id", r.ID).Info("running installation script for server in container") - if err := ip.client.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil { + if err := ip.client.ContainerStart(ip.context, r.ID, types.ContainerStartOptions{}); err != nil { return "", err } @@ -363,7 +425,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.") }(r.ID) - sChann, eChann := ip.client.ContainerWait(ctx, r.ID, container.WaitConditionNotRunning) + sChann, eChann := ip.client.ContainerWait(ip.context, r.ID, container.WaitConditionNotRunning) select { case err := <-eChann: if err != nil { @@ -379,7 +441,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { // directory, as well as to a websocket listener so that the process can be viewed in // the panel by administrators. func (ip *InstallationProcess) StreamOutput(id string) error { - reader, err := ip.client.ContainerLogs(context.Background(), id, types.ContainerLogsOptions{ + reader, err := ip.client.ContainerLogs(ip.context, id, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, Follow: true, @@ -399,7 +461,7 @@ func (ip *InstallationProcess) StreamOutput(id string) error { if err := s.Err(); err != nil { ip.Server.Log().WithFields(log.Fields{ "container_id": id, - "error": errors.WithStack(err), + "error": errors.WithStack(err), }).Warn("error processing scanner line in installation output for server") } diff --git a/server/server.go b/server/server.go index b510185..5389841 100644 --- a/server/server.go +++ b/server/server.go @@ -1,6 +1,7 @@ package server import ( + "context" "fmt" "github.com/apex/log" "github.com/creasty/defaults" @@ -9,6 +10,7 @@ import ( "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" "github.com/remeh/sizedwaitgroup" + "golang.org/x/sync/semaphore" "math" "os" "strings" @@ -71,11 +73,27 @@ type Server struct { // started, and then cached here. processConfiguration *api.ProcessConfiguration + // Tracks the installation process for this server and prevents a server from running + // two installer processes at the same time. This also allows us to cancel a running + // installation process, for example when a server is deleted from the panel while the + // installer process is still running. + installer InstallerDetails + // Internal mutex used to block actions that need to occur sequentially, such as // writing the configuration to the disk. sync.RWMutex } +type InstallerDetails struct { + // The cancel function for the installer. This will be a non-nil value while there + // is an installer running for the server. + cancel *context.CancelFunc + + // Installer lock. You should obtain an exclusive lock on this context while running + // the installation process and release it when finished. + sem *semaphore.Weighted +} + // The build settings for a given server that impact docker container creation and // resource limits for a server instance. type BuildSettings struct {