MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Acquire exclusive lock when installing a server

Browse Source 
Also allows aborting a server install mid-process when the server is deleted before the process finishes.
This commit is contained in:
Dane Everitt 2020-06-22 21:38:16 -07:00
parent 073247e4e1
commit 1f6789cba3
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
3 changed files with 100 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
router/router_server.go
View File

@ -164,6 +164,11 @@ func deleteServer(c *gin.Context) {
// to start it while this process is running.
s.Suspended = true
// If the server is currently installing, abort it.
if s.IsInstalling() {
s.AbortInstallation()
}
// Delete the server's archive if it exists. We intentionally don't return
// here, if the archive fails to delete, the server can still be removed.
if err := s.Archiver.DeleteIfExists(); err != nil {

92
server/install.go
View File

@ -12,12 +12,14 @@ import (
"github.com/pkg/errors"
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/config"
"golang.org/x/sync/semaphore"
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"sync"
"time"
)
// Executes the installation stack for a server process. Bubbles any errors up to the calling
@ -84,8 +86,8 @@ type InstallationProcess struct {
Server *Server
Script *api.InstallationScript
client *client.Client
mutex *sync.Mutex
client *client.Client
context context.Context
}
// Generates a new installation process struct that will be used to create containers,
@ -94,21 +96,70 @@ func NewInstallationProcess(s *Server, script *api.InstallationScript) (*Install
proc := &InstallationProcess{
Script: script,
Server: s,
mutex: &sync.Mutex{},
}
ctx, cancel := context.WithCancel(context.Background())
s.installer.cancel = &cancel
if c, err := client.NewClientWithOpts(client.FromEnv); err != nil {
return nil, errors.WithStack(err)
} else {
proc.client = c
proc.context = ctx
}
return proc, nil
}
// Try to obtain an exclusive lock on the installation process for the server. Waits up to 10
// seconds before aborting with a context timeout.
func (s *Server) acquireInstallationLock() error {
if s.installer.sem == nil {
s.installer.sem = semaphore.NewWeighted(1)
}
ctx, _ := context.WithTimeout(context.Background(), time.Second*10)
return s.installer.sem.Acquire(ctx, 1)
}
// Determines if the server is actively running the installation process by checking the status
// of the semaphore lock.
func (s *Server) IsInstalling() bool {
if s.installer.sem == nil {
return false
}
if s.installer.sem.TryAcquire(1) {
// If we made it into this block it means we were able to obtain an exclusive lock
// on the semaphore. In that case, go ahead and release that lock immediately, and
// return false.
s.installer.sem.Release(1)
return false
}
return true
}
// Aborts the server installation process by calling the cancel function on the installer
// context.
func (s *Server) AbortInstallation() {
if !s.IsInstalling() {
return
}
if s.installer.cancel != nil {
cancel := *s.installer.cancel
s.Log().Warn("aborting running installation process")
cancel()
}
}
// Removes the installer container for the server.
func (ip *InstallationProcess) RemoveContainer() {
err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{
err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{
RemoveVolumes: true,
Force: true,
})
@ -124,6 +175,20 @@ func (ip *InstallationProcess) RemoveContainer() {
// Once the container finishes installing the results will be stored in an installation
// log in the server's configuration directory.
func (ip *InstallationProcess) Run() error {
ip.Server.Log().Debug("acquiring installation process lock")
if err := ip.Server.acquireInstallationLock(); err != nil {
return err
}
// We now have an exclusive lock on this installation process. Ensure that whenever this
// process is finished that the semaphore is released so that other processes and be executed
// without encounting a wait timeout.
defer func() {
ip.Server.Log().Debug("releasing installation process lock")
ip.Server.installer.sem.Release(1)
ip.Server.installer.cancel = nil
}()
installPath, err := ip.BeforeExecute()
if err != nil {
return err
@ -183,7 +248,7 @@ func (ip *InstallationProcess) writeScriptToDisk() (string, error) {
// Pulls the docker image to be used for the installation container.
func (ip *InstallationProcess) pullInstallationImage() error {
r, err := ip.client.ImagePull(context.Background(), ip.Script.ContainerImage, types.ImagePullOptions{})
r, err := ip.client.ImagePull(ip.context, ip.Script.ContainerImage, types.ImagePullOptions{})
if err != nil {
return errors.WithStack(err)
}
@ -237,7 +302,7 @@ func (ip *InstallationProcess) BeforeExecute() (string, error) {
Force: true,
}
if err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", opts); err != nil {
if err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", opts); err != nil {
if !client.IsErrNotFound(err) {
e = append(e, err)
}
@ -264,11 +329,10 @@ func (ip *InstallationProcess) GetLogPath() string {
// process to store in the server configuration directory, and then destroys the associated
// installation container.
func (ip *InstallationProcess) AfterExecute(containerId string) error {
ctx := context.Background()
defer ip.RemoveContainer()
ip.Server.Log().WithField("container_id", containerId).Debug("pulling installation logs for server")
reader, err := ip.client.ContainerLogs(ctx, containerId, types.ContainerLogsOptions{
reader, err := ip.client.ContainerLogs(ip.context, containerId, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: false,
@ -295,8 +359,6 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error {
// Executes the installation process inside a specially created docker container.
func (ip *InstallationProcess) Execute(installPath string) (string, error) {
ctx := context.Background()
conf := &container.Config{
Hostname: "installer",
AttachStdout: true,
@ -345,13 +407,13 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) {
}
ip.Server.Log().WithField("install_script", installPath+"/install.sh").Info("creating install container for server process")
r, err := ip.client.ContainerCreate(ctx, conf, hostConf, nil, ip.Server.Uuid+"_installer")
r, err := ip.client.ContainerCreate(ip.context, conf, hostConf, nil, ip.Server.Uuid+"_installer")
if err != nil {
return "", errors.WithStack(err)
}
ip.Server.Log().WithField("container_id", r.ID).Info("running installation script for server in container")
if err := ip.client.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil {
if err := ip.client.ContainerStart(ip.context, r.ID, types.ContainerStartOptions{}); err != nil {
return "", err
}
@ -363,7 +425,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) {
ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.")
}(r.ID)
sChann, eChann := ip.client.ContainerWait(ctx, r.ID, container.WaitConditionNotRunning)
sChann, eChann := ip.client.ContainerWait(ip.context, r.ID, container.WaitConditionNotRunning)
select {
case err := <-eChann:
if err != nil {
@ -379,7 +441,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) {
// directory, as well as to a websocket listener so that the process can be viewed in
// the panel by administrators.
func (ip *InstallationProcess) StreamOutput(id string) error {
reader, err := ip.client.ContainerLogs(context.Background(), id, types.ContainerLogsOptions{
reader, err := ip.client.ContainerLogs(ip.context, id, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Follow: true,
@ -399,7 +461,7 @@ func (ip *InstallationProcess) StreamOutput(id string) error {
if err := s.Err(); err != nil {
ip.Server.Log().WithFields(log.Fields{
"container_id": id,
"error": errors.WithStack(err),
"error": errors.WithStack(err),
}).Warn("error processing scanner line in installation output for server")
}

18
server/server.go
View File

@ -1,6 +1,7 @@
package server
import (
"context"
"fmt"
"github.com/apex/log"
"github.com/creasty/defaults"
@ -9,6 +10,7 @@ import (
"github.com/pterodactyl/wings/api"
"github.com/pterodactyl/wings/config"
"github.com/remeh/sizedwaitgroup"
"golang.org/x/sync/semaphore"
"math"
"os"
"strings"
@ -71,11 +73,27 @@ type Server struct {
// started, and then cached here.
processConfiguration *api.ProcessConfiguration
// Tracks the installation process for this server and prevents a server from running
// two installer processes at the same time. This also allows us to cancel a running
// installation process, for example when a server is deleted from the panel while the
// installer process is still running.
installer InstallerDetails
// Internal mutex used to block actions that need to occur sequentially, such as
// writing the configuration to the disk.
sync.RWMutex
}
type InstallerDetails struct {
// The cancel function for the installer. This will be a non-nil value while there
// is an installer running for the server.
cancel *context.CancelFunc
// Installer lock. You should obtain an exclusive lock on this context while running
// the installation process and release it when finished.
sem *semaphore.Weighted
}
// The build settings for a given server that impact docker container creation and
// resource limits for a server instance.
type BuildSettings struct {