diff --git a/go.mod b/go.mod index 0c57322..9da1925 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 // indirect golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect - golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect + golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect diff --git a/go.sum b/go.sum index 4c90238..fb13015 100644 --- a/go.sum +++ b/go.sum @@ -334,6 +334,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/server/environment_docker.go b/server/environment_docker.go index 37bcf04..825289f 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -146,6 +146,10 @@ func (d *DockerEnvironment) OnBeforeStart() error { return err } + if !d.Server.Filesystem.HasSpaceAvailable() { + return errors.New("cannot start server, not enough disk space available") + } + // Always destroy and re-create the server container to ensure that synced data from // the Panel is used. if err := d.Client.ContainerRemove(context.Background(), d.Server.Uuid, types.ContainerRemoveOptions{RemoveVolumes: true}); err != nil { diff --git a/server/filesystem.go b/server/filesystem.go index 1fa3ee6..9adc6ee 100644 --- a/server/filesystem.go +++ b/server/filesystem.go @@ -3,6 +3,7 @@ package server import ( "bufio" "bytes" + "context" "encoding/json" "fmt" "github.com/gabriel-vasile/mimetype" @@ -67,7 +68,7 @@ func (fs *Filesystem) SafePath(p string) (string, error) { // Range over all of the path parts and form directory pathings from the end // moving up until we have a valid resolution or we run out of paths to try. for k := range parts { - try = strings.Join(parts[:(len(parts) - k)], "/") + try = strings.Join(parts[:(len(parts)-k)], "/") if !strings.HasPrefix(try, fs.Path()) { break @@ -118,23 +119,27 @@ func (fs *Filesystem) HasSpaceAvailable() bool { return true } - var size int64 + // If we have a match in the cache, use that value in the return. No need to perform an expensive + // disk operation, even if this is an empty value. if x, exists := fs.Server.Cache.Get("disk_used"); exists { - size = x.(int64) + fs.Server.Resources.Disk = x.(int64) + return (x.(int64) / 1000.0 / 1000.0) <= space } // If there is no size its either because there is no data (in which case running this function // will have effectively no impact), or there is nothing in the cache, in which case we need to // grab the size of their data directory. This is a taxing operation, so we want to store it in // the cache once we've gotten it. - if size == 0 { - if size, err := fs.DirectorySize("/"); err != nil { - zap.S().Warnw("failed to determine directory size", zap.String("server", fs.Server.Uuid), zap.Error(err)) - } else { - fs.Server.Cache.Set("disk_used", size, time.Second * 60) - } + size, err := fs.DirectorySize("/") + if err != nil { + zap.S().Warnw("failed to determine directory size", zap.String("server", fs.Server.Uuid), zap.Error(err)) } + // Always cache the size, even if there is an error. We want to always return that value + // so that we don't cause an endless loop of determining the disk size if there is a temporary + // error encountered. + fs.Server.Cache.Set("disk_used", size, time.Second*60) + // Determine if their folder size, in bytes, is smaller than the amount of space they've // been allocated. fs.Server.Resources.Disk = size @@ -146,42 +151,15 @@ func (fs *Filesystem) HasSpaceAvailable() bool { // through all of the folders. Returns the size in bytes. This can be a fairly taxing operation // on locations with tons of files, so it is recommended that you cache the output. func (fs *Filesystem) DirectorySize(dir string) (int64, error) { + w := fs.NewWalker() + ctx := context.Background() + var size int64 - var wg sync.WaitGroup + err := w.Walk(dir, ctx, func(f os.FileInfo) { + atomic.AddInt64(&size, f.Size()) + }) - cleaned, err := fs.SafePath(dir) - if err != nil { - return 0, err - } - - files, err := ioutil.ReadDir(cleaned) - if err != nil { - return 0, err - } - - // Iterate over all of the files and directories. If it is a file, immediately add its size - // to the total size being returned. If we're dealing with a directory, call this function - // on a seperate thread until we have gotten the size of everything nested within the given - // directory. - for _, f := range files { - if f.IsDir() { - wg.Add(1) - - go func(p string) { - defer wg.Done() - - s, _ := fs.DirectorySize(p) - - atomic.AddInt64(&size, s) - }(filepath.Join(cleaned, f.Name())) - } else { - atomic.AddInt64(&size, f.Size()) - } - } - - wg.Wait() - - return size, nil + return size, err } // Reads a file on the system and returns it as a byte representation in a file diff --git a/server/filesystem_walker.go b/server/filesystem_walker.go new file mode 100644 index 0000000..c6e870f --- /dev/null +++ b/server/filesystem_walker.go @@ -0,0 +1,61 @@ +package server + +import ( + "context" + "golang.org/x/sync/errgroup" + "io/ioutil" + "os" + "path/filepath" +) + +type FileWalker struct { + *Filesystem +} + +// Returns a new walker instance. +func (fs *Filesystem) NewWalker() *FileWalker { + return &FileWalker{fs} +} + +// Iterate over all of the files and directories within a given directory. When a file is +// found the callback will be called with the file information. If a directory is encountered +// it will be recursively passed back through to this function. +func (fw *FileWalker) Walk(dir string, ctx context.Context, callback func (os.FileInfo)) error { + cleaned, err := fw.SafePath(dir) + if err != nil { + return err + } + + // Get all of the files from this directory. + files, err := ioutil.ReadDir(cleaned) + if err != nil { + return err + } + + // Create an error group that we can use to run processes in parallel while retaining + // the ability to cancel the entire process immediately should any of it fail. + g, ctx := errgroup.WithContext(ctx) + + for _, f := range files { + if f.IsDir() { + p := filepath.Join(dir, f.Name()) + // Recursively call this function to continue digging through the directory tree within + // a seperate goroutine. If the context is canceled abort this process. + g.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return fw.Walk(p, ctx, callback) + } + }) + } else { + // If this isn't a directory, go ahead and pass the file information into the + // callback. + callback(f) + } + } + + // Block until all of the routines finish and have returned a value. + return g.Wait() +} \ No newline at end of file