Cleanup logic for getting a directory size; support cancelation during goroutine
This commit is contained in:
parent
9e0cacc076
commit
71d38ff62e
2
go.mod
2
go.mod
|
@ -62,7 +62,7 @@ require (
|
|||
golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 // indirect
|
||||
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
|
||||
golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect
|
||||
golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
|
||||
|
|
1
go.sum
1
go.sum
|
@ -334,6 +334,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o=
|
||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
|
|
|
@ -146,6 +146,10 @@ func (d *DockerEnvironment) OnBeforeStart() error {
|
|||
return err
|
||||
}
|
||||
|
||||
if !d.Server.Filesystem.HasSpaceAvailable() {
|
||||
return errors.New("cannot start server, not enough disk space available")
|
||||
}
|
||||
|
||||
// Always destroy and re-create the server container to ensure that synced data from
|
||||
// the Panel is used.
|
||||
if err := d.Client.ContainerRemove(context.Background(), d.Server.Uuid, types.ContainerRemoveOptions{RemoveVolumes: true}); err != nil {
|
||||
|
|
|
@ -3,6 +3,7 @@ package server
|
|||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/gabriel-vasile/mimetype"
|
||||
|
@ -67,7 +68,7 @@ func (fs *Filesystem) SafePath(p string) (string, error) {
|
|||
// Range over all of the path parts and form directory pathings from the end
|
||||
// moving up until we have a valid resolution or we run out of paths to try.
|
||||
for k := range parts {
|
||||
try = strings.Join(parts[:(len(parts) - k)], "/")
|
||||
try = strings.Join(parts[:(len(parts)-k)], "/")
|
||||
|
||||
if !strings.HasPrefix(try, fs.Path()) {
|
||||
break
|
||||
|
@ -118,23 +119,27 @@ func (fs *Filesystem) HasSpaceAvailable() bool {
|
|||
return true
|
||||
}
|
||||
|
||||
var size int64
|
||||
// If we have a match in the cache, use that value in the return. No need to perform an expensive
|
||||
// disk operation, even if this is an empty value.
|
||||
if x, exists := fs.Server.Cache.Get("disk_used"); exists {
|
||||
size = x.(int64)
|
||||
fs.Server.Resources.Disk = x.(int64)
|
||||
return (x.(int64) / 1000.0 / 1000.0) <= space
|
||||
}
|
||||
|
||||
// If there is no size its either because there is no data (in which case running this function
|
||||
// will have effectively no impact), or there is nothing in the cache, in which case we need to
|
||||
// grab the size of their data directory. This is a taxing operation, so we want to store it in
|
||||
// the cache once we've gotten it.
|
||||
if size == 0 {
|
||||
if size, err := fs.DirectorySize("/"); err != nil {
|
||||
size, err := fs.DirectorySize("/")
|
||||
if err != nil {
|
||||
zap.S().Warnw("failed to determine directory size", zap.String("server", fs.Server.Uuid), zap.Error(err))
|
||||
} else {
|
||||
fs.Server.Cache.Set("disk_used", size, time.Second * 60)
|
||||
}
|
||||
}
|
||||
|
||||
// Always cache the size, even if there is an error. We want to always return that value
|
||||
// so that we don't cause an endless loop of determining the disk size if there is a temporary
|
||||
// error encountered.
|
||||
fs.Server.Cache.Set("disk_used", size, time.Second*60)
|
||||
|
||||
// Determine if their folder size, in bytes, is smaller than the amount of space they've
|
||||
// been allocated.
|
||||
fs.Server.Resources.Disk = size
|
||||
|
@ -146,42 +151,15 @@ func (fs *Filesystem) HasSpaceAvailable() bool {
|
|||
// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation
|
||||
// on locations with tons of files, so it is recommended that you cache the output.
|
||||
func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
|
||||
w := fs.NewWalker()
|
||||
ctx := context.Background()
|
||||
|
||||
var size int64
|
||||
var wg sync.WaitGroup
|
||||
|
||||
cleaned, err := fs.SafePath(dir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
files, err := ioutil.ReadDir(cleaned)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Iterate over all of the files and directories. If it is a file, immediately add its size
|
||||
// to the total size being returned. If we're dealing with a directory, call this function
|
||||
// on a seperate thread until we have gotten the size of everything nested within the given
|
||||
// directory.
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
wg.Add(1)
|
||||
|
||||
go func(p string) {
|
||||
defer wg.Done()
|
||||
|
||||
s, _ := fs.DirectorySize(p)
|
||||
|
||||
atomic.AddInt64(&size, s)
|
||||
}(filepath.Join(cleaned, f.Name()))
|
||||
} else {
|
||||
err := w.Walk(dir, ctx, func(f os.FileInfo) {
|
||||
atomic.AddInt64(&size, f.Size())
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return size, nil
|
||||
return size, err
|
||||
}
|
||||
|
||||
// Reads a file on the system and returns it as a byte representation in a file
|
||||
|
|
61
server/filesystem_walker.go
Normal file
61
server/filesystem_walker.go
Normal file
|
@ -0,0 +1,61 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type FileWalker struct {
|
||||
*Filesystem
|
||||
}
|
||||
|
||||
// Returns a new walker instance.
|
||||
func (fs *Filesystem) NewWalker() *FileWalker {
|
||||
return &FileWalker{fs}
|
||||
}
|
||||
|
||||
// Iterate over all of the files and directories within a given directory. When a file is
|
||||
// found the callback will be called with the file information. If a directory is encountered
|
||||
// it will be recursively passed back through to this function.
|
||||
func (fw *FileWalker) Walk(dir string, ctx context.Context, callback func (os.FileInfo)) error {
|
||||
cleaned, err := fw.SafePath(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get all of the files from this directory.
|
||||
files, err := ioutil.ReadDir(cleaned)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create an error group that we can use to run processes in parallel while retaining
|
||||
// the ability to cancel the entire process immediately should any of it fail.
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
p := filepath.Join(dir, f.Name())
|
||||
// Recursively call this function to continue digging through the directory tree within
|
||||
// a seperate goroutine. If the context is canceled abort this process.
|
||||
g.Go(func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
return fw.Walk(p, ctx, callback)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
// If this isn't a directory, go ahead and pass the file information into the
|
||||
// callback.
|
||||
callback(f)
|
||||
}
|
||||
}
|
||||
|
||||
// Block until all of the routines finish and have returned a value.
|
||||
return g.Wait()
|
||||
}
|
Loading…
Reference in New Issue
Block a user