Change filewalker implementation to use a pool

This commit is contained in:
Dane Everitt 2020-07-16 21:51:31 -07:00
parent 7e1b7e7f36
commit f3c8220bd9
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
5 changed files with 102 additions and 81 deletions

1
go.mod
View File

@ -31,6 +31,7 @@ require (
github.com/docker/go-units v0.3.3 // indirect github.com/docker/go-units v0.3.3 // indirect
github.com/fatih/color v1.9.0 github.com/fatih/color v1.9.0
github.com/gabriel-vasile/mimetype v0.1.4 github.com/gabriel-vasile/mimetype v0.1.4
github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753
github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0 github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0
github.com/gin-gonic/gin v1.6.3 github.com/gin-gonic/gin v1.6.3
github.com/golang/protobuf v1.3.5 // indirect github.com/golang/protobuf v1.3.5 // indirect

4
go.sum
View File

@ -79,6 +79,10 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gabriel-vasile/mimetype v0.1.4 h1:5mcsq3+DXypREUkW+1juhjeKmE/XnWgs+paHMJn7lf8= github.com/gabriel-vasile/mimetype v0.1.4 h1:5mcsq3+DXypREUkW+1juhjeKmE/XnWgs+paHMJn7lf8=
github.com/gabriel-vasile/mimetype v0.1.4/go.mod h1:kMJbg3SlWZCsj4R73F1WDzbT9AyGCOVmUtIxxwO5pmI= github.com/gabriel-vasile/mimetype v0.1.4/go.mod h1:kMJbg3SlWZCsj4R73F1WDzbT9AyGCOVmUtIxxwO5pmI=
github.com/gammazero/deque v0.0.0-20200227231300-1e9af0e52b46 h1:iX4+rD9Fjdx8SkmSO/O5WAIX/j79ll3kuqv5VdYt9J8=
github.com/gammazero/deque v0.0.0-20200227231300-1e9af0e52b46/go.mod h1:D90+MBHVc9Sk1lJAbEVgws0eYEurY4mv2TDso3Nxh3w=
github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753 h1:oSQ61LxZkz3Z4La0O5cbyVDvLWEfbNgiD43cSPdjPQQ=
github.com/gammazero/workerpool v0.0.0-20200608033439-1a5ca90a5753/go.mod h1:/XWO2YAUUpPi3smDlFBl0vpX0JHwUomDM/oRMwRmnSs=
github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0 h1:7KeiSrO5puFH1+vdAdbpiie2TrNnkvFc/eOQzT60Z2k= github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0 h1:7KeiSrO5puFH1+vdAdbpiie2TrNnkvFc/eOQzT60Z2k=
github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0/go.mod h1:D1+3UtCYAJ1os1PI+zhTVEj6Tb+IHJvXjXKz83OstmM= github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0/go.mod h1:D1+3UtCYAJ1os1PI+zhTVEj6Tb+IHJvXjXKz83OstmM=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=

View File

@ -33,7 +33,7 @@ func TrackedError(err error) *RequestError {
// generated this server for the purposes of logging. // generated this server for the purposes of logging.
func TrackedServerError(err error, s *server.Server) *RequestError { func TrackedServerError(err error, s *server.Server) *RequestError {
return &RequestError{ return &RequestError{
Err: err, Err: errors.WithStack(err),
Uuid: uuid.Must(uuid.NewRandom()).String(), Uuid: uuid.Must(uuid.NewRandom()).String(),
Message: "", Message: "",
server: s, server: s,

View File

@ -29,9 +29,9 @@ import (
var InvalidPathResolution = errors.New("invalid path resolution") var InvalidPathResolution = errors.New("invalid path resolution")
type Filesystem struct { type Filesystem struct {
Server *Server Server *Server
Configuration *config.SystemConfiguration Configuration *config.SystemConfiguration
cacheDiskMu sync.Mutex cacheDiskMu sync.Mutex
} }
// Returns the root path that contains all of a server's data. // Returns the root path that contains all of a server's data.
@ -222,17 +222,13 @@ func (fs *Filesystem) getCachedDiskUsage() (int64, error) {
// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation // through all of the folders. Returns the size in bytes. This can be a fairly taxing operation
// on locations with tons of files, so it is recommended that you cache the output. // on locations with tons of files, so it is recommended that you cache the output.
func (fs *Filesystem) DirectorySize(dir string) (int64, error) { func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
w := fs.NewWalker()
ctx := context.Background()
var size int64 var size int64
err := w.Walk(dir, ctx, func(f os.FileInfo, _ string) bool { err := fs.Walk(dir, func(_ string, f os.FileInfo, err error) error {
// Only increment the size when we're dealing with a file specifically, otherwise
// just continue digging deeper until there are no more directories to iterate over.
if !f.IsDir() { if !f.IsDir() {
atomic.AddInt64(&size, f.Size()) atomic.AddInt64(&size, f.Size())
} }
return true
return nil
}) })
return size, err return size, err
@ -657,9 +653,6 @@ func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.In
return nil, err return nil, err
} }
w := fs.NewWalker()
ctx := context.Background()
i, err := ignore.CompileIgnoreLines(ignored...) i, err := ignore.CompileIgnoreLines(ignored...)
if err != nil { if err != nil {
return nil, err return nil, err
@ -668,7 +661,8 @@ func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.In
// Walk through all of the files and directories on a server. This callback only returns // Walk through all of the files and directories on a server. This callback only returns
// files found, and will keep walking deeper and deeper into directories. // files found, and will keep walking deeper and deeper into directories.
inc := new(backup.IncludedFiles) inc := new(backup.IncludedFiles)
if err := w.Walk(cleaned, ctx, func(f os.FileInfo, p string) bool {
if err := fs.Walk(cleaned, func(p string, f os.FileInfo, err error) error {
// Avoid unnecessary parsing if there are no ignored files, nothing will match anyways // Avoid unnecessary parsing if there are no ignored files, nothing will match anyways
// so no reason to call the function. // so no reason to call the function.
if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(p, fs.Path()+"/")) { if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(p, fs.Path()+"/")) {
@ -678,7 +672,7 @@ func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.In
// We can't just abort if the path is technically ignored. It is possible there is a nested // We can't just abort if the path is technically ignored. It is possible there is a nested
// file or folder that should not be excluded, so in this case we need to just keep going // file or folder that should not be excluded, so in this case we need to just keep going
// until we get to a final state. // until we get to a final state.
return true return nil
}); err != nil { }); err != nil {
return nil, err return nil, err
} }
@ -709,43 +703,34 @@ func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, er
return nil, err return nil, err
} }
w := fs.NewWalker()
wg := new(sync.WaitGroup)
inc := new(backup.IncludedFiles) inc := new(backup.IncludedFiles)
// Iterate over all of the cleaned paths and merge them into a large object of final file // Iterate over all of the cleaned paths and merge them into a large object of final file
// paths to pass into the archiver. As directories are encountered this will drop into them // paths to pass into the archiver. As directories are encountered this will drop into them
// and look for all of the files. // and look for all of the files.
for _, p := range cleaned { for _, p := range cleaned {
wg.Add(1) f, err := os.Stat(p)
if err != nil {
fs.Server.Log().WithField("error", err).WithField("path", p).Debug("failed to stat file or directory for compression")
continue
}
go func(pa string) { if f.IsDir() {
defer wg.Done() err := fs.Walk(p, func(s string, info os.FileInfo, err error) error {
if !info.IsDir() {
inc.Push(&info, s)
}
return nil
})
f, err := os.Stat(pa)
if err != nil { if err != nil {
fs.Server.Log().WithField("error", err).WithField("path", pa).Warn("failed to stat file or directory for compression") return nil, err
return
} }
} else {
if f.IsDir() { inc.Push(&f, p)
// Recursively drop into directory and get all of the additional files and directories within }
// it that should be included in this backup.
w.Walk(pa, context.Background(), func(info os.FileInfo, s string) bool {
if !info.IsDir() {
inc.Push(&info, s)
}
return true
})
} else {
inc.Push(&f, pa)
}
}(p)
} }
wg.Wait()
a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc} a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc}
d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", ""))) d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")))

View File

@ -1,70 +1,101 @@
package server package server
import ( import (
"context" "github.com/gammazero/workerpool"
"golang.org/x/sync/errgroup"
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"sync"
) )
type FileWalker struct { type FileWalker struct {
*Filesystem *Filesystem
} }
type PooledFileWalker struct {
wg sync.WaitGroup
pool *workerpool.WorkerPool
callback filepath.WalkFunc
Filesystem *Filesystem
}
// Returns a new walker instance. // Returns a new walker instance.
func (fs *Filesystem) NewWalker() *FileWalker { func (fs *Filesystem) NewWalker() *FileWalker {
return &FileWalker{fs} return &FileWalker{fs}
} }
// Iterate over all of the files and directories within a given directory. When a file is // Creates a new pooled file walker that will concurrently walk over a given directory but limit itself
// found the callback will be called with the file information. If a directory is encountered // to a worker pool as to not completely flood out the system or cause a process crash.
// it will be recursively passed back through to this function. func newPooledWalker(fs *Filesystem) *PooledFileWalker {
func (fw *FileWalker) Walk(dir string, ctx context.Context, callback func (os.FileInfo, string) bool) error { return &PooledFileWalker{
cleaned, err := fw.SafePath(dir) Filesystem: fs,
// Create a worker pool that is the same size as the number of processors available on the
// system. Going much higher doesn't provide much of a performance boost, and is only more
// likely to lead to resource overloading anyways.
pool: workerpool.New(runtime.GOMAXPROCS(0)),
}
}
// Process a given path by calling the callback function for all of the files and directories within
// the path, and then dropping into any directories that we come across.
func (w *PooledFileWalker) process(path string) error {
defer w.wg.Done()
p, err := w.Filesystem.SafePath(path)
if err != nil { if err != nil {
return err return err
} }
// Get all of the files from this directory. files, err := ioutil.ReadDir(p)
files, err := ioutil.ReadDir(cleaned)
if err != nil { if err != nil {
return err return err
} }
// Create an error group that we can use to run processes in parallel while retaining // Loop over all of the files and directories in the given directory and call the provided
// the ability to cancel the entire process immediately should any of it fail. // callback function. If we encounter a directory, push that directory onto the worker queue
g, ctx := errgroup.WithContext(ctx) // to be processed.
for _, f := range files { for _, f := range files {
if f.IsDir() { sp := filepath.Join(p, f.Name())
fi := f i, err := os.Stat(sp)
p := filepath.Join(cleaned, f.Name())
// Recursively call this function to continue digging through the directory tree within
// a seperate goroutine. If the context is canceled abort this process.
g.Go(func() error {
select {
case <-ctx.Done():
return ctx.Err()
default:
// If the callback returns true, go ahead and keep walking deeper. This allows
// us to programatically continue deeper into directories, or stop digging
// if that pathway knows it needs nothing else.
if callback(fi, p) {
return fw.Walk(p, ctx, callback)
}
return nil if err = w.callback(sp, i, err); err != nil {
} if err == filepath.SkipDir {
}) return nil
} else { }
// If this isn't a directory, go ahead and pass the file information into the
// callback. We don't care about the response since we won't be stepping into return err
// anything from here. }
callback(f, filepath.Join(cleaned, f.Name()))
if i.IsDir() {
w.push(sp)
} }
} }
// Block until all of the routines finish and have returned a value. return nil
return g.Wait() }
// Push a new path into the worker pool.
//
// @todo probably helps to handle errors.
func (w *PooledFileWalker) push(path string) {
w.wg.Add(1)
w.pool.Submit(func() {
w.process(path)
})
}
// Walks the given directory and executes the callback function for all of the files and directories
// that are encountered.
func (fs *Filesystem) Walk(dir string, callback filepath.WalkFunc) error {
w := newPooledWalker(fs)
w.callback = callback
w.push(dir)
w.wg.Wait()
w.pool.StopWait()
return nil
} }