2.5 minutes for lookups, not every minute

Fix log directory not being created

cmd/root.go

@@ -337,6 +337,10 @@ func Execute() error {
 // Configures the global logger for Zap so that we can call it from any location
 // in the code without having to pass around a logger instance.
 func configureLogging(logDir string, debug bool) error {
+	if err := os.MkdirAll(path.Join(logDir, "/install"), 0700); err != nil {
+		return errors.WithStack(err)
+	}
+
 	cfg := zap.NewProductionConfig()
 	if debug {
 		cfg = zap.NewDevelopmentConfig()

config/config_system.go

@@ -49,11 +49,6 @@ func (sc *SystemConfiguration) ConfigureDirectories() error {
 		return err
 	}
 
-	log.WithField("path", sc.LogDirectory).Debug("ensuring log directory exists")
-	if err := os.MkdirAll(path.Join(sc.LogDirectory, "/install"), 0700); err != nil {
-		return err
-	}
-
 	log.WithField("path", sc.Data).Debug("ensuring server data directory exists")
 	if err := os.MkdirAll(sc.Data, 0700); err != nil {
 		return err

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
 	github.com/icza/dyno v0.0.0-20200205103839-49cb13720835
 	github.com/imdario/mergo v0.3.8
+	github.com/karrick/godirwalk v1.16.1
 	github.com/klauspost/compress v1.10.10 // indirect
 	github.com/klauspost/pgzip v1.2.4
 	github.com/magefile/mage v1.10.0 // indirect

go.sum

@@ -317,6 +317,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

server/backup/archiver.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"github.com/apex/log"
 	gzip "github.com/klauspost/pgzip"
+	"github.com/pkg/errors"
 	"github.com/remeh/sizedwaitgroup"
 	"golang.org/x/sync/errgroup"
 	"io"
@@ -49,14 +50,8 @@ func (a *Archive) Create(dst string, ctx context.Context) (os.FileInfo, error) {
 	// Iterate over all of the files to be included and put them into the archive. This is
 	// done as a concurrent goroutine to speed things along. If an error is encountered at
 	// any step, the entire process is aborted.
-	for p, s := range a.Files.All() {
-		if (*s).IsDir() {
-			continue
-		}
-
-		pa := p
-		st := s
-
+	for _, p := range a.Files.All() {
+		p := p
 		g.Go(func() error {
 			wg.Add()
 			defer wg.Done()
@@ -65,7 +60,7 @@ func (a *Archive) Create(dst string, ctx context.Context) (os.FileInfo, error) {
 			case <-ctx.Done():
 				return ctx.Err()
 			default:
-				return a.addToArchive(pa, st, tw)
+				return a.addToArchive(p, tw)
 			}
 		})
 	}
@@ -92,21 +87,25 @@ func (a *Archive) Create(dst string, ctx context.Context) (os.FileInfo, error) {
 }
 
 // Adds a single file to the existing tar archive writer.
-func (a *Archive) addToArchive(p string, s *os.FileInfo, w *tar.Writer) error {
+func (a *Archive) addToArchive(p string, w *tar.Writer) error {
 	f, err := os.Open(p)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	st := *s
+	s, err := f.Stat()
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
 	header := &tar.Header{
 		// Trim the long server path from the name of the file so that the resulting
 		// archive is exactly how the user would see it in the panel file manager.
 		Name:    strings.TrimPrefix(p, a.TrimPrefix),
-		Size:    st.Size(),
-		Mode:    int64(st.Mode()),
-		ModTime: st.ModTime(),
+		Size:    s.Size(),
+		Mode:    int64(s.Mode()),
+		ModTime: s.ModTime(),
 	}
 
 	// These actions must occur sequentially, even if this function is called multiple

server/backup/included.go

@@ -1,29 +1,23 @@
 package backup
 
 import (
-	"os"
 	"sync"
 )
 
 type IncludedFiles struct {
 	sync.RWMutex
-	files map[string]*os.FileInfo
+	files []string
 }
 
 // Pushes an additional file or folder onto the struct.
-func (i *IncludedFiles) Push(info *os.FileInfo, p string) {
+func (i *IncludedFiles) Push(p string) {
 	i.Lock()
-	defer i.Unlock()
-
-	if i.files == nil {
-		i.files = make(map[string]*os.FileInfo)
-	}
-
-	i.files[p] = info
+	i.files = append(i.files, p) // ~~
+	i.Unlock()
 }
 
 // Returns all of the files that were marked as being included.
-func (i *IncludedFiles) All() map[string]*os.FileInfo {
+func (i *IncludedFiles) All() []string {
 	i.RLock()
 	defer i.RUnlock()
 

server/filesystem.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/gabriel-vasile/mimetype"
+	"github.com/karrick/godirwalk"
 	"github.com/pkg/errors"
 	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/server/backup"
@@ -21,6 +22,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 )
 
@@ -210,8 +212,6 @@ func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) {
 // Because determining the amount of space being used by a server is a taxing operation we
 // will load it all up into a cache and pull from that as long as the key is not expired.
 func (fs *Filesystem) HasSpaceAvailable() bool {
-	space := fs.Server.Build().DiskSpace
-
 	size, err := fs.getCachedDiskUsage()
 	if err != nil {
 		fs.Server.Log().WithField("error", err).Warn("failed to determine root server directory size")
@@ -221,6 +221,7 @@ func (fs *Filesystem) HasSpaceAvailable() bool {
 	// been allocated.
 	fs.Server.Proc().SetDisk(size)
 
+	space := fs.Server.Build().DiskSpace
 	// If space is -1 or 0 just return true, means they're allowed unlimited.
 	//
 	// Technically we could skip disk space calculation because we don't need to check if the server exceeds it's limit
@@ -247,7 +248,8 @@ func (fs *Filesystem) getCachedDiskUsage() (int64, error) {
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
 
-	if fs.lastLookupTime.After(time.Now().Add(time.Second * -60)) {
+	// Expire the cache after 2.5 minutes.
+	if fs.lastLookupTime.After(time.Now().Add(time.Second * -150)) {
 		return fs.diskUsage, nil
 	}
 
@@ -270,20 +272,40 @@ func (fs *Filesystem) getCachedDiskUsage() (int64, error) {
 // through all of the folders. Returns the size in bytes. This can be a fairly taxing operation
 // on locations with tons of files, so it is recommended that you cache the output.
 func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
-	var size int64
-	err := fs.Walk(dir, func(_ string, f os.FileInfo, err error) error {
+	d, err := fs.SafePath(dir)
 	if err != nil {
-			return fs.handleWalkerError(err, f)
+		return 0, errors.WithStack(err)
 	}
 
-		if !f.IsDir() {
-			atomic.AddInt64(&size, f.Size())
+	var size int64
+	var st syscall.Stat_t
+
+	err = godirwalk.Walk(d, &godirwalk.Options{
+		Unsorted: true,
+		Callback: func(p string, e *godirwalk.Dirent) error {
+			// If this is a symlink then resolve the final destination of it before trying to continue walking
+			// over its contents. If it resolves outside the server data directory just skip everything else for
+			// it. Otherwise, allow it to continue.
+			if e.IsSymlink() {
+				if _, err := fs.SafePath(p); err != nil {
+					if IsPathResolutionError(err) {
+						return godirwalk.SkipThis
+					}
+
+					return err
+				}
+			}
+
+			if !e.IsDir() {
+				syscall.Lstat(p, &st)
+				atomic.AddInt64(&size, st.Size)
 			}
 
 			return nil
+		},
 	})
 
-	return size, err
+	return size, errors.WithStack(err)
 }
 
 // Reads a file on the system and returns it as a byte representation in a file
@@ -485,19 +507,22 @@ func (fs *Filesystem) Chown(path string) error {
 
 	// If this was a directory, begin walking over its contents recursively and ensure that all
 	// of the subfiles and directories get their permissions updated as well.
-	return fs.Walk(cleaned, func(path string, f os.FileInfo, err error) error {
-		if err != nil {
-			return fs.handleWalkerError(err, f)
-		}
-
+	return godirwalk.Walk(cleaned, &godirwalk.Options{
+		Unsorted: true,
+		Callback: func(p string, e *godirwalk.Dirent) error {
 			// Do not attempt to chmod a symlink. Go's os.Chown function will affect the symlink
 			// so if it points to a location outside the data directory the user would be able to
 			// (un)intentionally modify that files permissions.
-		if f.Mode()&os.ModeSymlink != 0 {
+			if e.IsSymlink() {
+				if e.IsDir() {
+					return godirwalk.SkipThis
+				}
+
 				return nil
 			}
 
-		return os.Chown(path, uid, gid)
+			return os.Chown(p, uid, gid)
+		},
 	})
 }
 
@@ -574,7 +599,8 @@ func (fs *Filesystem) Copy(p string) error {
 	}
 	defer dest.Close()
 
-	if _, err := io.Copy(dest, source); err != nil {
+	buf := make([]byte, 1024*4)
+	if _, err := io.CopyBuffer(dest, source, buf); err != nil {
 		return errors.WithStack(err)
 	}
 
@@ -732,26 +758,35 @@ func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.In
 	// files found, and will keep walking deeper and deeper into directories.
 	inc := new(backup.IncludedFiles)
 
-	if err := fs.Walk(cleaned, func(p string, f os.FileInfo, err error) error {
+	err = godirwalk.Walk(cleaned, &godirwalk.Options{
+		Unsorted: true,
+		Callback: func(p string, e *godirwalk.Dirent) error {
+			sp := p
+			if e.IsSymlink() {
+				sp, err = fs.SafePath(p)
 				if err != nil {
-			return fs.handleWalkerError(err, f)
+					if IsPathResolutionError(err) {
+						return godirwalk.SkipThis
+					}
+
+					return err
+				}
 			}
 
 			// Avoid unnecessary parsing if there are no ignored files, nothing will match anyways
 			// so no reason to call the function.
-		if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(p, fs.Path()+"/")) {
-			inc.Push(&f, p)
+			if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(sp, fs.Path()+"/")) {
+				inc.Push(sp)
 			}
 
 			// We can't just abort if the path is technically ignored. It is possible there is a nested
 			// file or folder that should not be excluded, so in this case we need to just keep going
 			// until we get to a final state.
 			return nil
-	}); err != nil {
-		return nil, err
-	}
+		},
+	})
 
-	return inc, nil
+	return inc, errors.WithStack(err)
 }
 
 // Compresses all of the files matching the given paths in the specified directory. This function
@@ -788,24 +823,38 @@ func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, er
 			continue
 		}
 
-		if f.IsDir() {
-			err := fs.Walk(p, func(s string, info os.FileInfo, err error) error {
+		if !f.IsDir() {
+			inc.Push(p)
+		} else {
+			err := godirwalk.Walk(p, &godirwalk.Options{
+				Unsorted: true,
+				Callback: func(p string, e *godirwalk.Dirent) error {
+					sp := p
+					if e.IsSymlink() {
+						// Ensure that any symlinks are properly resolved to their final destination. If
+						// that destination is outside the server directory skip over this entire item, otherwise
+						// use the resolved location for the rest of this function.
+						sp, err = fs.SafePath(p)
 						if err != nil {
-					return fs.handleWalkerError(err, info)
+							if IsPathResolutionError(err) {
+								return godirwalk.SkipThis
 							}
 
-				if !info.IsDir() {
-					inc.Push(&info, s)
+							return err
+						}
+					}
+
+					if !e.IsDir() {
+						inc.Push(sp)
 					}
 
 					return nil
+				},
 			})
 
 			if err != nil {
 				return nil, err
 			}
-		} else {
-			inc.Push(&f, p)
 		}
 	}
 

server/filesystem_walker.go

@@ -1,141 +0,0 @@
-package server
-
-import (
-	"context"
-	"github.com/gammazero/workerpool"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sync"
-)
-
-type FileWalker struct {
-	*Filesystem
-}
-
-type PooledFileWalker struct {
-	wg       sync.WaitGroup
-	pool     *workerpool.WorkerPool
-	callback filepath.WalkFunc
-	cancel   context.CancelFunc
-
-	err     error
-	errOnce sync.Once
-
-	Filesystem *Filesystem
-}
-
-// Returns a new walker instance.
-func (fs *Filesystem) NewWalker() *FileWalker {
-	return &FileWalker{fs}
-}
-
-// Creates a new pooled file walker that will concurrently walk over a given directory but limit itself
-// to a worker pool as to not completely flood out the system or cause a process crash.
-func newPooledWalker(fs *Filesystem) *PooledFileWalker {
-	return &PooledFileWalker{
-		Filesystem: fs,
-		// Create a worker pool that is the same size as the number of processors available on the
-		// system. Going much higher doesn't provide much of a performance boost, and is only more
-		// likely to lead to resource overloading anyways.
-		pool: workerpool.New(runtime.NumCPU()),
-	}
-}
-
-// Process a given path by calling the callback function for all of the files and directories within
-// the path, and then dropping into any directories that we come across.
-func (w *PooledFileWalker) process(path string) error {
-	p, err := w.Filesystem.SafePath(path)
-	if err != nil {
-		return err
-	}
-
-	files, err := ioutil.ReadDir(p)
-	if err != nil {
-		return err
-	}
-
-	// Loop over all of the files and directories in the given directory and call the provided
-	// callback function. If we encounter a directory, push that directory onto the worker queue
-	// to be processed.
-	for _, f := range files {
-		sp, err := w.Filesystem.SafeJoin(p, f)
-		if err != nil {
-			// Let the callback function handle what to do if there is a path resolution error because a
-			// dangerous path was resolved. If there is an error returned, return from this entire process
-			// otherwise just skip over this specific file. We don't care if its a file or a directory at
-			// this point since either way we're skipping it, however, still check for the SkipDir since that
-			// would be thrown otherwise.
-			if err = w.callback(sp, f, err); err != nil && err != filepath.SkipDir {
-				return err
-			}
-
-			continue
-		}
-
-		i, err := os.Stat(sp)
-		// You might end up getting an error about a file or folder not existing if the given path
-		// if it is an invalid symlink. We can safely just skip over these files I believe.
-		if os.IsNotExist(err) {
-			continue
-		}
-
-		// Call the user-provided callback for this file or directory. If an error is returned that is
-		// not a SkipDir call, abort the entire process and bubble that error up.
-		if err = w.callback(sp, i, err); err != nil && err != filepath.SkipDir {
-			return err
-		}
-
-		// If this is a directory, and we didn't get a SkipDir error, continue through by pushing another
-		// job to the pool to handle it. If we requested a skip, don't do anything just continue on to the
-		// next item.
-		if i.IsDir() && err != filepath.SkipDir {
-			w.push(sp)
-		} else if !i.IsDir() && err == filepath.SkipDir {
-			// Per the spec for the callback, if we get a SkipDir error but it is returned for an item
-			// that is _not_ a directory, abort the remaining operations on the directory.
-			return nil
-		}
-	}
-
-	return nil
-}
-
-// Push a new path into the worker pool and increment the waitgroup so that we do not return too
-// early and cause panic's as internal directories attempt to submit to the pool.
-func (w *PooledFileWalker) push(path string) {
-	w.wg.Add(1)
-	w.pool.Submit(func() {
-		defer w.wg.Done()
-		if err := w.process(path); err != nil {
-			w.errOnce.Do(func() {
-				w.err = err
-				if w.cancel != nil {
-					w.cancel()
-				}
-			})
-		}
-	})
-}
-
-// Walks the given directory and executes the callback function for all of the files and directories
-// that are encountered.
-func (fs *Filesystem) Walk(dir string, callback filepath.WalkFunc) error {
-	w := newPooledWalker(fs)
-	w.callback = callback
-
-	_, cancel := context.WithCancel(context.Background())
-	w.cancel = cancel
-
-	w.push(dir)
-
-	w.wg.Wait()
-	w.pool.StopWait()
-
-	if w.err != nil {
-		return w.err
-	}
-
-	return nil
-}