Add FS logic for determining folder size as well as safe path resolution
This commit is contained in:
parent
94223bafec
commit
314a5ad546
1
go.mod
1
go.mod
|
@ -21,6 +21,7 @@ require (
|
||||||
github.com/onsi/gomega v1.5.0 // indirect
|
github.com/onsi/gomega v1.5.0 // indirect
|
||||||
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
|
||||||
github.com/opencontainers/image-spec v1.0.1 // indirect
|
github.com/opencontainers/image-spec v1.0.1 // indirect
|
||||||
|
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||||
github.com/pkg/errors v0.8.0
|
github.com/pkg/errors v0.8.0
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
|
github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -44,6 +44,8 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i
|
||||||
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
|
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
|
||||||
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
|
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
|
||||||
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
|
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
|
||||||
|
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||||
|
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
|
|
@ -1,6 +1,18 @@
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import "path"
|
import (
|
||||||
|
"errors"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Error returned when there is a bad path provided to one of the FS calls.
|
||||||
|
var InvalidPathResolution = errors.New("invalid path resolution")
|
||||||
|
|
||||||
type Filesystem struct {
|
type Filesystem struct {
|
||||||
// The root directory where all of the server data is contained. By default
|
// The root directory where all of the server data is contained. By default
|
||||||
|
@ -13,10 +25,146 @@ type Filesystem struct {
|
||||||
|
|
||||||
// Returns the root path that contains all of a server's data.
|
// Returns the root path that contains all of a server's data.
|
||||||
func (fs *Filesystem) Path() string {
|
func (fs *Filesystem) Path() string {
|
||||||
return path.Join(fs.Root, fs.Server.Uuid)
|
return filepath.Join(fs.Root, fs.Server.Uuid)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns a safe path for a server object.
|
// Normalizes a directory being passed in to ensure the user is not able to escape
|
||||||
func (fs *Filesystem) SafePath(p string) string {
|
// from their data directory. After normalization if the directory is still within their home
|
||||||
return fs.Path()
|
// path it is returned. If they managed to "escape" an error will be returned.
|
||||||
|
//
|
||||||
|
// This logic is actually copied over from the SFTP server code. Ideally that eventually
|
||||||
|
// either gets ported into this application, or is able to make use of this package.
|
||||||
|
func (fs *Filesystem) SafePath(p string) (string, error) {
|
||||||
|
var nonExistentPathResolution string
|
||||||
|
|
||||||
|
// Calling filpath.Clean on the joined directory will resolve it to the absolute path,
|
||||||
|
// removing any ../ type of resolution arguments, and leaving us with a direct path link.
|
||||||
|
r := filepath.Clean(filepath.Join(fs.Path(), p))
|
||||||
|
|
||||||
|
// At the same time, evaluate the symlink status and determine where this file or folder
|
||||||
|
// is truly pointing to.
|
||||||
|
p, err := filepath.EvalSymlinks(r)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", err
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
// The requested directory doesn't exist, so at this point we need to iterate up the
|
||||||
|
// path chain until we hit a directory that _does_ exist and can be validated.
|
||||||
|
parts := strings.Split(filepath.Dir(r), "/")
|
||||||
|
|
||||||
|
var try string
|
||||||
|
// Range over all of the path parts and form directory pathings from the end
|
||||||
|
// moving up until we have a valid resolution or we run out of paths to try.
|
||||||
|
for k := range parts {
|
||||||
|
try = strings.Join(parts[:(len(parts) - k)], "/")
|
||||||
|
|
||||||
|
if !strings.HasPrefix(try, fs.Path()) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
t, err := filepath.EvalSymlinks(try)
|
||||||
|
if err == nil {
|
||||||
|
nonExistentPathResolution = t
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the new path doesn't start with their root directory there is clearly an escape
|
||||||
|
// attempt going on, and we should NOT resolve this path for them.
|
||||||
|
if nonExistentPathResolution != "" {
|
||||||
|
if !strings.HasPrefix(nonExistentPathResolution, fs.Path()) {
|
||||||
|
return "", InvalidPathResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the nonExistentPathResoltion variable is not empty then the initial path requested
|
||||||
|
// did not exist and we looped through the pathway until we found a match. At this point
|
||||||
|
// we've confirmed the first matched pathway exists in the root server directory, so we
|
||||||
|
// can go ahead and just return the path that was requested initially.
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the requested directory from EvalSymlinks begins with the server root directory go
|
||||||
|
// ahead and return it. If not we'll return an error which will block any further action
|
||||||
|
// on the file.
|
||||||
|
if strings.HasPrefix(p, fs.Path()) {
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", InvalidPathResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determines if the directory a file is trying to be added to has enough space available
|
||||||
|
// for the file to be written to.
|
||||||
|
//
|
||||||
|
// Because determining the amount of space being used by a server is a taxing operation we
|
||||||
|
// will load it all up into a cache and pull from that as long as the key is not expired.
|
||||||
|
func (fs *Filesystem) HasSpaceAvailable() bool {
|
||||||
|
var space = fs.Server.Build.DiskSpace
|
||||||
|
|
||||||
|
// If space is -1 or 0 just return true, means they're allowed unlimited.
|
||||||
|
if space <= 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
var size int64
|
||||||
|
if x, exists := fs.Server.Cache().Get("disk_used"); exists {
|
||||||
|
size = x.(int64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If there is no size its either because there is no data (in which case running this function
|
||||||
|
// will have effectively no impact), or there is nothing in the cache, in which case we need to
|
||||||
|
// grab the size of their data directory. This is a taxing operation, so we want to store it in
|
||||||
|
// the cache once we've gotten it.
|
||||||
|
if size == 0 {
|
||||||
|
if size, err := fs.DirectorySize("/"); err != nil {
|
||||||
|
zap.S().Warnw("failed to determine directory size", zap.String("server", fs.Server.Uuid), zap.Error(err))
|
||||||
|
} else {
|
||||||
|
fs.Server.Cache().Set("disk_used", size, time.Minute * 5)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine if their folder size, in bytes, is smaller than the amount of space they've
|
||||||
|
// been allocated.
|
||||||
|
return (size / 1024.0 / 1024.0) <= space
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determines the directory size of a given location by running parallel tasks to iterate
|
||||||
|
// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation
|
||||||
|
// on locations with tons of files, so it is recommended that you cache the output.
|
||||||
|
func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
|
||||||
|
var size int64
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
cleaned, err := fs.SafePath(dir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := ioutil.ReadDir(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iterate over all of the files and directories. If it is a file, immediately add its size
|
||||||
|
// to the total size being returned. If we're dealing with a directory, call this function
|
||||||
|
// on a seperate thread until we have gotten the size of everything nested within the given
|
||||||
|
// directory.
|
||||||
|
for _, f := range files {
|
||||||
|
if f.IsDir() {
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func(p string) {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
s, _ := fs.DirectorySize(p)
|
||||||
|
size += s
|
||||||
|
}(filepath.Join(cleaned, f.Name()))
|
||||||
|
} else {
|
||||||
|
size += f.Size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
return size, nil
|
||||||
}
|
}
|
|
@ -1,6 +1,7 @@
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"github.com/patrickmn/go-cache"
|
||||||
"github.com/pterodactyl/wings/config"
|
"github.com/pterodactyl/wings/config"
|
||||||
"github.com/remeh/sizedwaitgroup"
|
"github.com/remeh/sizedwaitgroup"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
|
@ -9,6 +10,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// High level definition for a server instance being controlled by Wings.
|
// High level definition for a server instance being controlled by Wings.
|
||||||
|
@ -43,6 +45,10 @@ type Server struct {
|
||||||
environment Environment
|
environment Environment
|
||||||
|
|
||||||
fs *Filesystem
|
fs *Filesystem
|
||||||
|
|
||||||
|
// Server cache used to store frequently requested information in memory and make
|
||||||
|
// certain long operations return faster. For example, FS disk space usage.
|
||||||
|
cache *cache.Cache
|
||||||
}
|
}
|
||||||
|
|
||||||
// The build settings for a given server that impact docker container creation and
|
// The build settings for a given server that impact docker container creation and
|
||||||
|
@ -187,7 +193,7 @@ func FromConfiguration(data []byte, cfg *config.SystemConfiguration) (*Server, e
|
||||||
}
|
}
|
||||||
|
|
||||||
s.environment = env
|
s.environment = env
|
||||||
|
s.cache = cache.New(time.Minute * 10, time.Minute * 15)
|
||||||
s.fs = &Filesystem{
|
s.fs = &Filesystem{
|
||||||
Root: cfg.Data,
|
Root: cfg.Data,
|
||||||
Server: s,
|
Server: s,
|
||||||
|
@ -209,6 +215,10 @@ func (s *Server) Filesystem() *Filesystem {
|
||||||
return s.fs
|
return s.fs
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) Cache() *cache.Cache {
|
||||||
|
return s.cache
|
||||||
|
}
|
||||||
|
|
||||||
// Determine if the server is bootable in it's current state or not. This will not
|
// Determine if the server is bootable in it's current state or not. This will not
|
||||||
// indicate why a server is not bootable, only if it is.
|
// indicate why a server is not bootable, only if it is.
|
||||||
func (s *Server) IsBootable() bool {
|
func (s *Server) IsBootable() bool {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user