diff --git a/router/error.go b/router/error.go index 654407a..fdd90ea 100644 --- a/router/error.go +++ b/router/error.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" "github.com/pterodactyl/wings/server" + "github.com/pterodactyl/wings/server/filesystem" "net/http" "os" "strings" @@ -105,9 +106,9 @@ func (e *RequestError) AbortFilesystemError(c *gin.Context) { return } - if errors.Is(e.Err, server.ErrNotEnoughDiskSpace) { + if errors.Is(e.Err, filesystem.ErrNotEnoughDiskSpace) { c.AbortWithStatusJSON(http.StatusConflict, gin.H{ - "error": server.ErrNotEnoughDiskSpace.Error(), + "error": "There is not enough disk space available to perform that action.", }) return } @@ -126,6 +127,13 @@ func (e *RequestError) AbortFilesystemError(c *gin.Context) { return } + if strings.HasSuffix(e.Err.Error(), "file name too long") { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "Cannot perform that action: file name is too long.", + }) + return + } + e.AbortWithServerError(c) } diff --git a/router/router_download.go b/router/router_download.go index acb3ce9..ae5ff6e 100644 --- a/router/router_download.go +++ b/router/router_download.go @@ -70,7 +70,7 @@ func getDownloadFile(c *gin.Context) { return } - p, _ := s.Filesystem.SafePath(token.FilePath) + p, _ := s.Filesystem().SafePath(token.FilePath) st, err := os.Stat(p) // If there is an error or we're somehow trying to download a directory, just // respond with the appropriate error. diff --git a/router/router_server.go b/router/router_server.go index 0ed370a..faaad90 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -228,7 +228,7 @@ func deleteServer(c *gin.Context) { "error": errors.WithStack(err), }).Warn("failed to remove server files during deletion process") } - }(s.Filesystem.Path()) + }(s.Filesystem().Path()) var uuid = s.Id() server.GetServers().Remove(func(s2 *server.Server) bool { diff --git a/router/router_server_files.go b/router/router_server_files.go index 341ba2e..2027dc6 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/router/tokens" "github.com/pterodactyl/wings/server" + "github.com/pterodactyl/wings/server/filesystem" "golang.org/x/sync/errgroup" "mime/multipart" "net/http" @@ -29,7 +30,7 @@ func getServerFileContents(c *gin.Context) { } p = "/" + strings.TrimLeft(p, "/") - cleaned, err := s.Filesystem.SafePath(p) + cleaned, err := s.Filesystem().SafePath(p) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The file requested could not be found.", @@ -37,7 +38,7 @@ func getServerFileContents(c *gin.Context) { return } - st, err := s.Filesystem.Stat(cleaned) + st, err := s.Filesystem().Stat(cleaned) if err != nil { TrackedServerError(err, s).AbortWithServerError(c) return @@ -80,7 +81,7 @@ func getServerListDirectory(c *gin.Context) { return } - stats, err := s.Filesystem.ListDirectory(d) + stats, err := s.Filesystem().ListDirectory(d) if err != nil { TrackedServerError(err, s).AbortFilesystemError(c) return @@ -126,7 +127,7 @@ func putServerRenameFiles(c *gin.Context) { case <-ctx.Done(): return ctx.Err() default: - if err := s.Filesystem.Rename(pf, pt); err != nil { + if err := s.Filesystem().Rename(pf, pt); err != nil { // Return nil if the error is an is not exists. // NOTE: os.IsNotExist() does not work if the error is wrapped. if errors.Is(err, os.ErrNotExist) { @@ -168,7 +169,7 @@ func postServerCopyFile(c *gin.Context) { return } - if err := s.Filesystem.Copy(data.Location); err != nil { + if err := s.Filesystem().Copy(data.Location); err != nil { TrackedServerError(err, s).AbortFilesystemError(c) return } @@ -208,7 +209,7 @@ func postServerDeleteFiles(c *gin.Context) { case <-ctx.Done(): return ctx.Err() default: - return s.Filesystem.Delete(pi) + return s.Filesystem().Delete(pi) } }) } @@ -232,28 +233,15 @@ func postServerWriteFile(c *gin.Context) { } f = "/" + strings.TrimLeft(f, "/") - // Check if there is enough space available to perform this action. - if err := s.Filesystem.HasSpaceFor(c.Request.ContentLength); err != nil { - TrackedServerError(err, s).AbortFilesystemError(c) - return - } - - if err := s.Filesystem.Writefile(f, c.Request.Body); err != nil { - if errors.Is(err, server.ErrIsDirectory) { + if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil { + if errors.Is(err, filesystem.ErrIsDirectory) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Cannot write file, name conflicts with an existing directory by the same name.", }) return } - if strings.HasSuffix(err.Error(), "file name too long") { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "Cannot move or rename file, name is too long.", - }) - return - } - - TrackedServerError(err, s).AbortWithServerError(c) + TrackedServerError(err, s).AbortFilesystemError(c) return } @@ -273,7 +261,7 @@ func postServerCreateDirectory(c *gin.Context) { return } - if err := s.Filesystem.CreateDirectory(data.Name, data.Path); err != nil { + if err := s.Filesystem().CreateDirectory(data.Name, data.Path); err != nil { if err.Error() == "not a directory" { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Part of the path being created is not a directory (ENOTDIR).", @@ -307,20 +295,20 @@ func postServerCompressFiles(c *gin.Context) { return } - if !s.Filesystem.HasSpaceAvailable(true) { + if !s.Filesystem().HasSpaceAvailable(true) { c.AbortWithStatusJSON(http.StatusConflict, gin.H{ "error": "This server does not have enough available disk space to generate a compressed archive.", }) return } - f, err := s.Filesystem.CompressFiles(data.RootPath, data.Files) + f, err := s.Filesystem().CompressFiles(data.RootPath, data.Files) if err != nil { TrackedServerError(err, s).AbortFilesystemError(c) return } - c.JSON(http.StatusOK, &server.Stat{ + c.JSON(http.StatusOK, &filesystem.Stat{ Info: f, Mimetype: "application/tar+gzip", }) @@ -338,10 +326,10 @@ func postServerDecompressFiles(c *gin.Context) { return } - hasSpace, err := s.Filesystem.SpaceAvailableForDecompression(data.RootPath, data.File) + hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File) if err != nil { // Handle an unknown format error. - if errors.Is(err, server.ErrUnknownArchiveFormat) { + if errors.Is(err, filesystem.ErrUnknownArchiveFormat) { s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ @@ -361,7 +349,7 @@ func postServerDecompressFiles(c *gin.Context) { return } - if err := s.Filesystem.DecompressFile(data.RootPath, data.File); err != nil { + if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil { if errors.Is(err, os.ErrNotExist) { c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The requested archive was not found.", @@ -426,13 +414,8 @@ func postServerUploadFiles(c *gin.Context) { totalSize += header.Size } - if err := s.Filesystem.HasSpaceFor(totalSize); err != nil { - TrackedServerError(err, s).AbortFilesystemError(c) - return - } - for _, header := range headers { - p, err := s.Filesystem.SafePath(filepath.Join(directory, header.Filename)) + p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename)) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -454,7 +437,7 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) } defer file.Close() - if err := s.Filesystem.Writefile(p, file); err != nil { + if err := s.Filesystem().Writefile(p, file); err != nil { return errors.WithStack(err) } diff --git a/router/router_transfer.go b/router/router_transfer.go index b36a87c..d5d7760 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -67,7 +67,7 @@ func getServerArchive(c *gin.Context) { return } - file, err := os.Open(s.Archiver.ArchivePath()) + file, err := os.Open(s.Archiver.Path()) if err != nil { tserr := TrackedServerError(err, s) if !os.IsNotExist(err) { @@ -84,7 +84,7 @@ func getServerArchive(c *gin.Context) { c.Header("X-Checksum", checksum) c.Header("X-Mime-Type", st.Mimetype) c.Header("Content-Length", strconv.Itoa(int(st.Info.Size()))) - c.Header("Content-Disposition", "attachment; filename="+s.Archiver.ArchiveName()) + c.Header("Content-Disposition", "attachment; filename="+s.Archiver.Name()) c.Header("Content-Type", "application/octet-stream") bufio.NewReader(file).WriteTo(c.Writer) @@ -283,7 +283,7 @@ func postTransfer(c *gin.Context) { } // Un-archive the archive. That sounds weird.. - if err := archiver.NewTarGz().Unarchive(archivePath, i.Server().Filesystem.Path()); err != nil { + if err := archiver.NewTarGz().Unarchive(archivePath, i.Server().Filesystem().Path()); err != nil { l.WithField("error", errors.WithStack(err)).Error("failed to extract server archive") return } diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index 45ddf9f..9157ffc 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -14,6 +14,7 @@ import ( "github.com/pterodactyl/wings/environment/docker" "github.com/pterodactyl/wings/router/tokens" "github.com/pterodactyl/wings/server" + "github.com/pterodactyl/wings/server/filesystem" "net/http" "strings" "sync" @@ -193,7 +194,7 @@ func (h *Handler) SendErrorJson(msg Message, err error, shouldLog ...bool) error j := h.GetJwt() expected := errors.Is(err, server.ErrSuspended) || errors.Is(err, server.ErrIsRunning) || - errors.Is(err, server.ErrNotEnoughDiskSpace) + errors.Is(err, filesystem.ErrNotEnoughDiskSpace) message := "an unexpected error was encountered while handling this request" if expected || (j != nil && j.HasPermission(PermissionReceiveErrors)) { @@ -300,7 +301,7 @@ func (h *Handler) HandleInbound(m Message) error { // Only send the current disk usage if the server is offline, if docker container is running, // Environment#EnableResourcePolling() will send this data to all clients. if state == environment.ProcessOfflineState { - _ = h.server.Filesystem.HasSpaceAvailable(false) + _ = h.server.Filesystem().HasSpaceAvailable(false) b, _ := json.Marshal(h.server.Proc()) h.SendJson(&Message{ diff --git a/server/archiver.go b/server/archiver.go index 360dc19..9626b4b 100644 --- a/server/archiver.go +++ b/server/archiver.go @@ -4,7 +4,9 @@ import ( "crypto/sha256" "encoding/hex" "github.com/mholt/archiver/v3" + "github.com/pkg/errors" "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/server/filesystem" "io" "io/ioutil" "os" @@ -16,19 +18,19 @@ type Archiver struct { Server *Server } -// ArchivePath returns the path to the server's archive. -func (a *Archiver) ArchivePath() string { - return filepath.Join(config.Get().System.ArchiveDirectory, a.ArchiveName()) +// Path returns the path to the server's archive. +func (a *Archiver) Path() string { + return filepath.Join(config.Get().System.ArchiveDirectory, a.Name()) } -// ArchiveName returns the name of the server's archive. -func (a *Archiver) ArchiveName() string { +// Name returns the name of the server's archive. +func (a *Archiver) Name() string { return a.Server.Id() + ".tar.gz" } // Exists returns a boolean based off if the archive exists. func (a *Archiver) Exists() bool { - if _, err := os.Stat(a.ArchivePath()); os.IsNotExist(err) { + if _, err := os.Stat(a.Path()); os.IsNotExist(err) { return false } @@ -36,13 +38,21 @@ func (a *Archiver) Exists() bool { } // Stat stats the archive file. -func (a *Archiver) Stat() (*Stat, error) { - return a.Server.Filesystem.unsafeStat(a.ArchivePath()) +func (a *Archiver) Stat() (*filesystem.Stat, error) { + s, err := os.Stat(a.Path()) + if err != nil { + return nil, errors.WithStack(err) + } + + return &filesystem.Stat{ + Info: s, + Mimetype: "application/tar+gzip", + }, nil } // Archive creates an archive of the server and deletes the previous one. func (a *Archiver) Archive() error { - path := a.Server.Filesystem.Path() + path := a.Server.Filesystem().Path() // Get the list of root files and directories to archive. var files []string @@ -52,7 +62,7 @@ func (a *Archiver) Archive() error { } for _, file := range fileInfo { - f, err := a.Server.Filesystem.SafeJoin(path, file) + f, err := a.Server.Filesystem().SafeJoin(path, file) if err != nil { return err } @@ -67,12 +77,12 @@ func (a *Archiver) Archive() error { // Check if the file exists. if stat != nil { - if err := os.Remove(a.ArchivePath()); err != nil { + if err := os.Remove(a.Path()); err != nil { return err } } - return archiver.NewTarGz().Archive(files, a.ArchivePath()) + return archiver.NewTarGz().Archive(files, a.Path()) } // DeleteIfExists deletes the archive if it exists. @@ -84,7 +94,7 @@ func (a *Archiver) DeleteIfExists() error { // Check if the file exists. if stat != nil { - if err := os.Remove(a.ArchivePath()); err != nil { + if err := os.Remove(a.Path()); err != nil { return err } } @@ -94,7 +104,7 @@ func (a *Archiver) DeleteIfExists() error { // Checksum computes a SHA256 checksum of the server's archive. func (a *Archiver) Checksum() (string, error) { - file, err := os.Open(a.ArchivePath()) + file, err := os.Open(a.Path()) if err != nil { return "", err } diff --git a/server/backup.go b/server/backup.go index e13cbf4..e76e5fe 100644 --- a/server/backup.go +++ b/server/backup.go @@ -35,7 +35,7 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc func (s *Server) getServerwideIgnoredFiles() ([]string, error) { var ignored []string - f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore")) + f, err := os.Open(path.Join(s.Filesystem().Path(), ".pteroignore")) if err != nil { if !os.IsNotExist(err) { return nil, err @@ -70,7 +70,7 @@ func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles } // Get the included files based on the root path and the ignored files provided. - return s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), ignored) + return s.Filesystem().GetIncludedFiles(s.Filesystem().Path(), ignored) } // Performs a server backup and then emits the event over the server websocket. We @@ -83,7 +83,7 @@ func (s *Server) Backup(b backup.BackupInterface) error { return errors.WithStack(err) } - ad, err := b.Generate(inc, s.Filesystem.Path()) + ad, err := b.Generate(inc, s.Filesystem().Path()) if err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { s.Log().WithFields(log.Fields{ diff --git a/server/config_parser.go b/server/config_parser.go index 3a87452..c8f231b 100644 --- a/server/config_parser.go +++ b/server/config_parser.go @@ -15,7 +15,7 @@ func (s *Server) UpdateConfigurationFiles() { f := cf pool.Submit(func() { - p, err := s.Filesystem.SafePath(f.FileName) + p, err := s.Filesystem().SafePath(f.FileName) if err != nil { s.Log().WithField("error", err).Error("failed to generate safe path for configuration file") diff --git a/server/filesystem.go b/server/filesystem.go index 5192d1e..c996938 100644 --- a/server/filesystem.go +++ b/server/filesystem.go @@ -1,1022 +1,30 @@ package server import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/gabriel-vasile/mimetype" - "github.com/karrick/godirwalk" "github.com/pkg/errors" - "github.com/pterodactyl/wings/config" - "github.com/pterodactyl/wings/server/backup" - ignore "github.com/sabhiram/go-gitignore" - "golang.org/x/sync/errgroup" - "io" - "io/ioutil" + "github.com/pterodactyl/wings/server/filesystem" "os" - "path" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" ) -// Error returned when there is a bad path provided to one of the FS calls. -type PathResolutionError struct{} - -var ErrIsDirectory = errors.New("is a directory") -var ErrNotEnoughDiskSpace = errors.New("not enough disk space is available to perform this operation") - -// Returns the error response in a string form that can be more easily consumed. -func (pre PathResolutionError) Error() string { - return "invalid path resolution" -} - -func IsPathResolutionError(err error) bool { - _, ok := err.(PathResolutionError) - - return ok -} - -type Filesystem struct { - mu sync.Mutex - lookupTimeMu sync.RWMutex - - lastLookupTime time.Time - lookupInProgress int32 - disk int64 - - Server *Server -} - -// Returns the root path that contains all of a server's data. -func (fs *Filesystem) Path() string { - return filepath.Join(config.Get().System.Data, fs.Server.Id()) -} - -// Normalizes a directory being passed in to ensure the user is not able to escape -// from their data directory. After normalization if the directory is still within their home -// path it is returned. If they managed to "escape" an error will be returned. -// -// This logic is actually copied over from the SFTP server code. Ideally that eventually -// either gets ported into this application, or is able to make use of this package. -func (fs *Filesystem) SafePath(p string) (string, error) { - var nonExistentPathResolution string - - // Start with a cleaned up path before checking the more complex bits. - r := fs.unsafeFilePath(p) - - // At the same time, evaluate the symlink status and determine where this file or folder - // is truly pointing to. - p, err := filepath.EvalSymlinks(r) - if err != nil && !os.IsNotExist(err) { - return "", err - } else if os.IsNotExist(err) { - // The requested directory doesn't exist, so at this point we need to iterate up the - // path chain until we hit a directory that _does_ exist and can be validated. - parts := strings.Split(filepath.Dir(r), "/") - - var try string - // Range over all of the path parts and form directory pathings from the end - // moving up until we have a valid resolution or we run out of paths to try. - for k := range parts { - try = strings.Join(parts[:(len(parts)-k)], "/") - - if !fs.unsafeIsInDataDirectory(try) { - break - } - - t, err := filepath.EvalSymlinks(try) - if err == nil { - nonExistentPathResolution = t - break - } - } - } - - // If the new path doesn't start with their root directory there is clearly an escape - // attempt going on, and we should NOT resolve this path for them. - if nonExistentPathResolution != "" { - if !fs.unsafeIsInDataDirectory(nonExistentPathResolution) { - return "", PathResolutionError{} - } - - // If the nonExistentPathResolution variable is not empty then the initial path requested - // did not exist and we looped through the pathway until we found a match. At this point - // we've confirmed the first matched pathway exists in the root server directory, so we - // can go ahead and just return the path that was requested initially. - return r, nil - } - - // If the requested directory from EvalSymlinks begins with the server root directory go - // ahead and return it. If not we'll return an error which will block any further action - // on the file. - if fs.unsafeIsInDataDirectory(p) { - return p, nil - } - - return "", PathResolutionError{} -} - -// Generate a path to the file by cleaning it up and appending the root server path to it. This -// DOES NOT guarantee that the file resolves within the server data directory. You'll want to use -// the fs.unsafeIsInDataDirectory(p) function to confirm. -func (fs *Filesystem) unsafeFilePath(p string) string { - // Calling filepath.Clean on the joined directory will resolve it to the absolute path, - // removing any ../ type of resolution arguments, and leaving us with a direct path link. - // - // This will also trim the existing root path off the beginning of the path passed to - // the function since that can get a bit messy. - return filepath.Clean(filepath.Join(fs.Path(), strings.TrimPrefix(p, fs.Path()))) -} - -// Check that that path string starts with the server data directory path. This function DOES NOT -// validate that the rest of the path does not end up resolving out of this directory, or that the -// targeted file or folder is not a symlink doing the same thing. -func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool { - return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/") -} - -// Helper function to keep some of the codebase a little cleaner. Returns a "safe" version of the path -// joined with a file. This is important because you cannot just assume that appending a file to a cleaned -// path will result in a cleaned path to that file. For example, imagine you have the following scenario: -// -// my_bad_file -> symlink:/etc/passwd -// -// cleaned := SafePath("../../etc") -> "/" -// filepath.Join(cleaned, my_bad_file) -> "/my_bad_file" -// -// You might think that "/my_bad_file" is fine since it isn't pointing to the original "../../etc/my_bad_file". -// However, this doesn't account for symlinks where the file might be pointing outside of the directory, so -// calling a function such as Chown against it would chown the symlinked location, and not the file within the -// Wings daemon. -func (fs *Filesystem) SafeJoin(dir string, f os.FileInfo) (string, error) { - if f.Mode()&os.ModeSymlink != 0 { - return fs.SafePath(filepath.Join(dir, f.Name())) - } - - return filepath.Join(dir, f.Name()), nil -} - -// Executes the fs.SafePath function in parallel against an array of paths. If any of the calls -// fails an error will be returned. -func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) { - var cleaned []string - - // Simple locker function to avoid racy appends to the array of cleaned paths. - var m = new(sync.Mutex) - var push = func(c string) { - m.Lock() - cleaned = append(cleaned, c) - m.Unlock() - } - - // Create an error group that we can use to run processes in parallel while retaining - // the ability to cancel the entire process immediately should any of it fail. - g, ctx := errgroup.WithContext(context.Background()) - - // Iterate over all of the paths and generate a cleaned path, if there is an error for any - // of the files, abort the process. - for _, p := range paths { - // Create copy so we can use it within the goroutine correctly. - pi := p - - // Recursively call this function to continue digging through the directory tree within - // a separate goroutine. If the context is canceled abort this process. - g.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - // If the callback returns true, go ahead and keep walking deeper. This allows - // us to programmatically continue deeper into directories, or stop digging - // if that pathway knows it needs nothing else. - if c, err := fs.SafePath(pi); err != nil { - return err - } else { - push(c) - } - - return nil - } - }) - } - - // Block until all of the routines finish and have returned a value. - return cleaned, g.Wait() -} - -type SpaceCheckingOpts struct { - AllowStaleResponse bool -} - -// Helper function to determine if a server has space available for a file of a given size. -// If space is available, no error will be returned, otherwise an ErrNotEnoughSpace error -// will be raised. -func (fs *Filesystem) HasSpaceFor(size int64) error { - if fs.Server.DiskSpace() <= 0 { - return nil - } - - s, err := fs.DiskUsage(true) - if err != nil { - return err - } - - if (s + size) > fs.Server.DiskSpace() { - return ErrNotEnoughDiskSpace - } - - return nil -} - -// Determines if the directory a file is trying to be added to has enough space available -// for the file to be written to. -// -// Because determining the amount of space being used by a server is a taxing operation we -// will load it all up into a cache and pull from that as long as the key is not expired. -// -// This operation will potentially block unless allowStaleValue is set to true. See the -// documentation on DiskUsage for how this affects the call. -func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { - size, err := fs.DiskUsage(allowStaleValue) - if err != nil { - fs.Server.Log().WithField("error", err).Warn("failed to determine root server directory size") - } - - // Determine if their folder size, in bytes, is smaller than the amount of space they've - // been allocated. - fs.Server.Proc().SetDisk(size) - - space := fs.Server.DiskSpace() - // If space is -1 or 0 just return true, means they're allowed unlimited. - // - // Technically we could skip disk space calculation because we don't need to check if the server exceeds it's limit - // but because this method caches the disk usage it would be best to calculate the disk usage and always - // return true. - if space <= 0 { - return true - } - - return size <= space -} - -// Internal helper function to allow other parts of the codebase to check the total used disk space -// as needed without overly taxing the system. This will prioritize the value from the cache to avoid -// excessive IO usage. We will only walk the filesystem and determine the size of the directory if there -// is no longer a cached value. -// -// If "allowStaleValue" is set to true, a stale value MAY be returned to the caller if there is an -// expired cache value AND there is currently another lookup in progress. If there is no cached value but -// no other lookup is in progress, a fresh disk space response will be returned to the caller. -// -// This is primarily to avoid a bunch of I/O operations from piling up on the server, especially on servers -// with a large amount of files. -func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { - // Check if cache is expired. - fs.lookupTimeMu.RLock() - isValidInCache := fs.lastLookupTime.After(time.Now().Add(time.Second * time.Duration(-1*config.Get().System.DiskCheckInterval))) - fs.lookupTimeMu.RUnlock() - - if !isValidInCache { - // If we are now allowing a stale response go ahead and perform the lookup and return the fresh - // value. This is a blocking operation to the calling process. - if !allowStaleValue { - return fs.updateCachedDiskUsage() - } else if atomic.LoadInt32(&fs.lookupInProgress) == 0 { - // Otherwise, if we allow a stale value and there isn't a valid item in the cache and we aren't - // currently performing a lookup, just do the disk usage calculation in the background. - go func(fs *Filesystem) { - if _, err := fs.updateCachedDiskUsage(); err != nil { - fs.Server.Log().WithField("error", errors.WithStack(err)).Warn("failed to determine disk usage in go-routine") - } - }(fs) - } - } - - // Return the currently cached value back to the calling function. - return atomic.LoadInt64(&fs.disk), nil -} - -// Updates the currently used disk space for a server. -func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { - // Obtain an exclusive lock on this process so that we don't unintentionally run it at the same - // time as another running process. Once the lock is available it'll read from the cache for the - // second call rather than hitting the disk in parallel. - fs.mu.Lock() - defer fs.mu.Unlock() - - // Signal that we're currently updating the disk size so that other calls to the disk checking - // functions can determine if they should queue up additional calls to this function. Ensure that - // we always set this back to 0 when this process is done executing. - atomic.StoreInt32(&fs.lookupInProgress, 1) - defer atomic.StoreInt32(&fs.lookupInProgress, 0) - - // If there is no size its either because there is no data (in which case running this function - // will have effectively no impact), or there is nothing in the cache, in which case we need to - // grab the size of their data directory. This is a taxing operation, so we want to store it in - // the cache once we've gotten it. - size, err := fs.DirectorySize("/") - - // Always cache the size, even if there is an error. We want to always return that value - // so that we don't cause an endless loop of determining the disk size if there is a temporary - // error encountered. - fs.lookupTimeMu.Lock() - fs.lastLookupTime = time.Now() - fs.lookupTimeMu.Unlock() - - atomic.StoreInt64(&fs.disk, size) - - return size, err -} - -// Determines the directory size of a given location by running parallel tasks to iterate -// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation -// on locations with tons of files, so it is recommended that you cache the output. -func (fs *Filesystem) DirectorySize(dir string) (int64, error) { - d, err := fs.SafePath(dir) - if err != nil { - return 0, errors.WithStack(err) - } - - var size int64 - var st syscall.Stat_t - - err = godirwalk.Walk(d, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - // If this is a symlink then resolve the final destination of it before trying to continue walking - // over its contents. If it resolves outside the server data directory just skip everything else for - // it. Otherwise, allow it to continue. - if e.IsSymlink() { - if _, err := fs.SafePath(p); err != nil { - if IsPathResolutionError(err) { - return godirwalk.SkipThis - } - - return err - } - } - - if !e.IsDir() { - syscall.Lstat(p, &st) - atomic.AddInt64(&size, st.Size) - } - - return nil - }, - }) - - return size, errors.WithStack(err) -} - -// Reads a file on the system and returns it as a byte representation in a file -// reader. This is not the most memory efficient usage since it will be reading the -// entirety of the file into memory. -func (fs *Filesystem) Readfile(p string) (io.Reader, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return nil, err - } - - b, err := ioutil.ReadFile(cleaned) - if err != nil { - return nil, err - } - - return bytes.NewReader(b), nil -} - -// Writes a file to the system. If the file does not already exist one will be created. -func (fs *Filesystem) Writefile(p string, r io.Reader) error { - cleaned, err := fs.SafePath(p) - if err != nil { - return errors.WithStack(err) - } - - var currentSize int64 - - // If the file does not exist on the system already go ahead and create the pathway - // to it and an empty file. We'll then write to it later on after this completes. - if stat, err := os.Stat(cleaned); err != nil { - if !os.IsNotExist(err) { - return errors.WithStack(err) - } - - if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil { - return errors.WithStack(err) - } - - if err := fs.Chown(filepath.Dir(cleaned)); err != nil { - return errors.WithStack(err) - } - } else { - if stat.IsDir() { - return ErrIsDirectory - } - - currentSize = stat.Size() - } - - o := &fileOpener{} - // This will either create the file if it does not already exist, or open and - // truncate the existing file. - file, err := o.open(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return errors.WithStack(err) - } - defer file.Close() - - buf := make([]byte, 1024*4) - sz, err := io.CopyBuffer(file, r, buf) - - // Adjust the disk usage to account for the old size and the new size of the file. - atomic.AddInt64(&fs.disk, sz-currentSize) - - // Finally, chown the file to ensure the permissions don't end up out-of-whack - // if we had just created it. - return fs.Chown(cleaned) -} - -// Defines the stat struct object. -type Stat struct { - Info os.FileInfo - Mimetype string -} - -func (s *Stat) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Name string `json:"name"` - Created string `json:"created"` - Modified string `json:"modified"` - Mode string `json:"mode"` - Size int64 `json:"size"` - Directory bool `json:"directory"` - File bool `json:"file"` - Symlink bool `json:"symlink"` - Mime string `json:"mime"` - }{ - Name: s.Info.Name(), - Created: s.CTime().Format(time.RFC3339), - Modified: s.Info.ModTime().Format(time.RFC3339), - Mode: s.Info.Mode().String(), - Size: s.Info.Size(), - Directory: s.Info.IsDir(), - File: !s.Info.IsDir(), - Symlink: s.Info.Mode().Perm()&os.ModeSymlink != 0, - Mime: s.Mimetype, - }) -} - -// Stats a file or folder and returns the base stat object from go along with the -// MIME data that can be used for editing files. -func (fs *Filesystem) Stat(p string) (*Stat, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return nil, err - } - - return fs.unsafeStat(cleaned) -} - -func (fs *Filesystem) unsafeStat(p string) (*Stat, error) { - s, err := os.Stat(p) - if err != nil { - return nil, err - } - - var m *mimetype.MIME - if !s.IsDir() { - m, err = mimetype.DetectFile(p) - if err != nil { - return nil, err - } - } - - st := &Stat{ - Info: s, - Mimetype: "inode/directory", - } - - if m != nil { - st.Mimetype = m.String() - } - - return st, nil -} - -// Creates a new directory (name) at a specified path (p) for the server. -func (fs *Filesystem) CreateDirectory(name string, p string) error { - cleaned, err := fs.SafePath(path.Join(p, name)) - if err != nil { - return errors.WithStack(err) - } - - return os.MkdirAll(cleaned, 0755) -} - -// Moves (or renames) a file or directory. -func (fs *Filesystem) Rename(from string, to string) error { - cleanedFrom, err := fs.SafePath(from) - if err != nil { - return errors.WithStack(err) - } - - cleanedTo, err := fs.SafePath(to) - if err != nil { - return errors.WithStack(err) - } - - // If the target file or directory already exists the rename function will fail, so just - // bail out now. - if _, err := os.Stat(cleanedTo); err == nil { - return os.ErrExist - } - - if cleanedTo == fs.Path() { - return errors.New("attempting to rename into an invalid directory space") - } - - d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo)) - // Ensure that the directory we're moving into exists correctly on the system. Only do this if - // we're not at the root directory level. - if d != fs.Path() { - if mkerr := os.MkdirAll(d, 0644); mkerr != nil { - return errors.Wrap(mkerr, "failed to create directory structure for file rename") - } - } - - return os.Rename(cleanedFrom, cleanedTo) -} - -// Recursively iterates over a file or directory and sets the permissions on all of the -// underlying files. Iterate over all of the files and directories. If it is a file just -// go ahead and perform the chown operation. Otherwise dig deeper into the directory until -// we've run out of directories to dig into. -func (fs *Filesystem) Chown(path string) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return errors.WithStack(err) - } - - uid := config.Get().System.User.Uid - gid := config.Get().System.User.Gid - - // Start by just chowning the initial path that we received. - if err := os.Chown(cleaned, uid, gid); err != nil { - return errors.WithStack(err) - } - - // If this is not a directory we can now return from the function, there is nothing - // left that we need to do. - if st, _ := os.Stat(cleaned); !st.IsDir() { - return nil - } - - // If this was a directory, begin walking over its contents recursively and ensure that all - // of the subfiles and directories get their permissions updated as well. - return godirwalk.Walk(cleaned, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - // Do not attempt to chmod a symlink. Go's os.Chown function will affect the symlink - // so if it points to a location outside the data directory the user would be able to - // (un)intentionally modify that files permissions. - if e.IsSymlink() { - if e.IsDir() { - return godirwalk.SkipThis - } - - return nil - } - - return os.Chown(p, uid, gid) - }, - }) -} - -// Copies a given file to the same location and appends a suffix to the file to indicate that -// it has been copied. -func (fs *Filesystem) Copy(p string) error { - cleaned, err := fs.SafePath(p) - if err != nil { - return errors.WithStack(err) - } - - s, err := os.Stat(cleaned) - if err != nil { - return errors.WithStack(err) - } else if s.IsDir() || !s.Mode().IsRegular() { - // If this is a directory or not a regular file, just throw a not-exist error - // since anything calling this function should understand what that means. - return os.ErrNotExist - } - - // Check that copying this file wouldn't put the server over its limit. - if err := fs.HasSpaceFor(s.Size()); err != nil { - return err - } - - base := filepath.Base(cleaned) - relative := strings.TrimSuffix(strings.TrimPrefix(cleaned, fs.Path()), base) - extension := filepath.Ext(base) - name := strings.TrimSuffix(base, extension) - - // Ensure that ".tar" is also counted as apart of the file extension. - // There might be a better way to handle this for other double file extensions, - // but this is a good workaround for now. - if strings.HasSuffix(name, ".tar") { - extension = ".tar" + extension - name = strings.TrimSuffix(name, ".tar") - } - - // Begin looping up to 50 times to try and create a unique copy file name. This will take - // an input of "file.txt" and generate "file copy.txt". If that name is already taken, it will - // then try to write "file copy 2.txt" and so on, until reaching 50 loops. At that point we - // won't waste anymore time, just use the current timestamp and make that copy. - // - // Could probably make this more efficient by checking if there are any files matching the copy - // pattern, and trying to find the highest number and then incrementing it by one rather than - // looping endlessly. - var i int - copySuffix := " copy" - for i = 0; i < 51; i++ { - if i > 0 { - copySuffix = " copy " + strconv.Itoa(i) - } - - tryName := fmt.Sprintf("%s%s%s", name, copySuffix, extension) - tryLocation, err := fs.SafePath(path.Join(relative, tryName)) - if err != nil { - return errors.WithStack(err) - } - - // If the file exists, continue to the next loop, otherwise we're good to start a copy. - if _, err := os.Stat(tryLocation); err != nil && !os.IsNotExist(err) { - return errors.WithStack(err) - } else if os.IsNotExist(err) { - break - } - - if i == 50 { - copySuffix = "." + time.Now().Format(time.RFC3339) - } - } - - finalPath, err := fs.SafePath(path.Join(relative, fmt.Sprintf("%s%s%s", name, copySuffix, extension))) - if err != nil { - return errors.WithStack(err) - } - - source, err := os.Open(cleaned) - if err != nil { - return errors.WithStack(err) - } - defer source.Close() - - dest, err := os.Create(finalPath) - if err != nil { - return errors.WithStack(err) - } - defer dest.Close() - - buf := make([]byte, 1024*4) - if _, err := io.CopyBuffer(dest, source, buf); err != nil { - return errors.WithStack(err) - } - - // Once everything is done, increment the disk space used. - atomic.AddInt64(&fs.disk, s.Size()) - - return nil -} - -// Deletes a file or folder from the system. Prevents the user from accidentally -// (or maliciously) removing their root server data directory. -func (fs *Filesystem) Delete(p string) error { - // This is one of the few (only?) places in the codebase where we're explicitly not using - // the SafePath functionality when working with user provided input. If we did, you would - // not be able to delete a file that is a symlink pointing to a location outside of the data - // directory. - // - // We also want to avoid resolving a symlink that points _within_ the data directory and thus - // deleting the actual source file for the symlink rather than the symlink itself. For these - // purposes just resolve the actual file path using filepath.Join() and confirm that the path - // exists within the data directory. - resolved := fs.unsafeFilePath(p) - if !fs.unsafeIsInDataDirectory(resolved) { - return PathResolutionError{} - } - - // Block any whoopsies. - if resolved == fs.Path() { - return errors.New("cannot delete root server directory") - } - - if st, err := os.Stat(resolved); err != nil { - if !os.IsNotExist(err) { - fs.Server.Log().WithField("error", err).WithField("path", resolved).Warn("error while attempting to stat file before deletion") - } - } else { - if !st.IsDir() { - atomic.AddInt64(&fs.disk, -st.Size()) - } else { - go func(st os.FileInfo, resolved string) { - if s, err := fs.DirectorySize(resolved); err == nil { - atomic.AddInt64(&fs.disk, -s) - } - }(st, resolved) - } - } - - return os.RemoveAll(resolved) -} - -// Lists the contents of a given directory and returns stat information about each -// file and folder within it. -func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return nil, err - } - - files, err := ioutil.ReadDir(cleaned) - if err != nil { - return nil, err - } - - var wg sync.WaitGroup - - // You must initialize the output of this directory as a non-nil value otherwise - // when it is marshaled into a JSON object you'll just get 'null' back, which will - // break the panel badly. - out := make([]*Stat, len(files)) - - // Iterate over all of the files and directories returned and perform an async process - // to get the mime-type for them all. - for i, file := range files { - wg.Add(1) - - go func(idx int, f os.FileInfo) { - defer wg.Done() - - var m *mimetype.MIME - var d = "inode/directory" - if !f.IsDir() { - cleanedp, _ := fs.SafeJoin(cleaned, f) - if cleanedp != "" { - m, _ = mimetype.DetectFile(filepath.Join(cleaned, f.Name())) - } else { - // Just pass this for an unknown type because the file could not safely be resolved within - // the server data path. - d = "application/octet-stream" - } - } - - st := &Stat{ - Info: f, - Mimetype: d, - } - - if m != nil { - st.Mimetype = m.String() - } - - out[idx] = st - }(i, file) - } - - wg.Wait() - - // Sort the output alphabetically to begin with since we've run the output - // through an asynchronous process and the order is gonna be very random. - sort.SliceStable(out, func(i, j int) bool { - if out[i].Info.Name() == out[j].Info.Name() || out[i].Info.Name() > out[j].Info.Name() { - return true - } - - return false - }) - - // Then, sort it so that directories are listed first in the output. Everything - // will continue to be alphabetized at this point. - sort.SliceStable(out, func(i, j int) bool { - return out[i].Info.IsDir() - }) - - return out, nil +func (s *Server) Filesystem() *filesystem.Filesystem { + return s.fs } // Ensures that the data directory for the server instance exists. -func (fs *Filesystem) EnsureDataDirectory() error { - if _, err := os.Stat(fs.Path()); err != nil && !os.IsNotExist(err) { +func (s *Server) EnsureDataDirectoryExists() error { + if _, err := os.Stat(s.fs.Path()); err != nil && !os.IsNotExist(err) { return errors.WithStack(err) } else if err != nil { // Create the server data directory because it does not currently exist // on the system. - if err := os.MkdirAll(fs.Path(), 0700); err != nil { + if err := os.MkdirAll(s.fs.Path(), 0700); err != nil { return errors.WithStack(err) } - if err := fs.Chown("/"); err != nil { - fs.Server.Log().WithField("error", err).Warn("failed to chown server data directory") + if err := s.fs.Chown("/"); err != nil { + s.Log().WithField("error", err).Warn("failed to chown server data directory") } } return nil } - -// Given a directory, iterate through all of the files and folders within it and determine -// if they should be included in the output based on an array of ignored matches. This uses -// standard .gitignore formatting to make that determination. -// -// If no ignored files are passed through you'll get the entire directory listing. -func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.IncludedFiles, error) { - cleaned, err := fs.SafePath(dir) - if err != nil { - return nil, err - } - - i, err := ignore.CompileIgnoreLines(ignored...) - if err != nil { - return nil, err - } - - // Walk through all of the files and directories on a server. This callback only returns - // files found, and will keep walking deeper and deeper into directories. - inc := new(backup.IncludedFiles) - - err = godirwalk.Walk(cleaned, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - sp := p - if e.IsSymlink() { - sp, err = fs.SafePath(p) - if err != nil { - if IsPathResolutionError(err) { - return godirwalk.SkipThis - } - - return err - } - } - - // Only push files into the result array since archives can't create an empty directory within them. - if !e.IsDir() { - // Avoid unnecessary parsing if there are no ignored files, nothing will match anyways - // so no reason to call the function. - if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(sp, fs.Path()+"/")) { - inc.Push(sp) - } - } - - // We can't just abort if the path is technically ignored. It is possible there is a nested - // file or folder that should not be excluded, so in this case we need to just keep going - // until we get to a final state. - return nil - }, - }) - - return inc, errors.WithStack(err) -} - -// Compresses all of the files matching the given paths in the specified directory. This function -// also supports passing nested paths to only compress certain files and folders when working in -// a larger directory. This effectively creates a local backup, but rather than ignoring specific -// files and folders, it takes an allow-list of files and folders. -// -// All paths are relative to the dir that is passed in as the first argument, and the compressed -// file will be placed at that location named `archive-{date}.tar.gz`. -func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) { - cleanedRootDir, err := fs.SafePath(dir) - if err != nil { - return nil, err - } - - // Take all of the paths passed in and merge them together with the root directory we've gotten. - for i, p := range paths { - paths[i] = filepath.Join(cleanedRootDir, p) - } - - cleaned, err := fs.ParallelSafePath(paths) - if err != nil { - return nil, err - } - - inc := new(backup.IncludedFiles) - // Iterate over all of the cleaned paths and merge them into a large object of final file - // paths to pass into the archiver. As directories are encountered this will drop into them - // and look for all of the files. - for _, p := range cleaned { - f, err := os.Stat(p) - if err != nil { - fs.Server.Log().WithField("error", err).WithField("path", p).Debug("failed to stat file or directory for compression") - continue - } - - if !f.IsDir() { - inc.Push(p) - } else { - err := godirwalk.Walk(p, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - sp := p - if e.IsSymlink() { - // Ensure that any symlinks are properly resolved to their final destination. If - // that destination is outside the server directory skip over this entire item, otherwise - // use the resolved location for the rest of this function. - sp, err = fs.SafePath(p) - if err != nil { - if IsPathResolutionError(err) { - return godirwalk.SkipThis - } - - return err - } - } - - if !e.IsDir() { - inc.Push(sp) - } - - return nil - }, - }) - - if err != nil { - return nil, err - } - } - } - - a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc} - d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", ""))) - - if err := a.Create(d, context.Background()); err != nil { - return nil, errors.WithStack(err) - } - - f, err := os.Stat(d) - if err != nil { - _ = os.Remove(d) - - return nil, err - } - - if err := fs.HasSpaceFor(f.Size()); err != nil { - _ = os.Remove(d) - - return nil, err - } - - atomic.AddInt64(&fs.disk, f.Size()) - - return f, nil -} - -// Handle errors encountered when walking through directories. -// -// If there is a path resolution error just skip the item entirely. Only return this for a -// directory, otherwise return nil. Returning this error for a file will stop the walking -// for the remainder of the directory. This is assuming an os.FileInfo struct was even returned. -func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error { - if !IsPathResolutionError(err) { - return err - } - - if f != nil && f.IsDir() { - return filepath.SkipDir - } - - return nil -} - -type fileOpener struct { - busy uint -} - -// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file -// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts -// has been exhaused, at which point we will abort with an error. -func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) { - for { - f, err := os.OpenFile(path, flags, perm) - - // If there is an error because the text file is busy, go ahead and sleep for a few - // hundred milliseconds and then try again up to three times before just returning the - // error back to the caller. - // - // Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122 - if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") { - time.Sleep(100 * time.Millisecond << fo.busy) - fo.busy++ - continue - } - - return f, err - } -} diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go new file mode 100644 index 0000000..fc2a4e2 --- /dev/null +++ b/server/filesystem/compress.go @@ -0,0 +1,163 @@ +package filesystem + +import ( + "context" + "fmt" + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "github.com/pterodactyl/wings/server/backup" + ignore "github.com/sabhiram/go-gitignore" + "os" + "path" + "path/filepath" + "strings" + "time" +) + +// Given a directory, iterate through all of the files and folders within it and determine +// if they should be included in the output based on an array of ignored matches. This uses +// standard .gitignore formatting to make that determination. +// +// If no ignored files are passed through you'll get the entire directory listing. +func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.IncludedFiles, error) { + cleaned, err := fs.SafePath(dir) + if err != nil { + return nil, err + } + + i, err := ignore.CompileIgnoreLines(ignored...) + if err != nil { + return nil, err + } + + // Walk through all of the files and directories on a server. This callback only returns + // files found, and will keep walking deeper and deeper into directories. + inc := new(backup.IncludedFiles) + + err = godirwalk.Walk(cleaned, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + sp := p + if e.IsSymlink() { + sp, err = fs.SafePath(p) + if err != nil { + if errors.Is(err, ErrBadPathResolution) { + return godirwalk.SkipThis + } + + return err + } + } + + // Only push files into the result array since archives can't create an empty directory within them. + if !e.IsDir() { + // Avoid unnecessary parsing if there are no ignored files, nothing will match anyways + // so no reason to call the function. + if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(sp, fs.Path()+"/")) { + inc.Push(sp) + } + } + + // We can't just abort if the path is technically ignored. It is possible there is a nested + // file or folder that should not be excluded, so in this case we need to just keep going + // until we get to a final state. + return nil + }, + }) + + return inc, errors.WithStack(err) +} + +// Compresses all of the files matching the given paths in the specified directory. This function +// also supports passing nested paths to only compress certain files and folders when working in +// a larger directory. This effectively creates a local backup, but rather than ignoring specific +// files and folders, it takes an allow-list of files and folders. +// +// All paths are relative to the dir that is passed in as the first argument, and the compressed +// file will be placed at that location named `archive-{date}.tar.gz`. +func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) { + cleanedRootDir, err := fs.SafePath(dir) + if err != nil { + return nil, err + } + + // Take all of the paths passed in and merge them together with the root directory we've gotten. + for i, p := range paths { + paths[i] = filepath.Join(cleanedRootDir, p) + } + + cleaned, err := fs.ParallelSafePath(paths) + if err != nil { + return nil, err + } + + inc := new(backup.IncludedFiles) + // Iterate over all of the cleaned paths and merge them into a large object of final file + // paths to pass into the archiver. As directories are encountered this will drop into them + // and look for all of the files. + for _, p := range cleaned { + f, err := os.Stat(p) + if err != nil { + fs.error(err).WithField("path", p).Debug("failed to stat file or directory for compression") + continue + } + + if !f.IsDir() { + inc.Push(p) + } else { + err := godirwalk.Walk(p, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + sp := p + if e.IsSymlink() { + // Ensure that any symlinks are properly resolved to their final destination. If + // that destination is outside the server directory skip over this entire item, otherwise + // use the resolved location for the rest of this function. + sp, err = fs.SafePath(p) + if err != nil { + if errors.Is(err, ErrBadPathResolution) { + return godirwalk.SkipThis + } + + return err + } + } + + if !e.IsDir() { + inc.Push(sp) + } + + return nil + }, + }) + + if err != nil { + return nil, err + } + } + } + + a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc} + d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", ""))) + + if err := a.Create(d, context.Background()); err != nil { + return nil, errors.WithStack(err) + } + + f, err := os.Stat(d) + if err != nil { + _ = os.Remove(d) + + return nil, err + } + + if err := fs.hasSpaceFor(f.Size()); err != nil { + _ = os.Remove(d) + + return nil, err + } + + fs.addDisk(f.Size()) + + return f, nil +} diff --git a/server/filesystem_unarchive.go b/server/filesystem/decompress.go similarity index 90% rename from server/filesystem_unarchive.go rename to server/filesystem/decompress.go index 128a1d1..1c5cb3a 100644 --- a/server/filesystem_unarchive.go +++ b/server/filesystem/decompress.go @@ -1,4 +1,4 @@ -package server +package filesystem import ( "archive/tar" @@ -14,14 +14,12 @@ import ( "sync/atomic" ) -var ErrUnknownArchiveFormat = errors.New("filesystem: unknown archive format") - // Look through a given archive and determine if decompressing it would put the server over // its allocated disk space limit. func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) { // Don't waste time trying to determine this if we know the server will have the space for // it since there is no limit. - if fs.Server.DiskSpace() <= 0 { + if fs.MaxDisk() <= 0 { return true, nil } @@ -35,18 +33,18 @@ func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (b dirSize, err := fs.DiskUsage(false) var size int64 - var max = fs.Server.DiskSpace() // Walk over the archive and figure out just how large the final output would be from unarchiving it. err = archiver.Walk(source, func(f archiver.File) error { - if atomic.AddInt64(&size, f.Size())+dirSize > max { - return errors.WithStack(ErrNotEnoughDiskSpace) + if atomic.AddInt64(&size, f.Size())+dirSize > fs.MaxDisk() { + return ErrNotEnoughDiskSpace } return nil }) + if err != nil { if strings.HasPrefix(err.Error(), "format ") { - return false, errors.WithStack(ErrUnknownArchiveFormat) + return false, ErrUnknownArchiveFormat } return false, errors.WithStack(err) diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go new file mode 100644 index 0000000..4eb3273 --- /dev/null +++ b/server/filesystem/disk_space.go @@ -0,0 +1,210 @@ +package filesystem + +import ( + "github.com/apex/log" + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "sync" + "sync/atomic" + "syscall" + "time" +) + +type SpaceCheckingOpts struct { + AllowStaleResponse bool +} + +type usageLookupTime struct { + sync.RWMutex + value time.Time +} + +// Update the last time that a disk space lookup was performed. +func (ult *usageLookupTime) Set(t time.Time) { + ult.Lock() + ult.value = t + ult.Unlock() +} + +// Get the last time that we performed a disk space usage lookup. +func (ult *usageLookupTime) Get() time.Time { + ult.RLock() + defer ult.RUnlock() + + return ult.value +} + +// Returns the maximum amount of disk space that this Filesystem instance is allowed to use. +func (fs *Filesystem) MaxDisk() int64 { + fs.mu.RLock() + defer fs.mu.RUnlock() + + return fs.diskLimit +} + +// Sets the disk space limit for this Filesystem instance. +func (fs *Filesystem) SetDiskLimit(i int64) { + fs.mu.Lock() + fs.diskLimit = i + fs.mu.Unlock() +} + +// Determines if the directory a file is trying to be added to has enough space available +// for the file to be written to. +// +// Because determining the amount of space being used by a server is a taxing operation we +// will load it all up into a cache and pull from that as long as the key is not expired. +// +// This operation will potentially block unless allowStaleValue is set to true. See the +// documentation on DiskUsage for how this affects the call. +func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { + size, err := fs.DiskUsage(allowStaleValue) + if err != nil { + log.WithField("root", fs.root).WithField("error", err).Warn("failed to determine root fs directory size") + } + + // If space is -1 or 0 just return true, means they're allowed unlimited. + // + // Technically we could skip disk space calculation because we don't need to check if the + // server exceeds it's limit but because this method caches the disk usage it would be best + // to calculate the disk usage and always return true. + if fs.MaxDisk() == 0 { + return true + } + + return size <= fs.MaxDisk() +} + +// Internal helper function to allow other parts of the codebase to check the total used disk space +// as needed without overly taxing the system. This will prioritize the value from the cache to avoid +// excessive IO usage. We will only walk the filesystem and determine the size of the directory if there +// is no longer a cached value. +// +// If "allowStaleValue" is set to true, a stale value MAY be returned to the caller if there is an +// expired cache value AND there is currently another lookup in progress. If there is no cached value but +// no other lookup is in progress, a fresh disk space response will be returned to the caller. +// +// This is primarily to avoid a bunch of I/O operations from piling up on the server, especially on servers +// with a large amount of files. +func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { + if !fs.lastLookupTime.Get().After(time.Now().Add(time.Second * fs.diskCheckInterval * -1)) { + // If we are now allowing a stale response go ahead and perform the lookup and return the fresh + // value. This is a blocking operation to the calling process. + if !allowStaleValue { + return fs.updateCachedDiskUsage() + } else if !fs.lookupInProgress.Get() { + // Otherwise, if we allow a stale value and there isn't a valid item in the cache and we aren't + // currently performing a lookup, just do the disk usage calculation in the background. + go func(fs *Filesystem) { + if _, err := fs.updateCachedDiskUsage(); err != nil { + log.WithField("root", fs.root).WithField("error", err).Warn("failed to update fs disk usage from within routine") + } + }(fs) + } + } + + // Return the currently cached value back to the calling function. + return atomic.LoadInt64(&fs.diskUsed), nil +} + +// Updates the currently used disk space for a server. +func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { + // Obtain an exclusive lock on this process so that we don't unintentionally run it at the same + // time as another running process. Once the lock is available it'll read from the cache for the + // second call rather than hitting the disk in parallel. + fs.mu.Lock() + defer fs.mu.Unlock() + + // Signal that we're currently updating the disk size so that other calls to the disk checking + // functions can determine if they should queue up additional calls to this function. Ensure that + // we always set this back to "false" when this process is done executing. + fs.lookupInProgress.Set(true) + defer fs.lookupInProgress.Set(false) + + // If there is no size its either because there is no data (in which case running this function + // will have effectively no impact), or there is nothing in the cache, in which case we need to + // grab the size of their data directory. This is a taxing operation, so we want to store it in + // the cache once we've gotten it. + size, err := fs.DirectorySize("/") + + // Always cache the size, even if there is an error. We want to always return that value + // so that we don't cause an endless loop of determining the disk size if there is a temporary + // error encountered. + fs.lastLookupTime.Set(time.Now()) + + atomic.StoreInt64(&fs.diskUsed, size) + + return size, err +} + +// Determines the directory size of a given location by running parallel tasks to iterate +// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation +// on locations with tons of files, so it is recommended that you cache the output. +func (fs *Filesystem) DirectorySize(dir string) (int64, error) { + d, err := fs.SafePath(dir) + if err != nil { + return 0, errors.WithStack(err) + } + + var size int64 + var st syscall.Stat_t + + err = godirwalk.Walk(d, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + // If this is a symlink then resolve the final destination of it before trying to continue walking + // over its contents. If it resolves outside the server data directory just skip everything else for + // it. Otherwise, allow it to continue. + if e.IsSymlink() { + if _, err := fs.SafePath(p); err != nil { + if errors.Is(err, ErrBadPathResolution) { + return godirwalk.SkipThis + } + + return err + } + } + + if !e.IsDir() { + syscall.Lstat(p, &st) + atomic.AddInt64(&size, st.Size) + } + + return nil + }, + }) + + return size, errors.WithStack(err) +} + +// Helper function to determine if a server has space available for a file of a given size. +// If space is available, no error will be returned, otherwise an ErrNotEnoughSpace error +// will be raised. +func (fs *Filesystem) hasSpaceFor(size int64) error { + if fs.MaxDisk() == 0 { + return nil + } + + s, err := fs.DiskUsage(true) + if err != nil { + return err + } + + if (s + size) > fs.MaxDisk() { + return ErrNotEnoughDiskSpace + } + + return nil +} + +// Updates the disk usage for the Filesystem instance. +func (fs *Filesystem) addDisk(i int64) int64 { + size, _ := fs.DiskUsage(true) + + // If we're dropping below 0 somehow just cap it to 0. + if (size + i) < 0 { + return atomic.SwapInt64(&fs.diskUsed, 0) + } + + return atomic.AddInt64(&fs.diskUsed, i) +} diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go new file mode 100644 index 0000000..d2cb01e --- /dev/null +++ b/server/filesystem/errors.go @@ -0,0 +1,35 @@ +package filesystem + +import ( + "github.com/apex/log" + "github.com/pkg/errors" + "os" + "path/filepath" +) + +var ErrIsDirectory = errors.New("filesystem: is a directory") +var ErrNotEnoughDiskSpace = errors.New("filesystem: not enough disk space") +var ErrBadPathResolution = errors.New("filesystem: invalid path resolution") +var ErrUnknownArchiveFormat = errors.New("filesystem: unknown archive format") + +// Generates an error logger instance with some basic information. +func (fs *Filesystem) error(err error) *log.Entry { + return log.WithField("subsystem", "filesystem").WithField("root", fs.root).WithField("error", err) +} + +// Handle errors encountered when walking through directories. +// +// If there is a path resolution error just skip the item entirely. Only return this for a +// directory, otherwise return nil. Returning this error for a file will stop the walking +// for the remainder of the directory. This is assuming an os.FileInfo struct was even returned. +func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error { + if !errors.Is(err, ErrBadPathResolution) { + return err + } + + if f != nil && f.IsDir() { + return filepath.SkipDir + } + + return nil +} \ No newline at end of file diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go new file mode 100644 index 0000000..a98e0f9 --- /dev/null +++ b/server/filesystem/filesystem.go @@ -0,0 +1,453 @@ +package filesystem + +import ( + "bufio" + "bytes" + "fmt" + "github.com/gabriel-vasile/mimetype" + "github.com/karrick/godirwalk" + "github.com/pkg/errors" + "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/system" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type Filesystem struct { + mu sync.RWMutex + lastLookupTime *usageLookupTime + lookupInProgress system.AtomicBool + diskUsed int64 + diskCheckInterval time.Duration + + // The maximum amount of disk space (in bytes) that this Filesystem instance can use. + diskLimit int64 + + // The root data directory path for this Filesystem instance. + root string +} + +// Creates a new Filesystem instance for a given server. +func New(root string, size int64) *Filesystem { + return &Filesystem{ + root: root, + diskLimit: size, + diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval), + lastLookupTime: &usageLookupTime{}, + } +} + +// Returns the root path for the Filesystem instance. +func (fs *Filesystem) Path() string { + return fs.root +} + +// Reads a file on the system and returns it as a byte representation in a file +// reader. This is not the most memory efficient usage since it will be reading the +// entirety of the file into memory. +func (fs *Filesystem) Readfile(p string) (io.Reader, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + + b, err := ioutil.ReadFile(cleaned) + if err != nil { + return nil, err + } + + return bytes.NewReader(b), nil +} + +// Writes a file to the system. If the file does not already exist one will be created. +func (fs *Filesystem) Writefile(p string, r io.Reader) error { + cleaned, err := fs.SafePath(p) + if err != nil { + return errors.WithStack(err) + } + + var currentSize int64 + // If the file does not exist on the system already go ahead and create the pathway + // to it and an empty file. We'll then write to it later on after this completes. + if stat, err := os.Stat(cleaned); err != nil { + if !os.IsNotExist(err) { + return errors.WithStack(err) + } + + if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil { + return errors.WithStack(err) + } + + if err := fs.Chown(filepath.Dir(cleaned)); err != nil { + return errors.WithStack(err) + } + } else { + if stat.IsDir() { + return ErrIsDirectory + } + + currentSize = stat.Size() + } + + br := bufio.NewReader(r) + // Check that the new size we're writing to the disk can fit. If there is currently a file + // we'll subtract that current file size from the size of the buffer to determine the amount + // of new data we're writing (or amount we're removing if smaller). + if err := fs.hasSpaceFor(int64(br.Size()) - currentSize); err != nil { + return err + } + + o := &fileOpener{} + // This will either create the file if it does not already exist, or open and + // truncate the existing file. + file, err := o.open(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return errors.WithStack(err) + } + defer file.Close() + + buf := make([]byte, 1024*4) + sz, err := io.CopyBuffer(file, r, buf) + + // Adjust the disk usage to account for the old size and the new size of the file. + fs.addDisk(sz - currentSize) + + // Finally, chown the file to ensure the permissions don't end up out-of-whack + // if we had just created it. + return fs.Chown(cleaned) +} + +// Creates a new directory (name) at a specified path (p) for the server. +func (fs *Filesystem) CreateDirectory(name string, p string) error { + cleaned, err := fs.SafePath(path.Join(p, name)) + if err != nil { + return errors.WithStack(err) + } + + return os.MkdirAll(cleaned, 0755) +} + +// Moves (or renames) a file or directory. +func (fs *Filesystem) Rename(from string, to string) error { + cleanedFrom, err := fs.SafePath(from) + if err != nil { + return errors.WithStack(err) + } + + cleanedTo, err := fs.SafePath(to) + if err != nil { + return errors.WithStack(err) + } + + // If the target file or directory already exists the rename function will fail, so just + // bail out now. + if _, err := os.Stat(cleanedTo); err == nil { + return os.ErrExist + } + + if cleanedTo == fs.Path() { + return errors.New("attempting to rename into an invalid directory space") + } + + d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo)) + // Ensure that the directory we're moving into exists correctly on the system. Only do this if + // we're not at the root directory level. + if d != fs.Path() { + if mkerr := os.MkdirAll(d, 0644); mkerr != nil { + return errors.Wrap(mkerr, "failed to create directory structure for file rename") + } + } + + return os.Rename(cleanedFrom, cleanedTo) +} + +// Recursively iterates over a file or directory and sets the permissions on all of the +// underlying files. Iterate over all of the files and directories. If it is a file just +// go ahead and perform the chown operation. Otherwise dig deeper into the directory until +// we've run out of directories to dig into. +func (fs *Filesystem) Chown(path string) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return errors.WithStack(err) + } + + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid + + // Start by just chowning the initial path that we received. + if err := os.Chown(cleaned, uid, gid); err != nil { + return errors.WithStack(err) + } + + // If this is not a directory we can now return from the function, there is nothing + // left that we need to do. + if st, _ := os.Stat(cleaned); !st.IsDir() { + return nil + } + + // If this was a directory, begin walking over its contents recursively and ensure that all + // of the subfiles and directories get their permissions updated as well. + return godirwalk.Walk(cleaned, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + // Do not attempt to chmod a symlink. Go's os.Chown function will affect the symlink + // so if it points to a location outside the data directory the user would be able to + // (un)intentionally modify that files permissions. + if e.IsSymlink() { + if e.IsDir() { + return godirwalk.SkipThis + } + + return nil + } + + return os.Chown(p, uid, gid) + }, + }) +} + +// Copies a given file to the same location and appends a suffix to the file to indicate that +// it has been copied. +func (fs *Filesystem) Copy(p string) error { + cleaned, err := fs.SafePath(p) + if err != nil { + return errors.WithStack(err) + } + + s, err := os.Stat(cleaned) + if err != nil { + return errors.WithStack(err) + } else if s.IsDir() || !s.Mode().IsRegular() { + // If this is a directory or not a regular file, just throw a not-exist error + // since anything calling this function should understand what that means. + return os.ErrNotExist + } + + // Check that copying this file wouldn't put the server over its limit. + if err := fs.hasSpaceFor(s.Size()); err != nil { + return err + } + + base := filepath.Base(cleaned) + relative := strings.TrimSuffix(strings.TrimPrefix(cleaned, fs.Path()), base) + extension := filepath.Ext(base) + name := strings.TrimSuffix(base, extension) + + // Ensure that ".tar" is also counted as apart of the file extension. + // There might be a better way to handle this for other double file extensions, + // but this is a good workaround for now. + if strings.HasSuffix(name, ".tar") { + extension = ".tar" + extension + name = strings.TrimSuffix(name, ".tar") + } + + // Begin looping up to 50 times to try and create a unique copy file name. This will take + // an input of "file.txt" and generate "file copy.txt". If that name is already taken, it will + // then try to write "file copy 2.txt" and so on, until reaching 50 loops. At that point we + // won't waste anymore time, just use the current timestamp and make that copy. + // + // Could probably make this more efficient by checking if there are any files matching the copy + // pattern, and trying to find the highest number and then incrementing it by one rather than + // looping endlessly. + var i int + copySuffix := " copy" + for i = 0; i < 51; i++ { + if i > 0 { + copySuffix = " copy " + strconv.Itoa(i) + } + + tryName := fmt.Sprintf("%s%s%s", name, copySuffix, extension) + tryLocation, err := fs.SafePath(path.Join(relative, tryName)) + if err != nil { + return errors.WithStack(err) + } + + // If the file exists, continue to the next loop, otherwise we're good to start a copy. + if _, err := os.Stat(tryLocation); err != nil && !os.IsNotExist(err) { + return errors.WithStack(err) + } else if os.IsNotExist(err) { + break + } + + if i == 50 { + copySuffix = "." + time.Now().Format(time.RFC3339) + } + } + + finalPath, err := fs.SafePath(path.Join(relative, fmt.Sprintf("%s%s%s", name, copySuffix, extension))) + if err != nil { + return errors.WithStack(err) + } + + source, err := os.Open(cleaned) + if err != nil { + return errors.WithStack(err) + } + defer source.Close() + + dest, err := os.Create(finalPath) + if err != nil { + return errors.WithStack(err) + } + defer dest.Close() + + buf := make([]byte, 1024*4) + if _, err := io.CopyBuffer(dest, source, buf); err != nil { + return errors.WithStack(err) + } + + // Once everything is done, increment the disk space used. + fs.addDisk(s.Size()) + + return nil +} + +// Deletes a file or folder from the system. Prevents the user from accidentally +// (or maliciously) removing their root server data directory. +func (fs *Filesystem) Delete(p string) error { + // This is one of the few (only?) places in the codebase where we're explicitly not using + // the SafePath functionality when working with user provided input. If we did, you would + // not be able to delete a file that is a symlink pointing to a location outside of the data + // directory. + // + // We also want to avoid resolving a symlink that points _within_ the data directory and thus + // deleting the actual source file for the symlink rather than the symlink itself. For these + // purposes just resolve the actual file path using filepath.Join() and confirm that the path + // exists within the data directory. + resolved := fs.unsafeFilePath(p) + if !fs.unsafeIsInDataDirectory(resolved) { + return ErrBadPathResolution + } + + // Block any whoopsies. + if resolved == fs.Path() { + return errors.New("cannot delete root server directory") + } + + if st, err := os.Stat(resolved); err != nil { + if !os.IsNotExist(err) { + fs.error(err).Warn("error while attempting to stat file before deletion") + } + } else { + if !st.IsDir() { + fs.addDisk(-st.Size()) + } else { + go func(st os.FileInfo, resolved string) { + if s, err := fs.DirectorySize(resolved); err == nil { + fs.addDisk(-s) + } + }(st, resolved) + } + } + + return os.RemoveAll(resolved) +} + +type fileOpener struct { + busy uint +} + +// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file +// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts +// has been exhaused, at which point we will abort with an error. +func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) { + for { + f, err := os.OpenFile(path, flags, perm) + + // If there is an error because the text file is busy, go ahead and sleep for a few + // hundred milliseconds and then try again up to three times before just returning the + // error back to the caller. + // + // Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122 + if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") { + time.Sleep(100 * time.Millisecond << fo.busy) + fo.busy++ + continue + } + + return f, err + } +} + +// Lists the contents of a given directory and returns stat information about each +// file and folder within it. +func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + + files, err := ioutil.ReadDir(cleaned) + if err != nil { + return nil, err + } + + var wg sync.WaitGroup + + // You must initialize the output of this directory as a non-nil value otherwise + // when it is marshaled into a JSON object you'll just get 'null' back, which will + // break the panel badly. + out := make([]*Stat, len(files)) + + // Iterate over all of the files and directories returned and perform an async process + // to get the mime-type for them all. + for i, file := range files { + wg.Add(1) + + go func(idx int, f os.FileInfo) { + defer wg.Done() + + var m *mimetype.MIME + var d = "inode/directory" + if !f.IsDir() { + cleanedp, _ := fs.SafeJoin(cleaned, f) + if cleanedp != "" { + m, _ = mimetype.DetectFile(filepath.Join(cleaned, f.Name())) + } else { + // Just pass this for an unknown type because the file could not safely be resolved within + // the server data path. + d = "application/octet-stream" + } + } + + st := &Stat{ + Info: f, + Mimetype: d, + } + + if m != nil { + st.Mimetype = m.String() + } + + out[idx] = st + }(i, file) + } + + wg.Wait() + + // Sort the output alphabetically to begin with since we've run the output + // through an asynchronous process and the order is gonna be very random. + sort.SliceStable(out, func(i, j int) bool { + if out[i].Info.Name() == out[j].Info.Name() || out[i].Info.Name() > out[j].Info.Name() { + return true + } + + return false + }) + + // Then, sort it so that directories are listed first in the output. Everything + // will continue to be alphabetized at this point. + sort.SliceStable(out, func(i, j int) bool { + return out[i].Info.IsDir() + }) + + return out, nil +} diff --git a/server/filesystem/path.go b/server/filesystem/path.go new file mode 100644 index 0000000..dd98022 --- /dev/null +++ b/server/filesystem/path.go @@ -0,0 +1,162 @@ +package filesystem + +import ( + "context" + "golang.org/x/sync/errgroup" + "os" + "path/filepath" + "strings" + "sync" +) + +// Normalizes a directory being passed in to ensure the user is not able to escape +// from their data directory. After normalization if the directory is still within their home +// path it is returned. If they managed to "escape" an error will be returned. +// +// This logic is actually copied over from the SFTP server code. Ideally that eventually +// either gets ported into this application, or is able to make use of this package. +func (fs *Filesystem) SafePath(p string) (string, error) { + var nonExistentPathResolution string + + // Start with a cleaned up path before checking the more complex bits. + r := fs.unsafeFilePath(p) + + // At the same time, evaluate the symlink status and determine where this file or folder + // is truly pointing to. + p, err := filepath.EvalSymlinks(r) + if err != nil && !os.IsNotExist(err) { + return "", err + } else if os.IsNotExist(err) { + // The requested directory doesn't exist, so at this point we need to iterate up the + // path chain until we hit a directory that _does_ exist and can be validated. + parts := strings.Split(filepath.Dir(r), "/") + + var try string + // Range over all of the path parts and form directory pathings from the end + // moving up until we have a valid resolution or we run out of paths to try. + for k := range parts { + try = strings.Join(parts[:(len(parts)-k)], "/") + + if !fs.unsafeIsInDataDirectory(try) { + break + } + + t, err := filepath.EvalSymlinks(try) + if err == nil { + nonExistentPathResolution = t + break + } + } + } + + // If the new path doesn't start with their root directory there is clearly an escape + // attempt going on, and we should NOT resolve this path for them. + if nonExistentPathResolution != "" { + if !fs.unsafeIsInDataDirectory(nonExistentPathResolution) { + return "", ErrBadPathResolution + } + + // If the nonExistentPathResolution variable is not empty then the initial path requested + // did not exist and we looped through the pathway until we found a match. At this point + // we've confirmed the first matched pathway exists in the root server directory, so we + // can go ahead and just return the path that was requested initially. + return r, nil + } + + // If the requested directory from EvalSymlinks begins with the server root directory go + // ahead and return it. If not we'll return an error which will block any further action + // on the file. + if fs.unsafeIsInDataDirectory(p) { + return p, nil + } + + return "", ErrBadPathResolution +} + +// Generate a path to the file by cleaning it up and appending the root server path to it. This +// DOES NOT guarantee that the file resolves within the server data directory. You'll want to use +// the fs.unsafeIsInDataDirectory(p) function to confirm. +func (fs *Filesystem) unsafeFilePath(p string) string { + // Calling filepath.Clean on the joined directory will resolve it to the absolute path, + // removing any ../ type of resolution arguments, and leaving us with a direct path link. + // + // This will also trim the existing root path off the beginning of the path passed to + // the function since that can get a bit messy. + return filepath.Clean(filepath.Join(fs.Path(), strings.TrimPrefix(p, fs.Path()))) +} + +// Check that that path string starts with the server data directory path. This function DOES NOT +// validate that the rest of the path does not end up resolving out of this directory, or that the +// targeted file or folder is not a symlink doing the same thing. +func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool { + return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/") +} + +// Helper function to keep some of the codebase a little cleaner. Returns a "safe" version of the path +// joined with a file. This is important because you cannot just assume that appending a file to a cleaned +// path will result in a cleaned path to that file. For example, imagine you have the following scenario: +// +// my_bad_file -> symlink:/etc/passwd +// +// cleaned := SafePath("../../etc") -> "/" +// filepath.Join(cleaned, my_bad_file) -> "/my_bad_file" +// +// You might think that "/my_bad_file" is fine since it isn't pointing to the original "../../etc/my_bad_file". +// However, this doesn't account for symlinks where the file might be pointing outside of the directory, so +// calling a function such as Chown against it would chown the symlinked location, and not the file within the +// Wings daemon. +func (fs *Filesystem) SafeJoin(dir string, f os.FileInfo) (string, error) { + if f.Mode()&os.ModeSymlink != 0 { + return fs.SafePath(filepath.Join(dir, f.Name())) + } + + return filepath.Join(dir, f.Name()), nil +} + +// Executes the fs.SafePath function in parallel against an array of paths. If any of the calls +// fails an error will be returned. +func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) { + var cleaned []string + + // Simple locker function to avoid racy appends to the array of cleaned paths. + var m = new(sync.Mutex) + var push = func(c string) { + m.Lock() + cleaned = append(cleaned, c) + m.Unlock() + } + + // Create an error group that we can use to run processes in parallel while retaining + // the ability to cancel the entire process immediately should any of it fail. + g, ctx := errgroup.WithContext(context.Background()) + + // Iterate over all of the paths and generate a cleaned path, if there is an error for any + // of the files, abort the process. + for _, p := range paths { + // Create copy so we can use it within the goroutine correctly. + pi := p + + // Recursively call this function to continue digging through the directory tree within + // a separate goroutine. If the context is canceled abort this process. + g.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // If the callback returns true, go ahead and keep walking deeper. This allows + // us to programmatically continue deeper into directories, or stop digging + // if that pathway knows it needs nothing else. + if c, err := fs.SafePath(pi); err != nil { + return err + } else { + push(c) + } + + return nil + } + }) + } + + // Block until all of the routines finish and have returned a value. + return cleaned, g.Wait() +} diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go new file mode 100644 index 0000000..7896f9f --- /dev/null +++ b/server/filesystem/stat.go @@ -0,0 +1,74 @@ +package filesystem + +import ( + "encoding/json" + "github.com/gabriel-vasile/mimetype" + "os" + "time" +) + +type Stat struct { + Info os.FileInfo + Mimetype string +} + +func (s *Stat) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Name string `json:"name"` + Created string `json:"created"` + Modified string `json:"modified"` + Mode string `json:"mode"` + Size int64 `json:"size"` + Directory bool `json:"directory"` + File bool `json:"file"` + Symlink bool `json:"symlink"` + Mime string `json:"mime"` + }{ + Name: s.Info.Name(), + Created: s.CTime().Format(time.RFC3339), + Modified: s.Info.ModTime().Format(time.RFC3339), + Mode: s.Info.Mode().String(), + Size: s.Info.Size(), + Directory: s.Info.IsDir(), + File: !s.Info.IsDir(), + Symlink: s.Info.Mode().Perm()&os.ModeSymlink != 0, + Mime: s.Mimetype, + }) +} + +// Stats a file or folder and returns the base stat object from go along with the +// MIME data that can be used for editing files. +func (fs *Filesystem) Stat(p string) (*Stat, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + + return fs.unsafeStat(cleaned) +} + +func (fs *Filesystem) unsafeStat(p string) (*Stat, error) { + s, err := os.Stat(p) + if err != nil { + return nil, err + } + + var m *mimetype.MIME + if !s.IsDir() { + m, err = mimetype.DetectFile(p) + if err != nil { + return nil, err + } + } + + st := &Stat{ + Info: s, + Mimetype: "inode/directory", + } + + if m != nil { + st.Mimetype = m.String() + } + + return st, nil +} diff --git a/server/filesystem_darwin.go b/server/filesystem/stat_darwin.go similarity index 91% rename from server/filesystem_darwin.go rename to server/filesystem/stat_darwin.go index dff5324..4bc6abd 100644 --- a/server/filesystem_darwin.go +++ b/server/filesystem/stat_darwin.go @@ -1,4 +1,4 @@ -package server +package filesystem import ( "syscall" diff --git a/server/filesystem_linux.go b/server/filesystem/stat_linux.go similarity index 91% rename from server/filesystem_linux.go rename to server/filesystem/stat_linux.go index e0e0ea9..5bec32e 100644 --- a/server/filesystem_linux.go +++ b/server/filesystem/stat_linux.go @@ -1,4 +1,4 @@ -package server +package filesystem import ( "syscall" diff --git a/server/filesystem_windows.go b/server/filesystem/stat_windows.go similarity index 92% rename from server/filesystem_windows.go rename to server/filesystem/stat_windows.go index 2e3d408..4cd6dff 100644 --- a/server/filesystem_windows.go +++ b/server/filesystem/stat_windows.go @@ -1,4 +1,4 @@ -package server +package filesystem import ( "time" diff --git a/server/install.go b/server/install.go index b668f0f..5e2ec0b 100644 --- a/server/install.go +++ b/server/install.go @@ -409,7 +409,7 @@ func (ip *InstallationProcess) Execute() (string, error) { Mounts: []mount.Mount{ { Target: "/mnt/server", - Source: ip.Server.Filesystem.Path(), + Source: ip.Server.Filesystem().Path(), Type: mount.TypeBind, ReadOnly: false, }, diff --git a/server/listeners.go b/server/listeners.go index 3b1cbbb..308c42f 100644 --- a/server/listeners.go +++ b/server/listeners.go @@ -81,7 +81,7 @@ func (s *Server) StartEventListeners() { s.resources.Stats = *st s.resources.mu.Unlock() - s.Filesystem.HasSpaceAvailable(true) + s.Filesystem().HasSpaceAvailable(true) s.emitProcUsage() } diff --git a/server/loader.go b/server/loader.go index dec6f79..e2e12da 100644 --- a/server/loader.go +++ b/server/loader.go @@ -7,9 +7,12 @@ import ( "github.com/gammazero/workerpool" "github.com/pkg/errors" "github.com/pterodactyl/wings/api" + "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment/docker" + "github.com/pterodactyl/wings/server/filesystem" "os" + "path/filepath" "runtime" "time" ) @@ -90,7 +93,7 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) { defaults.Set(&s.resources) s.Archiver = Archiver{Server: s} - s.Filesystem = Filesystem{Server: s} + s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace()) // Right now we only support a Docker based environment, so I'm going to hard code // this logic in. When we're ready to support other environment we'll need to make @@ -120,8 +123,8 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) { } // If the server's data directory exists, force disk usage calculation. - if _, err := os.Stat(s.Filesystem.Path()); err == nil { - s.Filesystem.HasSpaceAvailable(true) + if _, err := os.Stat(s.Filesystem().Path()); err == nil { + s.Filesystem().HasSpaceAvailable(true) } return s, nil diff --git a/server/mounts.go b/server/mounts.go index 3a1391a..bb1b699 100644 --- a/server/mounts.go +++ b/server/mounts.go @@ -24,7 +24,7 @@ func (s *Server) Mounts() []environment.Mount { m = append(m, environment.Mount{ Default: true, Target: "/home/container", - Source: s.Filesystem.Path(), + Source: s.Filesystem().Path(), ReadOnly: false, }) diff --git a/server/power.go b/server/power.go index d86f872..f9af920 100644 --- a/server/power.go +++ b/server/power.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/environment" + "github.com/pterodactyl/wings/server/filesystem" "golang.org/x/sync/semaphore" "os" "time" @@ -164,11 +165,11 @@ func (s *Server) onBeforeStart() error { // If a server has unlimited disk space, we don't care enough to block the startup to check remaining. // However, we should trigger a size anyway, as it'd be good to kick it off for other processes. if s.DiskSpace() <= 0 { - s.Filesystem.HasSpaceAvailable(true) + s.Filesystem().HasSpaceAvailable(true) } else { s.PublishConsoleOutputFromDaemon("Checking server disk space usage, this could take a few seconds...") - if !s.Filesystem.HasSpaceAvailable(false) { - return ErrNotEnoughDiskSpace + if !s.Filesystem().HasSpaceAvailable(false) { + return filesystem.ErrNotEnoughDiskSpace } } @@ -183,7 +184,7 @@ func (s *Server) onBeforeStart() error { if config.Get().System.CheckPermissionsOnBoot { s.PublishConsoleOutputFromDaemon("Ensuring file permissions are set correctly, this could take a few seconds...") // Ensure all of the server file permissions are set correctly before booting the process. - if err := s.Filesystem.Chown("/"); err != nil { + if err := s.Filesystem().Chown("/"); err != nil { return errors.Wrap(err, "failed to chown root server directory during pre-boot process") } } diff --git a/server/server.go b/server/server.go index c5f9cf3..2351bf9 100644 --- a/server/server.go +++ b/server/server.go @@ -9,6 +9,7 @@ import ( "github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment/docker" "github.com/pterodactyl/wings/events" + "github.com/pterodactyl/wings/server/filesystem" "golang.org/x/sync/semaphore" "strings" "sync" @@ -34,7 +35,8 @@ type Server struct { resources ResourceUsage Archiver Archiver `json:"-"` Environment environment.ProcessEnvironment `json:"-"` - Filesystem Filesystem `json:"-"` + + fs *filesystem.Filesystem // Events emitted by the server instance. emitter *events.EventBus @@ -133,6 +135,10 @@ func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) err s.procConfig = cfg.ProcessConfiguration s.Unlock() + // Update the disk space limits for the server whenever the configuration + // for it changes. + s.fs.SetDiskLimit(s.DiskSpace()) + // If this is a Docker environment we need to sync the stop configuration with it so that // the process isn't just terminated when a user requests it be stopped. if e, ok := s.Environment.(*docker.Environment); ok { @@ -161,7 +167,7 @@ func (s *Server) IsBootable() bool { // for the server is setup, and that all of the necessary files are created. func (s *Server) CreateEnvironment() error { // Ensure the data directory exists before getting too far through this process. - if err := s.Filesystem.EnsureDataDirectory(); err != nil { + if err := s.EnsureDataDirectoryExists(); err != nil { return errors.WithStack(err) } diff --git a/sftp/sftp.go b/sftp/sftp.go index 6fe044b..6fc8482 100644 --- a/sftp/sftp.go +++ b/sftp/sftp.go @@ -51,7 +51,7 @@ func validatePath(fs FileSystem, p string) (string, error) { return "", noMatchingServerError } - return s.Filesystem.SafePath(p) + return s.Filesystem().SafePath(p) } func validateDiskSpace(fs FileSystem) bool { @@ -63,7 +63,7 @@ func validateDiskSpace(fs FileSystem) bool { return false } - return s.Filesystem.HasSpaceAvailable(true) + return s.Filesystem().HasSpaceAvailable(true) } // Validates a set of credentials for a SFTP login against Pterodactyl Panel and returns