Refactor filesystem to not be dependent on a server struct
This commit is contained in:
parent
de30e2fcc9
commit
0f7bb1a371
|
@ -7,6 +7,7 @@ import (
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/pterodactyl/wings/server"
|
"github.com/pterodactyl/wings/server"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
@ -105,9 +106,9 @@ func (e *RequestError) AbortFilesystemError(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if errors.Is(e.Err, server.ErrNotEnoughDiskSpace) {
|
if errors.Is(e.Err, filesystem.ErrNotEnoughDiskSpace) {
|
||||||
c.AbortWithStatusJSON(http.StatusConflict, gin.H{
|
c.AbortWithStatusJSON(http.StatusConflict, gin.H{
|
||||||
"error": server.ErrNotEnoughDiskSpace.Error(),
|
"error": "There is not enough disk space available to perform that action.",
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -126,6 +127,13 @@ func (e *RequestError) AbortFilesystemError(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if strings.HasSuffix(e.Err.Error(), "file name too long") {
|
||||||
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
||||||
|
"error": "Cannot perform that action: file name is too long.",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
e.AbortWithServerError(c)
|
e.AbortWithServerError(c)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -70,7 +70,7 @@ func getDownloadFile(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
p, _ := s.Filesystem.SafePath(token.FilePath)
|
p, _ := s.Filesystem().SafePath(token.FilePath)
|
||||||
st, err := os.Stat(p)
|
st, err := os.Stat(p)
|
||||||
// If there is an error or we're somehow trying to download a directory, just
|
// If there is an error or we're somehow trying to download a directory, just
|
||||||
// respond with the appropriate error.
|
// respond with the appropriate error.
|
||||||
|
|
|
@ -228,7 +228,7 @@ func deleteServer(c *gin.Context) {
|
||||||
"error": errors.WithStack(err),
|
"error": errors.WithStack(err),
|
||||||
}).Warn("failed to remove server files during deletion process")
|
}).Warn("failed to remove server files during deletion process")
|
||||||
}
|
}
|
||||||
}(s.Filesystem.Path())
|
}(s.Filesystem().Path())
|
||||||
|
|
||||||
var uuid = s.Id()
|
var uuid = s.Id()
|
||||||
server.GetServers().Remove(func(s2 *server.Server) bool {
|
server.GetServers().Remove(func(s2 *server.Server) bool {
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/pterodactyl/wings/router/tokens"
|
"github.com/pterodactyl/wings/router/tokens"
|
||||||
"github.com/pterodactyl/wings/server"
|
"github.com/pterodactyl/wings/server"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -29,7 +30,7 @@ func getServerFileContents(c *gin.Context) {
|
||||||
}
|
}
|
||||||
p = "/" + strings.TrimLeft(p, "/")
|
p = "/" + strings.TrimLeft(p, "/")
|
||||||
|
|
||||||
cleaned, err := s.Filesystem.SafePath(p)
|
cleaned, err := s.Filesystem().SafePath(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
|
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
|
||||||
"error": "The file requested could not be found.",
|
"error": "The file requested could not be found.",
|
||||||
|
@ -37,7 +38,7 @@ func getServerFileContents(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
st, err := s.Filesystem.Stat(cleaned)
|
st, err := s.Filesystem().Stat(cleaned)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
TrackedServerError(err, s).AbortWithServerError(c)
|
TrackedServerError(err, s).AbortWithServerError(c)
|
||||||
return
|
return
|
||||||
|
@ -80,7 +81,7 @@ func getServerListDirectory(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
stats, err := s.Filesystem.ListDirectory(d)
|
stats, err := s.Filesystem().ListDirectory(d)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
TrackedServerError(err, s).AbortFilesystemError(c)
|
TrackedServerError(err, s).AbortFilesystemError(c)
|
||||||
return
|
return
|
||||||
|
@ -126,7 +127,7 @@ func putServerRenameFiles(c *gin.Context) {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
default:
|
default:
|
||||||
if err := s.Filesystem.Rename(pf, pt); err != nil {
|
if err := s.Filesystem().Rename(pf, pt); err != nil {
|
||||||
// Return nil if the error is an is not exists.
|
// Return nil if the error is an is not exists.
|
||||||
// NOTE: os.IsNotExist() does not work if the error is wrapped.
|
// NOTE: os.IsNotExist() does not work if the error is wrapped.
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
@ -168,7 +169,7 @@ func postServerCopyFile(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.Filesystem.Copy(data.Location); err != nil {
|
if err := s.Filesystem().Copy(data.Location); err != nil {
|
||||||
TrackedServerError(err, s).AbortFilesystemError(c)
|
TrackedServerError(err, s).AbortFilesystemError(c)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -208,7 +209,7 @@ func postServerDeleteFiles(c *gin.Context) {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
default:
|
default:
|
||||||
return s.Filesystem.Delete(pi)
|
return s.Filesystem().Delete(pi)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -232,28 +233,15 @@ func postServerWriteFile(c *gin.Context) {
|
||||||
}
|
}
|
||||||
f = "/" + strings.TrimLeft(f, "/")
|
f = "/" + strings.TrimLeft(f, "/")
|
||||||
|
|
||||||
// Check if there is enough space available to perform this action.
|
if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil {
|
||||||
if err := s.Filesystem.HasSpaceFor(c.Request.ContentLength); err != nil {
|
if errors.Is(err, filesystem.ErrIsDirectory) {
|
||||||
TrackedServerError(err, s).AbortFilesystemError(c)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.Filesystem.Writefile(f, c.Request.Body); err != nil {
|
|
||||||
if errors.Is(err, server.ErrIsDirectory) {
|
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
||||||
"error": "Cannot write file, name conflicts with an existing directory by the same name.",
|
"error": "Cannot write file, name conflicts with an existing directory by the same name.",
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasSuffix(err.Error(), "file name too long") {
|
TrackedServerError(err, s).AbortFilesystemError(c)
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
|
||||||
"error": "Cannot move or rename file, name is too long.",
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
TrackedServerError(err, s).AbortWithServerError(c)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -273,7 +261,7 @@ func postServerCreateDirectory(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.Filesystem.CreateDirectory(data.Name, data.Path); err != nil {
|
if err := s.Filesystem().CreateDirectory(data.Name, data.Path); err != nil {
|
||||||
if err.Error() == "not a directory" {
|
if err.Error() == "not a directory" {
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
||||||
"error": "Part of the path being created is not a directory (ENOTDIR).",
|
"error": "Part of the path being created is not a directory (ENOTDIR).",
|
||||||
|
@ -307,20 +295,20 @@ func postServerCompressFiles(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !s.Filesystem.HasSpaceAvailable(true) {
|
if !s.Filesystem().HasSpaceAvailable(true) {
|
||||||
c.AbortWithStatusJSON(http.StatusConflict, gin.H{
|
c.AbortWithStatusJSON(http.StatusConflict, gin.H{
|
||||||
"error": "This server does not have enough available disk space to generate a compressed archive.",
|
"error": "This server does not have enough available disk space to generate a compressed archive.",
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := s.Filesystem.CompressFiles(data.RootPath, data.Files)
|
f, err := s.Filesystem().CompressFiles(data.RootPath, data.Files)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
TrackedServerError(err, s).AbortFilesystemError(c)
|
TrackedServerError(err, s).AbortFilesystemError(c)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
c.JSON(http.StatusOK, &server.Stat{
|
c.JSON(http.StatusOK, &filesystem.Stat{
|
||||||
Info: f,
|
Info: f,
|
||||||
Mimetype: "application/tar+gzip",
|
Mimetype: "application/tar+gzip",
|
||||||
})
|
})
|
||||||
|
@ -338,10 +326,10 @@ func postServerDecompressFiles(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
hasSpace, err := s.Filesystem.SpaceAvailableForDecompression(data.RootPath, data.File)
|
hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Handle an unknown format error.
|
// Handle an unknown format error.
|
||||||
if errors.Is(err, server.ErrUnknownArchiveFormat) {
|
if errors.Is(err, filesystem.ErrUnknownArchiveFormat) {
|
||||||
s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format")
|
s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format")
|
||||||
|
|
||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
|
||||||
|
@ -361,7 +349,7 @@ func postServerDecompressFiles(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.Filesystem.DecompressFile(data.RootPath, data.File); err != nil {
|
if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil {
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
|
c.AbortWithStatusJSON(http.StatusNotFound, gin.H{
|
||||||
"error": "The requested archive was not found.",
|
"error": "The requested archive was not found.",
|
||||||
|
@ -426,13 +414,8 @@ func postServerUploadFiles(c *gin.Context) {
|
||||||
totalSize += header.Size
|
totalSize += header.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.Filesystem.HasSpaceFor(totalSize); err != nil {
|
|
||||||
TrackedServerError(err, s).AbortFilesystemError(c)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, header := range headers {
|
for _, header := range headers {
|
||||||
p, err := s.Filesystem.SafePath(filepath.Join(directory, header.Filename))
|
p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.AbortWithError(http.StatusInternalServerError, err)
|
c.AbortWithError(http.StatusInternalServerError, err)
|
||||||
return
|
return
|
||||||
|
@ -454,7 +437,7 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
if err := s.Filesystem.Writefile(p, file); err != nil {
|
if err := s.Filesystem().Writefile(p, file); err != nil {
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -67,7 +67,7 @@ func getServerArchive(c *gin.Context) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
file, err := os.Open(s.Archiver.ArchivePath())
|
file, err := os.Open(s.Archiver.Path())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
tserr := TrackedServerError(err, s)
|
tserr := TrackedServerError(err, s)
|
||||||
if !os.IsNotExist(err) {
|
if !os.IsNotExist(err) {
|
||||||
|
@ -84,7 +84,7 @@ func getServerArchive(c *gin.Context) {
|
||||||
c.Header("X-Checksum", checksum)
|
c.Header("X-Checksum", checksum)
|
||||||
c.Header("X-Mime-Type", st.Mimetype)
|
c.Header("X-Mime-Type", st.Mimetype)
|
||||||
c.Header("Content-Length", strconv.Itoa(int(st.Info.Size())))
|
c.Header("Content-Length", strconv.Itoa(int(st.Info.Size())))
|
||||||
c.Header("Content-Disposition", "attachment; filename="+s.Archiver.ArchiveName())
|
c.Header("Content-Disposition", "attachment; filename="+s.Archiver.Name())
|
||||||
c.Header("Content-Type", "application/octet-stream")
|
c.Header("Content-Type", "application/octet-stream")
|
||||||
|
|
||||||
bufio.NewReader(file).WriteTo(c.Writer)
|
bufio.NewReader(file).WriteTo(c.Writer)
|
||||||
|
@ -283,7 +283,7 @@ func postTransfer(c *gin.Context) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Un-archive the archive. That sounds weird..
|
// Un-archive the archive. That sounds weird..
|
||||||
if err := archiver.NewTarGz().Unarchive(archivePath, i.Server().Filesystem.Path()); err != nil {
|
if err := archiver.NewTarGz().Unarchive(archivePath, i.Server().Filesystem().Path()); err != nil {
|
||||||
l.WithField("error", errors.WithStack(err)).Error("failed to extract server archive")
|
l.WithField("error", errors.WithStack(err)).Error("failed to extract server archive")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -14,6 +14,7 @@ import (
|
||||||
"github.com/pterodactyl/wings/environment/docker"
|
"github.com/pterodactyl/wings/environment/docker"
|
||||||
"github.com/pterodactyl/wings/router/tokens"
|
"github.com/pterodactyl/wings/router/tokens"
|
||||||
"github.com/pterodactyl/wings/server"
|
"github.com/pterodactyl/wings/server"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -193,7 +194,7 @@ func (h *Handler) SendErrorJson(msg Message, err error, shouldLog ...bool) error
|
||||||
j := h.GetJwt()
|
j := h.GetJwt()
|
||||||
expected := errors.Is(err, server.ErrSuspended) ||
|
expected := errors.Is(err, server.ErrSuspended) ||
|
||||||
errors.Is(err, server.ErrIsRunning) ||
|
errors.Is(err, server.ErrIsRunning) ||
|
||||||
errors.Is(err, server.ErrNotEnoughDiskSpace)
|
errors.Is(err, filesystem.ErrNotEnoughDiskSpace)
|
||||||
|
|
||||||
message := "an unexpected error was encountered while handling this request"
|
message := "an unexpected error was encountered while handling this request"
|
||||||
if expected || (j != nil && j.HasPermission(PermissionReceiveErrors)) {
|
if expected || (j != nil && j.HasPermission(PermissionReceiveErrors)) {
|
||||||
|
@ -300,7 +301,7 @@ func (h *Handler) HandleInbound(m Message) error {
|
||||||
// Only send the current disk usage if the server is offline, if docker container is running,
|
// Only send the current disk usage if the server is offline, if docker container is running,
|
||||||
// Environment#EnableResourcePolling() will send this data to all clients.
|
// Environment#EnableResourcePolling() will send this data to all clients.
|
||||||
if state == environment.ProcessOfflineState {
|
if state == environment.ProcessOfflineState {
|
||||||
_ = h.server.Filesystem.HasSpaceAvailable(false)
|
_ = h.server.Filesystem().HasSpaceAvailable(false)
|
||||||
|
|
||||||
b, _ := json.Marshal(h.server.Proc())
|
b, _ := json.Marshal(h.server.Proc())
|
||||||
h.SendJson(&Message{
|
h.SendJson(&Message{
|
||||||
|
|
|
@ -4,7 +4,9 @@ import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"github.com/mholt/archiver/v3"
|
"github.com/mholt/archiver/v3"
|
||||||
|
"github.com/pkg/errors"
|
||||||
"github.com/pterodactyl/wings/config"
|
"github.com/pterodactyl/wings/config"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
|
@ -16,19 +18,19 @@ type Archiver struct {
|
||||||
Server *Server
|
Server *Server
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchivePath returns the path to the server's archive.
|
// Path returns the path to the server's archive.
|
||||||
func (a *Archiver) ArchivePath() string {
|
func (a *Archiver) Path() string {
|
||||||
return filepath.Join(config.Get().System.ArchiveDirectory, a.ArchiveName())
|
return filepath.Join(config.Get().System.ArchiveDirectory, a.Name())
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveName returns the name of the server's archive.
|
// Name returns the name of the server's archive.
|
||||||
func (a *Archiver) ArchiveName() string {
|
func (a *Archiver) Name() string {
|
||||||
return a.Server.Id() + ".tar.gz"
|
return a.Server.Id() + ".tar.gz"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists returns a boolean based off if the archive exists.
|
// Exists returns a boolean based off if the archive exists.
|
||||||
func (a *Archiver) Exists() bool {
|
func (a *Archiver) Exists() bool {
|
||||||
if _, err := os.Stat(a.ArchivePath()); os.IsNotExist(err) {
|
if _, err := os.Stat(a.Path()); os.IsNotExist(err) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,13 +38,21 @@ func (a *Archiver) Exists() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat stats the archive file.
|
// Stat stats the archive file.
|
||||||
func (a *Archiver) Stat() (*Stat, error) {
|
func (a *Archiver) Stat() (*filesystem.Stat, error) {
|
||||||
return a.Server.Filesystem.unsafeStat(a.ArchivePath())
|
s, err := os.Stat(a.Path())
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &filesystem.Stat{
|
||||||
|
Info: s,
|
||||||
|
Mimetype: "application/tar+gzip",
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Archive creates an archive of the server and deletes the previous one.
|
// Archive creates an archive of the server and deletes the previous one.
|
||||||
func (a *Archiver) Archive() error {
|
func (a *Archiver) Archive() error {
|
||||||
path := a.Server.Filesystem.Path()
|
path := a.Server.Filesystem().Path()
|
||||||
|
|
||||||
// Get the list of root files and directories to archive.
|
// Get the list of root files and directories to archive.
|
||||||
var files []string
|
var files []string
|
||||||
|
@ -52,7 +62,7 @@ func (a *Archiver) Archive() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, file := range fileInfo {
|
for _, file := range fileInfo {
|
||||||
f, err := a.Server.Filesystem.SafeJoin(path, file)
|
f, err := a.Server.Filesystem().SafeJoin(path, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -67,12 +77,12 @@ func (a *Archiver) Archive() error {
|
||||||
|
|
||||||
// Check if the file exists.
|
// Check if the file exists.
|
||||||
if stat != nil {
|
if stat != nil {
|
||||||
if err := os.Remove(a.ArchivePath()); err != nil {
|
if err := os.Remove(a.Path()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return archiver.NewTarGz().Archive(files, a.ArchivePath())
|
return archiver.NewTarGz().Archive(files, a.Path())
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteIfExists deletes the archive if it exists.
|
// DeleteIfExists deletes the archive if it exists.
|
||||||
|
@ -84,7 +94,7 @@ func (a *Archiver) DeleteIfExists() error {
|
||||||
|
|
||||||
// Check if the file exists.
|
// Check if the file exists.
|
||||||
if stat != nil {
|
if stat != nil {
|
||||||
if err := os.Remove(a.ArchivePath()); err != nil {
|
if err := os.Remove(a.Path()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -94,7 +104,7 @@ func (a *Archiver) DeleteIfExists() error {
|
||||||
|
|
||||||
// Checksum computes a SHA256 checksum of the server's archive.
|
// Checksum computes a SHA256 checksum of the server's archive.
|
||||||
func (a *Archiver) Checksum() (string, error) {
|
func (a *Archiver) Checksum() (string, error) {
|
||||||
file, err := os.Open(a.ArchivePath())
|
file, err := os.Open(a.Path())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,7 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc
|
||||||
func (s *Server) getServerwideIgnoredFiles() ([]string, error) {
|
func (s *Server) getServerwideIgnoredFiles() ([]string, error) {
|
||||||
var ignored []string
|
var ignored []string
|
||||||
|
|
||||||
f, err := os.Open(path.Join(s.Filesystem.Path(), ".pteroignore"))
|
f, err := os.Open(path.Join(s.Filesystem().Path(), ".pteroignore"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !os.IsNotExist(err) {
|
if !os.IsNotExist(err) {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -70,7 +70,7 @@ func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the included files based on the root path and the ignored files provided.
|
// Get the included files based on the root path and the ignored files provided.
|
||||||
return s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), ignored)
|
return s.Filesystem().GetIncludedFiles(s.Filesystem().Path(), ignored)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Performs a server backup and then emits the event over the server websocket. We
|
// Performs a server backup and then emits the event over the server websocket. We
|
||||||
|
@ -83,7 +83,7 @@ func (s *Server) Backup(b backup.BackupInterface) error {
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ad, err := b.Generate(inc, s.Filesystem.Path())
|
ad, err := b.Generate(inc, s.Filesystem().Path())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil {
|
if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil {
|
||||||
s.Log().WithFields(log.Fields{
|
s.Log().WithFields(log.Fields{
|
||||||
|
|
|
@ -15,7 +15,7 @@ func (s *Server) UpdateConfigurationFiles() {
|
||||||
f := cf
|
f := cf
|
||||||
|
|
||||||
pool.Submit(func() {
|
pool.Submit(func() {
|
||||||
p, err := s.Filesystem.SafePath(f.FileName)
|
p, err := s.Filesystem().SafePath(f.FileName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.Log().WithField("error", err).Error("failed to generate safe path for configuration file")
|
s.Log().WithField("error", err).Error("failed to generate safe path for configuration file")
|
||||||
|
|
||||||
|
|
1008
server/filesystem.go
1008
server/filesystem.go
File diff suppressed because it is too large
Load Diff
163
server/filesystem/compress.go
Normal file
163
server/filesystem/compress.go
Normal file
|
@ -0,0 +1,163 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/karrick/godirwalk"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/pterodactyl/wings/server/backup"
|
||||||
|
ignore "github.com/sabhiram/go-gitignore"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Given a directory, iterate through all of the files and folders within it and determine
|
||||||
|
// if they should be included in the output based on an array of ignored matches. This uses
|
||||||
|
// standard .gitignore formatting to make that determination.
|
||||||
|
//
|
||||||
|
// If no ignored files are passed through you'll get the entire directory listing.
|
||||||
|
func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.IncludedFiles, error) {
|
||||||
|
cleaned, err := fs.SafePath(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
i, err := ignore.CompileIgnoreLines(ignored...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk through all of the files and directories on a server. This callback only returns
|
||||||
|
// files found, and will keep walking deeper and deeper into directories.
|
||||||
|
inc := new(backup.IncludedFiles)
|
||||||
|
|
||||||
|
err = godirwalk.Walk(cleaned, &godirwalk.Options{
|
||||||
|
Unsorted: true,
|
||||||
|
Callback: func(p string, e *godirwalk.Dirent) error {
|
||||||
|
sp := p
|
||||||
|
if e.IsSymlink() {
|
||||||
|
sp, err = fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrBadPathResolution) {
|
||||||
|
return godirwalk.SkipThis
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only push files into the result array since archives can't create an empty directory within them.
|
||||||
|
if !e.IsDir() {
|
||||||
|
// Avoid unnecessary parsing if there are no ignored files, nothing will match anyways
|
||||||
|
// so no reason to call the function.
|
||||||
|
if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(sp, fs.Path()+"/")) {
|
||||||
|
inc.Push(sp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// We can't just abort if the path is technically ignored. It is possible there is a nested
|
||||||
|
// file or folder that should not be excluded, so in this case we need to just keep going
|
||||||
|
// until we get to a final state.
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return inc, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compresses all of the files matching the given paths in the specified directory. This function
|
||||||
|
// also supports passing nested paths to only compress certain files and folders when working in
|
||||||
|
// a larger directory. This effectively creates a local backup, but rather than ignoring specific
|
||||||
|
// files and folders, it takes an allow-list of files and folders.
|
||||||
|
//
|
||||||
|
// All paths are relative to the dir that is passed in as the first argument, and the compressed
|
||||||
|
// file will be placed at that location named `archive-{date}.tar.gz`.
|
||||||
|
func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) {
|
||||||
|
cleanedRootDir, err := fs.SafePath(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Take all of the paths passed in and merge them together with the root directory we've gotten.
|
||||||
|
for i, p := range paths {
|
||||||
|
paths[i] = filepath.Join(cleanedRootDir, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleaned, err := fs.ParallelSafePath(paths)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
inc := new(backup.IncludedFiles)
|
||||||
|
// Iterate over all of the cleaned paths and merge them into a large object of final file
|
||||||
|
// paths to pass into the archiver. As directories are encountered this will drop into them
|
||||||
|
// and look for all of the files.
|
||||||
|
for _, p := range cleaned {
|
||||||
|
f, err := os.Stat(p)
|
||||||
|
if err != nil {
|
||||||
|
fs.error(err).WithField("path", p).Debug("failed to stat file or directory for compression")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !f.IsDir() {
|
||||||
|
inc.Push(p)
|
||||||
|
} else {
|
||||||
|
err := godirwalk.Walk(p, &godirwalk.Options{
|
||||||
|
Unsorted: true,
|
||||||
|
Callback: func(p string, e *godirwalk.Dirent) error {
|
||||||
|
sp := p
|
||||||
|
if e.IsSymlink() {
|
||||||
|
// Ensure that any symlinks are properly resolved to their final destination. If
|
||||||
|
// that destination is outside the server directory skip over this entire item, otherwise
|
||||||
|
// use the resolved location for the rest of this function.
|
||||||
|
sp, err = fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrBadPathResolution) {
|
||||||
|
return godirwalk.SkipThis
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !e.IsDir() {
|
||||||
|
inc.Push(sp)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc}
|
||||||
|
d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")))
|
||||||
|
|
||||||
|
if err := a.Create(d, context.Background()); err != nil {
|
||||||
|
return nil, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.Stat(d)
|
||||||
|
if err != nil {
|
||||||
|
_ = os.Remove(d)
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := fs.hasSpaceFor(f.Size()); err != nil {
|
||||||
|
_ = os.Remove(d)
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.addDisk(f.Size())
|
||||||
|
|
||||||
|
return f, nil
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package server
|
package filesystem
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"archive/tar"
|
"archive/tar"
|
||||||
|
@ -14,14 +14,12 @@ import (
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrUnknownArchiveFormat = errors.New("filesystem: unknown archive format")
|
|
||||||
|
|
||||||
// Look through a given archive and determine if decompressing it would put the server over
|
// Look through a given archive and determine if decompressing it would put the server over
|
||||||
// its allocated disk space limit.
|
// its allocated disk space limit.
|
||||||
func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) {
|
func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) {
|
||||||
// Don't waste time trying to determine this if we know the server will have the space for
|
// Don't waste time trying to determine this if we know the server will have the space for
|
||||||
// it since there is no limit.
|
// it since there is no limit.
|
||||||
if fs.Server.DiskSpace() <= 0 {
|
if fs.MaxDisk() <= 0 {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,18 +33,18 @@ func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (b
|
||||||
dirSize, err := fs.DiskUsage(false)
|
dirSize, err := fs.DiskUsage(false)
|
||||||
|
|
||||||
var size int64
|
var size int64
|
||||||
var max = fs.Server.DiskSpace()
|
|
||||||
// Walk over the archive and figure out just how large the final output would be from unarchiving it.
|
// Walk over the archive and figure out just how large the final output would be from unarchiving it.
|
||||||
err = archiver.Walk(source, func(f archiver.File) error {
|
err = archiver.Walk(source, func(f archiver.File) error {
|
||||||
if atomic.AddInt64(&size, f.Size())+dirSize > max {
|
if atomic.AddInt64(&size, f.Size())+dirSize > fs.MaxDisk() {
|
||||||
return errors.WithStack(ErrNotEnoughDiskSpace)
|
return ErrNotEnoughDiskSpace
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.HasPrefix(err.Error(), "format ") {
|
if strings.HasPrefix(err.Error(), "format ") {
|
||||||
return false, errors.WithStack(ErrUnknownArchiveFormat)
|
return false, ErrUnknownArchiveFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, errors.WithStack(err)
|
return false, errors.WithStack(err)
|
210
server/filesystem/disk_space.go
Normal file
210
server/filesystem/disk_space.go
Normal file
|
@ -0,0 +1,210 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/apex/log"
|
||||||
|
"github.com/karrick/godirwalk"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SpaceCheckingOpts struct {
|
||||||
|
AllowStaleResponse bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type usageLookupTime struct {
|
||||||
|
sync.RWMutex
|
||||||
|
value time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update the last time that a disk space lookup was performed.
|
||||||
|
func (ult *usageLookupTime) Set(t time.Time) {
|
||||||
|
ult.Lock()
|
||||||
|
ult.value = t
|
||||||
|
ult.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the last time that we performed a disk space usage lookup.
|
||||||
|
func (ult *usageLookupTime) Get() time.Time {
|
||||||
|
ult.RLock()
|
||||||
|
defer ult.RUnlock()
|
||||||
|
|
||||||
|
return ult.value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the maximum amount of disk space that this Filesystem instance is allowed to use.
|
||||||
|
func (fs *Filesystem) MaxDisk() int64 {
|
||||||
|
fs.mu.RLock()
|
||||||
|
defer fs.mu.RUnlock()
|
||||||
|
|
||||||
|
return fs.diskLimit
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sets the disk space limit for this Filesystem instance.
|
||||||
|
func (fs *Filesystem) SetDiskLimit(i int64) {
|
||||||
|
fs.mu.Lock()
|
||||||
|
fs.diskLimit = i
|
||||||
|
fs.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determines if the directory a file is trying to be added to has enough space available
|
||||||
|
// for the file to be written to.
|
||||||
|
//
|
||||||
|
// Because determining the amount of space being used by a server is a taxing operation we
|
||||||
|
// will load it all up into a cache and pull from that as long as the key is not expired.
|
||||||
|
//
|
||||||
|
// This operation will potentially block unless allowStaleValue is set to true. See the
|
||||||
|
// documentation on DiskUsage for how this affects the call.
|
||||||
|
func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool {
|
||||||
|
size, err := fs.DiskUsage(allowStaleValue)
|
||||||
|
if err != nil {
|
||||||
|
log.WithField("root", fs.root).WithField("error", err).Warn("failed to determine root fs directory size")
|
||||||
|
}
|
||||||
|
|
||||||
|
// If space is -1 or 0 just return true, means they're allowed unlimited.
|
||||||
|
//
|
||||||
|
// Technically we could skip disk space calculation because we don't need to check if the
|
||||||
|
// server exceeds it's limit but because this method caches the disk usage it would be best
|
||||||
|
// to calculate the disk usage and always return true.
|
||||||
|
if fs.MaxDisk() == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return size <= fs.MaxDisk()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Internal helper function to allow other parts of the codebase to check the total used disk space
|
||||||
|
// as needed without overly taxing the system. This will prioritize the value from the cache to avoid
|
||||||
|
// excessive IO usage. We will only walk the filesystem and determine the size of the directory if there
|
||||||
|
// is no longer a cached value.
|
||||||
|
//
|
||||||
|
// If "allowStaleValue" is set to true, a stale value MAY be returned to the caller if there is an
|
||||||
|
// expired cache value AND there is currently another lookup in progress. If there is no cached value but
|
||||||
|
// no other lookup is in progress, a fresh disk space response will be returned to the caller.
|
||||||
|
//
|
||||||
|
// This is primarily to avoid a bunch of I/O operations from piling up on the server, especially on servers
|
||||||
|
// with a large amount of files.
|
||||||
|
func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) {
|
||||||
|
if !fs.lastLookupTime.Get().After(time.Now().Add(time.Second * fs.diskCheckInterval * -1)) {
|
||||||
|
// If we are now allowing a stale response go ahead and perform the lookup and return the fresh
|
||||||
|
// value. This is a blocking operation to the calling process.
|
||||||
|
if !allowStaleValue {
|
||||||
|
return fs.updateCachedDiskUsage()
|
||||||
|
} else if !fs.lookupInProgress.Get() {
|
||||||
|
// Otherwise, if we allow a stale value and there isn't a valid item in the cache and we aren't
|
||||||
|
// currently performing a lookup, just do the disk usage calculation in the background.
|
||||||
|
go func(fs *Filesystem) {
|
||||||
|
if _, err := fs.updateCachedDiskUsage(); err != nil {
|
||||||
|
log.WithField("root", fs.root).WithField("error", err).Warn("failed to update fs disk usage from within routine")
|
||||||
|
}
|
||||||
|
}(fs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the currently cached value back to the calling function.
|
||||||
|
return atomic.LoadInt64(&fs.diskUsed), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Updates the currently used disk space for a server.
|
||||||
|
func (fs *Filesystem) updateCachedDiskUsage() (int64, error) {
|
||||||
|
// Obtain an exclusive lock on this process so that we don't unintentionally run it at the same
|
||||||
|
// time as another running process. Once the lock is available it'll read from the cache for the
|
||||||
|
// second call rather than hitting the disk in parallel.
|
||||||
|
fs.mu.Lock()
|
||||||
|
defer fs.mu.Unlock()
|
||||||
|
|
||||||
|
// Signal that we're currently updating the disk size so that other calls to the disk checking
|
||||||
|
// functions can determine if they should queue up additional calls to this function. Ensure that
|
||||||
|
// we always set this back to "false" when this process is done executing.
|
||||||
|
fs.lookupInProgress.Set(true)
|
||||||
|
defer fs.lookupInProgress.Set(false)
|
||||||
|
|
||||||
|
// If there is no size its either because there is no data (in which case running this function
|
||||||
|
// will have effectively no impact), or there is nothing in the cache, in which case we need to
|
||||||
|
// grab the size of their data directory. This is a taxing operation, so we want to store it in
|
||||||
|
// the cache once we've gotten it.
|
||||||
|
size, err := fs.DirectorySize("/")
|
||||||
|
|
||||||
|
// Always cache the size, even if there is an error. We want to always return that value
|
||||||
|
// so that we don't cause an endless loop of determining the disk size if there is a temporary
|
||||||
|
// error encountered.
|
||||||
|
fs.lastLookupTime.Set(time.Now())
|
||||||
|
|
||||||
|
atomic.StoreInt64(&fs.diskUsed, size)
|
||||||
|
|
||||||
|
return size, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determines the directory size of a given location by running parallel tasks to iterate
|
||||||
|
// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation
|
||||||
|
// on locations with tons of files, so it is recommended that you cache the output.
|
||||||
|
func (fs *Filesystem) DirectorySize(dir string) (int64, error) {
|
||||||
|
d, err := fs.SafePath(dir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var size int64
|
||||||
|
var st syscall.Stat_t
|
||||||
|
|
||||||
|
err = godirwalk.Walk(d, &godirwalk.Options{
|
||||||
|
Unsorted: true,
|
||||||
|
Callback: func(p string, e *godirwalk.Dirent) error {
|
||||||
|
// If this is a symlink then resolve the final destination of it before trying to continue walking
|
||||||
|
// over its contents. If it resolves outside the server data directory just skip everything else for
|
||||||
|
// it. Otherwise, allow it to continue.
|
||||||
|
if e.IsSymlink() {
|
||||||
|
if _, err := fs.SafePath(p); err != nil {
|
||||||
|
if errors.Is(err, ErrBadPathResolution) {
|
||||||
|
return godirwalk.SkipThis
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !e.IsDir() {
|
||||||
|
syscall.Lstat(p, &st)
|
||||||
|
atomic.AddInt64(&size, st.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return size, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to determine if a server has space available for a file of a given size.
|
||||||
|
// If space is available, no error will be returned, otherwise an ErrNotEnoughSpace error
|
||||||
|
// will be raised.
|
||||||
|
func (fs *Filesystem) hasSpaceFor(size int64) error {
|
||||||
|
if fs.MaxDisk() == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s, err := fs.DiskUsage(true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if (s + size) > fs.MaxDisk() {
|
||||||
|
return ErrNotEnoughDiskSpace
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Updates the disk usage for the Filesystem instance.
|
||||||
|
func (fs *Filesystem) addDisk(i int64) int64 {
|
||||||
|
size, _ := fs.DiskUsage(true)
|
||||||
|
|
||||||
|
// If we're dropping below 0 somehow just cap it to 0.
|
||||||
|
if (size + i) < 0 {
|
||||||
|
return atomic.SwapInt64(&fs.diskUsed, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
return atomic.AddInt64(&fs.diskUsed, i)
|
||||||
|
}
|
35
server/filesystem/errors.go
Normal file
35
server/filesystem/errors.go
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/apex/log"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrIsDirectory = errors.New("filesystem: is a directory")
|
||||||
|
var ErrNotEnoughDiskSpace = errors.New("filesystem: not enough disk space")
|
||||||
|
var ErrBadPathResolution = errors.New("filesystem: invalid path resolution")
|
||||||
|
var ErrUnknownArchiveFormat = errors.New("filesystem: unknown archive format")
|
||||||
|
|
||||||
|
// Generates an error logger instance with some basic information.
|
||||||
|
func (fs *Filesystem) error(err error) *log.Entry {
|
||||||
|
return log.WithField("subsystem", "filesystem").WithField("root", fs.root).WithField("error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle errors encountered when walking through directories.
|
||||||
|
//
|
||||||
|
// If there is a path resolution error just skip the item entirely. Only return this for a
|
||||||
|
// directory, otherwise return nil. Returning this error for a file will stop the walking
|
||||||
|
// for the remainder of the directory. This is assuming an os.FileInfo struct was even returned.
|
||||||
|
func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error {
|
||||||
|
if !errors.Is(err, ErrBadPathResolution) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if f != nil && f.IsDir() {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
453
server/filesystem/filesystem.go
Normal file
453
server/filesystem/filesystem.go
Normal file
|
@ -0,0 +1,453 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"github.com/gabriel-vasile/mimetype"
|
||||||
|
"github.com/karrick/godirwalk"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/pterodactyl/wings/config"
|
||||||
|
"github.com/pterodactyl/wings/system"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Filesystem struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
lastLookupTime *usageLookupTime
|
||||||
|
lookupInProgress system.AtomicBool
|
||||||
|
diskUsed int64
|
||||||
|
diskCheckInterval time.Duration
|
||||||
|
|
||||||
|
// The maximum amount of disk space (in bytes) that this Filesystem instance can use.
|
||||||
|
diskLimit int64
|
||||||
|
|
||||||
|
// The root data directory path for this Filesystem instance.
|
||||||
|
root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Creates a new Filesystem instance for a given server.
|
||||||
|
func New(root string, size int64) *Filesystem {
|
||||||
|
return &Filesystem{
|
||||||
|
root: root,
|
||||||
|
diskLimit: size,
|
||||||
|
diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval),
|
||||||
|
lastLookupTime: &usageLookupTime{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the root path for the Filesystem instance.
|
||||||
|
func (fs *Filesystem) Path() string {
|
||||||
|
return fs.root
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reads a file on the system and returns it as a byte representation in a file
|
||||||
|
// reader. This is not the most memory efficient usage since it will be reading the
|
||||||
|
// entirety of the file into memory.
|
||||||
|
func (fs *Filesystem) Readfile(p string) (io.Reader, error) {
|
||||||
|
cleaned, err := fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := ioutil.ReadFile(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return bytes.NewReader(b), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writes a file to the system. If the file does not already exist one will be created.
|
||||||
|
func (fs *Filesystem) Writefile(p string, r io.Reader) error {
|
||||||
|
cleaned, err := fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var currentSize int64
|
||||||
|
// If the file does not exist on the system already go ahead and create the pathway
|
||||||
|
// to it and an empty file. We'll then write to it later on after this completes.
|
||||||
|
if stat, err := os.Stat(cleaned); err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := fs.Chown(filepath.Dir(cleaned)); err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if stat.IsDir() {
|
||||||
|
return ErrIsDirectory
|
||||||
|
}
|
||||||
|
|
||||||
|
currentSize = stat.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
br := bufio.NewReader(r)
|
||||||
|
// Check that the new size we're writing to the disk can fit. If there is currently a file
|
||||||
|
// we'll subtract that current file size from the size of the buffer to determine the amount
|
||||||
|
// of new data we're writing (or amount we're removing if smaller).
|
||||||
|
if err := fs.hasSpaceFor(int64(br.Size()) - currentSize); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
o := &fileOpener{}
|
||||||
|
// This will either create the file if it does not already exist, or open and
|
||||||
|
// truncate the existing file.
|
||||||
|
file, err := o.open(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
buf := make([]byte, 1024*4)
|
||||||
|
sz, err := io.CopyBuffer(file, r, buf)
|
||||||
|
|
||||||
|
// Adjust the disk usage to account for the old size and the new size of the file.
|
||||||
|
fs.addDisk(sz - currentSize)
|
||||||
|
|
||||||
|
// Finally, chown the file to ensure the permissions don't end up out-of-whack
|
||||||
|
// if we had just created it.
|
||||||
|
return fs.Chown(cleaned)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Creates a new directory (name) at a specified path (p) for the server.
|
||||||
|
func (fs *Filesystem) CreateDirectory(name string, p string) error {
|
||||||
|
cleaned, err := fs.SafePath(path.Join(p, name))
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.MkdirAll(cleaned, 0755)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Moves (or renames) a file or directory.
|
||||||
|
func (fs *Filesystem) Rename(from string, to string) error {
|
||||||
|
cleanedFrom, err := fs.SafePath(from)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanedTo, err := fs.SafePath(to)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the target file or directory already exists the rename function will fail, so just
|
||||||
|
// bail out now.
|
||||||
|
if _, err := os.Stat(cleanedTo); err == nil {
|
||||||
|
return os.ErrExist
|
||||||
|
}
|
||||||
|
|
||||||
|
if cleanedTo == fs.Path() {
|
||||||
|
return errors.New("attempting to rename into an invalid directory space")
|
||||||
|
}
|
||||||
|
|
||||||
|
d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo))
|
||||||
|
// Ensure that the directory we're moving into exists correctly on the system. Only do this if
|
||||||
|
// we're not at the root directory level.
|
||||||
|
if d != fs.Path() {
|
||||||
|
if mkerr := os.MkdirAll(d, 0644); mkerr != nil {
|
||||||
|
return errors.Wrap(mkerr, "failed to create directory structure for file rename")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.Rename(cleanedFrom, cleanedTo)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recursively iterates over a file or directory and sets the permissions on all of the
|
||||||
|
// underlying files. Iterate over all of the files and directories. If it is a file just
|
||||||
|
// go ahead and perform the chown operation. Otherwise dig deeper into the directory until
|
||||||
|
// we've run out of directories to dig into.
|
||||||
|
func (fs *Filesystem) Chown(path string) error {
|
||||||
|
cleaned, err := fs.SafePath(path)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
uid := config.Get().System.User.Uid
|
||||||
|
gid := config.Get().System.User.Gid
|
||||||
|
|
||||||
|
// Start by just chowning the initial path that we received.
|
||||||
|
if err := os.Chown(cleaned, uid, gid); err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If this is not a directory we can now return from the function, there is nothing
|
||||||
|
// left that we need to do.
|
||||||
|
if st, _ := os.Stat(cleaned); !st.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// If this was a directory, begin walking over its contents recursively and ensure that all
|
||||||
|
// of the subfiles and directories get their permissions updated as well.
|
||||||
|
return godirwalk.Walk(cleaned, &godirwalk.Options{
|
||||||
|
Unsorted: true,
|
||||||
|
Callback: func(p string, e *godirwalk.Dirent) error {
|
||||||
|
// Do not attempt to chmod a symlink. Go's os.Chown function will affect the symlink
|
||||||
|
// so if it points to a location outside the data directory the user would be able to
|
||||||
|
// (un)intentionally modify that files permissions.
|
||||||
|
if e.IsSymlink() {
|
||||||
|
if e.IsDir() {
|
||||||
|
return godirwalk.SkipThis
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.Chown(p, uid, gid)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copies a given file to the same location and appends a suffix to the file to indicate that
|
||||||
|
// it has been copied.
|
||||||
|
func (fs *Filesystem) Copy(p string) error {
|
||||||
|
cleaned, err := fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s, err := os.Stat(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
} else if s.IsDir() || !s.Mode().IsRegular() {
|
||||||
|
// If this is a directory or not a regular file, just throw a not-exist error
|
||||||
|
// since anything calling this function should understand what that means.
|
||||||
|
return os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that copying this file wouldn't put the server over its limit.
|
||||||
|
if err := fs.hasSpaceFor(s.Size()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
base := filepath.Base(cleaned)
|
||||||
|
relative := strings.TrimSuffix(strings.TrimPrefix(cleaned, fs.Path()), base)
|
||||||
|
extension := filepath.Ext(base)
|
||||||
|
name := strings.TrimSuffix(base, extension)
|
||||||
|
|
||||||
|
// Ensure that ".tar" is also counted as apart of the file extension.
|
||||||
|
// There might be a better way to handle this for other double file extensions,
|
||||||
|
// but this is a good workaround for now.
|
||||||
|
if strings.HasSuffix(name, ".tar") {
|
||||||
|
extension = ".tar" + extension
|
||||||
|
name = strings.TrimSuffix(name, ".tar")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Begin looping up to 50 times to try and create a unique copy file name. This will take
|
||||||
|
// an input of "file.txt" and generate "file copy.txt". If that name is already taken, it will
|
||||||
|
// then try to write "file copy 2.txt" and so on, until reaching 50 loops. At that point we
|
||||||
|
// won't waste anymore time, just use the current timestamp and make that copy.
|
||||||
|
//
|
||||||
|
// Could probably make this more efficient by checking if there are any files matching the copy
|
||||||
|
// pattern, and trying to find the highest number and then incrementing it by one rather than
|
||||||
|
// looping endlessly.
|
||||||
|
var i int
|
||||||
|
copySuffix := " copy"
|
||||||
|
for i = 0; i < 51; i++ {
|
||||||
|
if i > 0 {
|
||||||
|
copySuffix = " copy " + strconv.Itoa(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
tryName := fmt.Sprintf("%s%s%s", name, copySuffix, extension)
|
||||||
|
tryLocation, err := fs.SafePath(path.Join(relative, tryName))
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the file exists, continue to the next loop, otherwise we're good to start a copy.
|
||||||
|
if _, err := os.Stat(tryLocation); err != nil && !os.IsNotExist(err) {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if i == 50 {
|
||||||
|
copySuffix = "." + time.Now().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
finalPath, err := fs.SafePath(path.Join(relative, fmt.Sprintf("%s%s%s", name, copySuffix, extension)))
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
source, err := os.Open(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
defer source.Close()
|
||||||
|
|
||||||
|
dest, err := os.Create(finalPath)
|
||||||
|
if err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
defer dest.Close()
|
||||||
|
|
||||||
|
buf := make([]byte, 1024*4)
|
||||||
|
if _, err := io.CopyBuffer(dest, source, buf); err != nil {
|
||||||
|
return errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once everything is done, increment the disk space used.
|
||||||
|
fs.addDisk(s.Size())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deletes a file or folder from the system. Prevents the user from accidentally
|
||||||
|
// (or maliciously) removing their root server data directory.
|
||||||
|
func (fs *Filesystem) Delete(p string) error {
|
||||||
|
// This is one of the few (only?) places in the codebase where we're explicitly not using
|
||||||
|
// the SafePath functionality when working with user provided input. If we did, you would
|
||||||
|
// not be able to delete a file that is a symlink pointing to a location outside of the data
|
||||||
|
// directory.
|
||||||
|
//
|
||||||
|
// We also want to avoid resolving a symlink that points _within_ the data directory and thus
|
||||||
|
// deleting the actual source file for the symlink rather than the symlink itself. For these
|
||||||
|
// purposes just resolve the actual file path using filepath.Join() and confirm that the path
|
||||||
|
// exists within the data directory.
|
||||||
|
resolved := fs.unsafeFilePath(p)
|
||||||
|
if !fs.unsafeIsInDataDirectory(resolved) {
|
||||||
|
return ErrBadPathResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
// Block any whoopsies.
|
||||||
|
if resolved == fs.Path() {
|
||||||
|
return errors.New("cannot delete root server directory")
|
||||||
|
}
|
||||||
|
|
||||||
|
if st, err := os.Stat(resolved); err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
fs.error(err).Warn("error while attempting to stat file before deletion")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if !st.IsDir() {
|
||||||
|
fs.addDisk(-st.Size())
|
||||||
|
} else {
|
||||||
|
go func(st os.FileInfo, resolved string) {
|
||||||
|
if s, err := fs.DirectorySize(resolved); err == nil {
|
||||||
|
fs.addDisk(-s)
|
||||||
|
}
|
||||||
|
}(st, resolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return os.RemoveAll(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
type fileOpener struct {
|
||||||
|
busy uint
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file
|
||||||
|
// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts
|
||||||
|
// has been exhaused, at which point we will abort with an error.
|
||||||
|
func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) {
|
||||||
|
for {
|
||||||
|
f, err := os.OpenFile(path, flags, perm)
|
||||||
|
|
||||||
|
// If there is an error because the text file is busy, go ahead and sleep for a few
|
||||||
|
// hundred milliseconds and then try again up to three times before just returning the
|
||||||
|
// error back to the caller.
|
||||||
|
//
|
||||||
|
// Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122
|
||||||
|
if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") {
|
||||||
|
time.Sleep(100 * time.Millisecond << fo.busy)
|
||||||
|
fo.busy++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lists the contents of a given directory and returns stat information about each
|
||||||
|
// file and folder within it.
|
||||||
|
func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) {
|
||||||
|
cleaned, err := fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := ioutil.ReadDir(cleaned)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
// You must initialize the output of this directory as a non-nil value otherwise
|
||||||
|
// when it is marshaled into a JSON object you'll just get 'null' back, which will
|
||||||
|
// break the panel badly.
|
||||||
|
out := make([]*Stat, len(files))
|
||||||
|
|
||||||
|
// Iterate over all of the files and directories returned and perform an async process
|
||||||
|
// to get the mime-type for them all.
|
||||||
|
for i, file := range files {
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func(idx int, f os.FileInfo) {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
|
var m *mimetype.MIME
|
||||||
|
var d = "inode/directory"
|
||||||
|
if !f.IsDir() {
|
||||||
|
cleanedp, _ := fs.SafeJoin(cleaned, f)
|
||||||
|
if cleanedp != "" {
|
||||||
|
m, _ = mimetype.DetectFile(filepath.Join(cleaned, f.Name()))
|
||||||
|
} else {
|
||||||
|
// Just pass this for an unknown type because the file could not safely be resolved within
|
||||||
|
// the server data path.
|
||||||
|
d = "application/octet-stream"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st := &Stat{
|
||||||
|
Info: f,
|
||||||
|
Mimetype: d,
|
||||||
|
}
|
||||||
|
|
||||||
|
if m != nil {
|
||||||
|
st.Mimetype = m.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
out[idx] = st
|
||||||
|
}(i, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
// Sort the output alphabetically to begin with since we've run the output
|
||||||
|
// through an asynchronous process and the order is gonna be very random.
|
||||||
|
sort.SliceStable(out, func(i, j int) bool {
|
||||||
|
if out[i].Info.Name() == out[j].Info.Name() || out[i].Info.Name() > out[j].Info.Name() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
})
|
||||||
|
|
||||||
|
// Then, sort it so that directories are listed first in the output. Everything
|
||||||
|
// will continue to be alphabetized at this point.
|
||||||
|
sort.SliceStable(out, func(i, j int) bool {
|
||||||
|
return out[i].Info.IsDir()
|
||||||
|
})
|
||||||
|
|
||||||
|
return out, nil
|
||||||
|
}
|
162
server/filesystem/path.go
Normal file
162
server/filesystem/path.go
Normal file
|
@ -0,0 +1,162 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Normalizes a directory being passed in to ensure the user is not able to escape
|
||||||
|
// from their data directory. After normalization if the directory is still within their home
|
||||||
|
// path it is returned. If they managed to "escape" an error will be returned.
|
||||||
|
//
|
||||||
|
// This logic is actually copied over from the SFTP server code. Ideally that eventually
|
||||||
|
// either gets ported into this application, or is able to make use of this package.
|
||||||
|
func (fs *Filesystem) SafePath(p string) (string, error) {
|
||||||
|
var nonExistentPathResolution string
|
||||||
|
|
||||||
|
// Start with a cleaned up path before checking the more complex bits.
|
||||||
|
r := fs.unsafeFilePath(p)
|
||||||
|
|
||||||
|
// At the same time, evaluate the symlink status and determine where this file or folder
|
||||||
|
// is truly pointing to.
|
||||||
|
p, err := filepath.EvalSymlinks(r)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", err
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
// The requested directory doesn't exist, so at this point we need to iterate up the
|
||||||
|
// path chain until we hit a directory that _does_ exist and can be validated.
|
||||||
|
parts := strings.Split(filepath.Dir(r), "/")
|
||||||
|
|
||||||
|
var try string
|
||||||
|
// Range over all of the path parts and form directory pathings from the end
|
||||||
|
// moving up until we have a valid resolution or we run out of paths to try.
|
||||||
|
for k := range parts {
|
||||||
|
try = strings.Join(parts[:(len(parts)-k)], "/")
|
||||||
|
|
||||||
|
if !fs.unsafeIsInDataDirectory(try) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
t, err := filepath.EvalSymlinks(try)
|
||||||
|
if err == nil {
|
||||||
|
nonExistentPathResolution = t
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the new path doesn't start with their root directory there is clearly an escape
|
||||||
|
// attempt going on, and we should NOT resolve this path for them.
|
||||||
|
if nonExistentPathResolution != "" {
|
||||||
|
if !fs.unsafeIsInDataDirectory(nonExistentPathResolution) {
|
||||||
|
return "", ErrBadPathResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the nonExistentPathResolution variable is not empty then the initial path requested
|
||||||
|
// did not exist and we looped through the pathway until we found a match. At this point
|
||||||
|
// we've confirmed the first matched pathway exists in the root server directory, so we
|
||||||
|
// can go ahead and just return the path that was requested initially.
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the requested directory from EvalSymlinks begins with the server root directory go
|
||||||
|
// ahead and return it. If not we'll return an error which will block any further action
|
||||||
|
// on the file.
|
||||||
|
if fs.unsafeIsInDataDirectory(p) {
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", ErrBadPathResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate a path to the file by cleaning it up and appending the root server path to it. This
|
||||||
|
// DOES NOT guarantee that the file resolves within the server data directory. You'll want to use
|
||||||
|
// the fs.unsafeIsInDataDirectory(p) function to confirm.
|
||||||
|
func (fs *Filesystem) unsafeFilePath(p string) string {
|
||||||
|
// Calling filepath.Clean on the joined directory will resolve it to the absolute path,
|
||||||
|
// removing any ../ type of resolution arguments, and leaving us with a direct path link.
|
||||||
|
//
|
||||||
|
// This will also trim the existing root path off the beginning of the path passed to
|
||||||
|
// the function since that can get a bit messy.
|
||||||
|
return filepath.Clean(filepath.Join(fs.Path(), strings.TrimPrefix(p, fs.Path())))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that that path string starts with the server data directory path. This function DOES NOT
|
||||||
|
// validate that the rest of the path does not end up resolving out of this directory, or that the
|
||||||
|
// targeted file or folder is not a symlink doing the same thing.
|
||||||
|
func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool {
|
||||||
|
return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to keep some of the codebase a little cleaner. Returns a "safe" version of the path
|
||||||
|
// joined with a file. This is important because you cannot just assume that appending a file to a cleaned
|
||||||
|
// path will result in a cleaned path to that file. For example, imagine you have the following scenario:
|
||||||
|
//
|
||||||
|
// my_bad_file -> symlink:/etc/passwd
|
||||||
|
//
|
||||||
|
// cleaned := SafePath("../../etc") -> "/"
|
||||||
|
// filepath.Join(cleaned, my_bad_file) -> "/my_bad_file"
|
||||||
|
//
|
||||||
|
// You might think that "/my_bad_file" is fine since it isn't pointing to the original "../../etc/my_bad_file".
|
||||||
|
// However, this doesn't account for symlinks where the file might be pointing outside of the directory, so
|
||||||
|
// calling a function such as Chown against it would chown the symlinked location, and not the file within the
|
||||||
|
// Wings daemon.
|
||||||
|
func (fs *Filesystem) SafeJoin(dir string, f os.FileInfo) (string, error) {
|
||||||
|
if f.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fs.SafePath(filepath.Join(dir, f.Name()))
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(dir, f.Name()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Executes the fs.SafePath function in parallel against an array of paths. If any of the calls
|
||||||
|
// fails an error will be returned.
|
||||||
|
func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) {
|
||||||
|
var cleaned []string
|
||||||
|
|
||||||
|
// Simple locker function to avoid racy appends to the array of cleaned paths.
|
||||||
|
var m = new(sync.Mutex)
|
||||||
|
var push = func(c string) {
|
||||||
|
m.Lock()
|
||||||
|
cleaned = append(cleaned, c)
|
||||||
|
m.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an error group that we can use to run processes in parallel while retaining
|
||||||
|
// the ability to cancel the entire process immediately should any of it fail.
|
||||||
|
g, ctx := errgroup.WithContext(context.Background())
|
||||||
|
|
||||||
|
// Iterate over all of the paths and generate a cleaned path, if there is an error for any
|
||||||
|
// of the files, abort the process.
|
||||||
|
for _, p := range paths {
|
||||||
|
// Create copy so we can use it within the goroutine correctly.
|
||||||
|
pi := p
|
||||||
|
|
||||||
|
// Recursively call this function to continue digging through the directory tree within
|
||||||
|
// a separate goroutine. If the context is canceled abort this process.
|
||||||
|
g.Go(func() error {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
// If the callback returns true, go ahead and keep walking deeper. This allows
|
||||||
|
// us to programmatically continue deeper into directories, or stop digging
|
||||||
|
// if that pathway knows it needs nothing else.
|
||||||
|
if c, err := fs.SafePath(pi); err != nil {
|
||||||
|
return err
|
||||||
|
} else {
|
||||||
|
push(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Block until all of the routines finish and have returned a value.
|
||||||
|
return cleaned, g.Wait()
|
||||||
|
}
|
74
server/filesystem/stat.go
Normal file
74
server/filesystem/stat.go
Normal file
|
@ -0,0 +1,74 @@
|
||||||
|
package filesystem
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"github.com/gabriel-vasile/mimetype"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Stat struct {
|
||||||
|
Info os.FileInfo
|
||||||
|
Mimetype string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Stat) MarshalJSON() ([]byte, error) {
|
||||||
|
return json.Marshal(struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Created string `json:"created"`
|
||||||
|
Modified string `json:"modified"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
Directory bool `json:"directory"`
|
||||||
|
File bool `json:"file"`
|
||||||
|
Symlink bool `json:"symlink"`
|
||||||
|
Mime string `json:"mime"`
|
||||||
|
}{
|
||||||
|
Name: s.Info.Name(),
|
||||||
|
Created: s.CTime().Format(time.RFC3339),
|
||||||
|
Modified: s.Info.ModTime().Format(time.RFC3339),
|
||||||
|
Mode: s.Info.Mode().String(),
|
||||||
|
Size: s.Info.Size(),
|
||||||
|
Directory: s.Info.IsDir(),
|
||||||
|
File: !s.Info.IsDir(),
|
||||||
|
Symlink: s.Info.Mode().Perm()&os.ModeSymlink != 0,
|
||||||
|
Mime: s.Mimetype,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats a file or folder and returns the base stat object from go along with the
|
||||||
|
// MIME data that can be used for editing files.
|
||||||
|
func (fs *Filesystem) Stat(p string) (*Stat, error) {
|
||||||
|
cleaned, err := fs.SafePath(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fs.unsafeStat(cleaned)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *Filesystem) unsafeStat(p string) (*Stat, error) {
|
||||||
|
s, err := os.Stat(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var m *mimetype.MIME
|
||||||
|
if !s.IsDir() {
|
||||||
|
m, err = mimetype.DetectFile(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
st := &Stat{
|
||||||
|
Info: s,
|
||||||
|
Mimetype: "inode/directory",
|
||||||
|
}
|
||||||
|
|
||||||
|
if m != nil {
|
||||||
|
st.Mimetype = m.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
return st, nil
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package server
|
package filesystem
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"syscall"
|
"syscall"
|
|
@ -1,4 +1,4 @@
|
||||||
package server
|
package filesystem
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"syscall"
|
"syscall"
|
|
@ -1,4 +1,4 @@
|
||||||
package server
|
package filesystem
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
|
@ -409,7 +409,7 @@ func (ip *InstallationProcess) Execute() (string, error) {
|
||||||
Mounts: []mount.Mount{
|
Mounts: []mount.Mount{
|
||||||
{
|
{
|
||||||
Target: "/mnt/server",
|
Target: "/mnt/server",
|
||||||
Source: ip.Server.Filesystem.Path(),
|
Source: ip.Server.Filesystem().Path(),
|
||||||
Type: mount.TypeBind,
|
Type: mount.TypeBind,
|
||||||
ReadOnly: false,
|
ReadOnly: false,
|
||||||
},
|
},
|
||||||
|
|
|
@ -81,7 +81,7 @@ func (s *Server) StartEventListeners() {
|
||||||
s.resources.Stats = *st
|
s.resources.Stats = *st
|
||||||
s.resources.mu.Unlock()
|
s.resources.mu.Unlock()
|
||||||
|
|
||||||
s.Filesystem.HasSpaceAvailable(true)
|
s.Filesystem().HasSpaceAvailable(true)
|
||||||
|
|
||||||
s.emitProcUsage()
|
s.emitProcUsage()
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,9 +7,12 @@ import (
|
||||||
"github.com/gammazero/workerpool"
|
"github.com/gammazero/workerpool"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/pterodactyl/wings/api"
|
"github.com/pterodactyl/wings/api"
|
||||||
|
"github.com/pterodactyl/wings/config"
|
||||||
"github.com/pterodactyl/wings/environment"
|
"github.com/pterodactyl/wings/environment"
|
||||||
"github.com/pterodactyl/wings/environment/docker"
|
"github.com/pterodactyl/wings/environment/docker"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
@ -90,7 +93,7 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) {
|
||||||
defaults.Set(&s.resources)
|
defaults.Set(&s.resources)
|
||||||
|
|
||||||
s.Archiver = Archiver{Server: s}
|
s.Archiver = Archiver{Server: s}
|
||||||
s.Filesystem = Filesystem{Server: s}
|
s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace())
|
||||||
|
|
||||||
// Right now we only support a Docker based environment, so I'm going to hard code
|
// Right now we only support a Docker based environment, so I'm going to hard code
|
||||||
// this logic in. When we're ready to support other environment we'll need to make
|
// this logic in. When we're ready to support other environment we'll need to make
|
||||||
|
@ -120,8 +123,8 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the server's data directory exists, force disk usage calculation.
|
// If the server's data directory exists, force disk usage calculation.
|
||||||
if _, err := os.Stat(s.Filesystem.Path()); err == nil {
|
if _, err := os.Stat(s.Filesystem().Path()); err == nil {
|
||||||
s.Filesystem.HasSpaceAvailable(true)
|
s.Filesystem().HasSpaceAvailable(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
|
|
|
@ -24,7 +24,7 @@ func (s *Server) Mounts() []environment.Mount {
|
||||||
m = append(m, environment.Mount{
|
m = append(m, environment.Mount{
|
||||||
Default: true,
|
Default: true,
|
||||||
Target: "/home/container",
|
Target: "/home/container",
|
||||||
Source: s.Filesystem.Path(),
|
Source: s.Filesystem().Path(),
|
||||||
ReadOnly: false,
|
ReadOnly: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"github.com/pterodactyl/wings/config"
|
"github.com/pterodactyl/wings/config"
|
||||||
"github.com/pterodactyl/wings/environment"
|
"github.com/pterodactyl/wings/environment"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"golang.org/x/sync/semaphore"
|
"golang.org/x/sync/semaphore"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
@ -164,11 +165,11 @@ func (s *Server) onBeforeStart() error {
|
||||||
// If a server has unlimited disk space, we don't care enough to block the startup to check remaining.
|
// If a server has unlimited disk space, we don't care enough to block the startup to check remaining.
|
||||||
// However, we should trigger a size anyway, as it'd be good to kick it off for other processes.
|
// However, we should trigger a size anyway, as it'd be good to kick it off for other processes.
|
||||||
if s.DiskSpace() <= 0 {
|
if s.DiskSpace() <= 0 {
|
||||||
s.Filesystem.HasSpaceAvailable(true)
|
s.Filesystem().HasSpaceAvailable(true)
|
||||||
} else {
|
} else {
|
||||||
s.PublishConsoleOutputFromDaemon("Checking server disk space usage, this could take a few seconds...")
|
s.PublishConsoleOutputFromDaemon("Checking server disk space usage, this could take a few seconds...")
|
||||||
if !s.Filesystem.HasSpaceAvailable(false) {
|
if !s.Filesystem().HasSpaceAvailable(false) {
|
||||||
return ErrNotEnoughDiskSpace
|
return filesystem.ErrNotEnoughDiskSpace
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -183,7 +184,7 @@ func (s *Server) onBeforeStart() error {
|
||||||
if config.Get().System.CheckPermissionsOnBoot {
|
if config.Get().System.CheckPermissionsOnBoot {
|
||||||
s.PublishConsoleOutputFromDaemon("Ensuring file permissions are set correctly, this could take a few seconds...")
|
s.PublishConsoleOutputFromDaemon("Ensuring file permissions are set correctly, this could take a few seconds...")
|
||||||
// Ensure all of the server file permissions are set correctly before booting the process.
|
// Ensure all of the server file permissions are set correctly before booting the process.
|
||||||
if err := s.Filesystem.Chown("/"); err != nil {
|
if err := s.Filesystem().Chown("/"); err != nil {
|
||||||
return errors.Wrap(err, "failed to chown root server directory during pre-boot process")
|
return errors.Wrap(err, "failed to chown root server directory during pre-boot process")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"github.com/pterodactyl/wings/environment"
|
"github.com/pterodactyl/wings/environment"
|
||||||
"github.com/pterodactyl/wings/environment/docker"
|
"github.com/pterodactyl/wings/environment/docker"
|
||||||
"github.com/pterodactyl/wings/events"
|
"github.com/pterodactyl/wings/events"
|
||||||
|
"github.com/pterodactyl/wings/server/filesystem"
|
||||||
"golang.org/x/sync/semaphore"
|
"golang.org/x/sync/semaphore"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
@ -34,7 +35,8 @@ type Server struct {
|
||||||
resources ResourceUsage
|
resources ResourceUsage
|
||||||
Archiver Archiver `json:"-"`
|
Archiver Archiver `json:"-"`
|
||||||
Environment environment.ProcessEnvironment `json:"-"`
|
Environment environment.ProcessEnvironment `json:"-"`
|
||||||
Filesystem Filesystem `json:"-"`
|
|
||||||
|
fs *filesystem.Filesystem
|
||||||
|
|
||||||
// Events emitted by the server instance.
|
// Events emitted by the server instance.
|
||||||
emitter *events.EventBus
|
emitter *events.EventBus
|
||||||
|
@ -133,6 +135,10 @@ func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) err
|
||||||
s.procConfig = cfg.ProcessConfiguration
|
s.procConfig = cfg.ProcessConfiguration
|
||||||
s.Unlock()
|
s.Unlock()
|
||||||
|
|
||||||
|
// Update the disk space limits for the server whenever the configuration
|
||||||
|
// for it changes.
|
||||||
|
s.fs.SetDiskLimit(s.DiskSpace())
|
||||||
|
|
||||||
// If this is a Docker environment we need to sync the stop configuration with it so that
|
// If this is a Docker environment we need to sync the stop configuration with it so that
|
||||||
// the process isn't just terminated when a user requests it be stopped.
|
// the process isn't just terminated when a user requests it be stopped.
|
||||||
if e, ok := s.Environment.(*docker.Environment); ok {
|
if e, ok := s.Environment.(*docker.Environment); ok {
|
||||||
|
@ -161,7 +167,7 @@ func (s *Server) IsBootable() bool {
|
||||||
// for the server is setup, and that all of the necessary files are created.
|
// for the server is setup, and that all of the necessary files are created.
|
||||||
func (s *Server) CreateEnvironment() error {
|
func (s *Server) CreateEnvironment() error {
|
||||||
// Ensure the data directory exists before getting too far through this process.
|
// Ensure the data directory exists before getting too far through this process.
|
||||||
if err := s.Filesystem.EnsureDataDirectory(); err != nil {
|
if err := s.EnsureDataDirectoryExists(); err != nil {
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,7 +51,7 @@ func validatePath(fs FileSystem, p string) (string, error) {
|
||||||
return "", noMatchingServerError
|
return "", noMatchingServerError
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.Filesystem.SafePath(p)
|
return s.Filesystem().SafePath(p)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateDiskSpace(fs FileSystem) bool {
|
func validateDiskSpace(fs FileSystem) bool {
|
||||||
|
@ -63,7 +63,7 @@ func validateDiskSpace(fs FileSystem) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.Filesystem.HasSpaceAvailable(true)
|
return s.Filesystem().HasSpaceAvailable(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validates a set of credentials for a SFTP login against Pterodactyl Panel and returns
|
// Validates a set of credentials for a SFTP login against Pterodactyl Panel and returns
|
||||||
|
|
Loading…
Reference in New Issue
Block a user