From 6fe2468a5a109b59fcae332c95d159d9b68630a8 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Fri, 8 Jan 2021 22:49:19 -0800 Subject: [PATCH 01/30] foundation for self-upgrade logic --- cmd/root.go | 1 + cmd/upgrade.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 cmd/upgrade.go diff --git a/cmd/root.go b/cmd/root.go index d1e26a0..65259f8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -75,6 +75,7 @@ func init() { rootCommand.AddCommand(versionCommand) rootCommand.AddCommand(configureCmd) rootCommand.AddCommand(diagnosticsCmd) + rootCommand.AddCommand(newUpgradeCommand()) } // Get the configuration path based on the arguments provided. diff --git a/cmd/upgrade.go b/cmd/upgrade.go new file mode 100644 index 0000000..5935bb1 --- /dev/null +++ b/cmd/upgrade.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "errors" + "fmt" + "github.com/apex/log" + "github.com/pterodactyl/wings/loggers/cli" + "github.com/spf13/cobra" + "runtime" +) + +type upgrader struct{} + +func newUpgradeCommand() *cobra.Command { + u := upgrader{} + command := &cobra.Command{ + Use: "upgrade", + Short: "Performs a self-upgrade for Wings.", + Long: `Queries GitHub to find the latest Wings release and then downloads it, replacing +the existing system binary. This will use checksums and GPG signatures present on +the uploaded assets to validate that they have been released by the Pterodactyl team. + +Once downloaded the Wings systemd process will be restarted if it is present on the +system, therefore this command MUST be executed as a root user. + +This command can only be executed on ARM64/AMD64 Linux systems. All other systems will +report an error when executing this command. +`, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + log.SetLevel(log.InfoLevel) + if debug { + log.SetLevel(log.DebugLevel) + } + log.SetHandler(cli.Default) + }, + PreRunE: func(cmd *cobra.Command, args []string) error { + if runtime.GOOS != "linux" { + return errors.New(fmt.Sprintf("upgrade: os not supported: %s", runtime.GOOS)) + } + if runtime.GOARCH != "arm64" && runtime.GOARCH != "amd64" { + return errors.New(fmt.Sprintf("upgrade: unexpected architecture: %s", runtime.GOARCH)) + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return u.execute() + }, + } + + command.PersistentFlags().String("version", "latest", "download a specific version of Wings") + command.PersistentFlags().String("repository", "pterodactyl/wings", "the repository to use when looking for updates -- if set, GPG verification is skipped") + command.PersistentFlags().String("auth-token", "", "a GitHub authentication token to use for private repositories") + command.PersistentFlags().Bool("download-only", false, "if set, do not restart wings after downloading") + + return command +} + +// Executes a self-upgrade of Wings by pulling down the latest version from GitHub +// (or the given flag version) and then restarting the Wings process. +func (u *upgrader) execute() error { + return nil +} From ff8926bba8b7dac8de9e458ed9c07298882a50e7 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 9 Jan 2021 17:37:58 -0800 Subject: [PATCH 02/30] bye bye command --- cmd/root.go | 1 - cmd/upgrade.go | 62 -------------------------------------------------- go.mod | 1 + 3 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 cmd/upgrade.go diff --git a/cmd/root.go b/cmd/root.go index 65259f8..d1e26a0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -75,7 +75,6 @@ func init() { rootCommand.AddCommand(versionCommand) rootCommand.AddCommand(configureCmd) rootCommand.AddCommand(diagnosticsCmd) - rootCommand.AddCommand(newUpgradeCommand()) } // Get the configuration path based on the arguments provided. diff --git a/cmd/upgrade.go b/cmd/upgrade.go deleted file mode 100644 index 5935bb1..0000000 --- a/cmd/upgrade.go +++ /dev/null @@ -1,62 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - "github.com/apex/log" - "github.com/pterodactyl/wings/loggers/cli" - "github.com/spf13/cobra" - "runtime" -) - -type upgrader struct{} - -func newUpgradeCommand() *cobra.Command { - u := upgrader{} - command := &cobra.Command{ - Use: "upgrade", - Short: "Performs a self-upgrade for Wings.", - Long: `Queries GitHub to find the latest Wings release and then downloads it, replacing -the existing system binary. This will use checksums and GPG signatures present on -the uploaded assets to validate that they have been released by the Pterodactyl team. - -Once downloaded the Wings systemd process will be restarted if it is present on the -system, therefore this command MUST be executed as a root user. - -This command can only be executed on ARM64/AMD64 Linux systems. All other systems will -report an error when executing this command. -`, - PersistentPreRun: func(cmd *cobra.Command, args []string) { - log.SetLevel(log.InfoLevel) - if debug { - log.SetLevel(log.DebugLevel) - } - log.SetHandler(cli.Default) - }, - PreRunE: func(cmd *cobra.Command, args []string) error { - if runtime.GOOS != "linux" { - return errors.New(fmt.Sprintf("upgrade: os not supported: %s", runtime.GOOS)) - } - if runtime.GOARCH != "arm64" && runtime.GOARCH != "amd64" { - return errors.New(fmt.Sprintf("upgrade: unexpected architecture: %s", runtime.GOARCH)) - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - return u.execute() - }, - } - - command.PersistentFlags().String("version", "latest", "download a specific version of Wings") - command.PersistentFlags().String("repository", "pterodactyl/wings", "the repository to use when looking for updates -- if set, GPG verification is skipped") - command.PersistentFlags().String("auth-token", "", "a GitHub authentication token to use for private repositories") - command.PersistentFlags().Bool("download-only", false, "if set, do not restart wings after downloading") - - return command -} - -// Executes a self-upgrade of Wings by pulling down the latest version from GitHub -// (or the given flag version) and then restarting the Wings process. -func (u *upgrader) execute() error { - return nil -} diff --git a/go.mod b/go.mod index 4659f61..079a014 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/cobra v1.1.1 + github.com/spf13/pflag v1.0.5 github.com/ugorji/go v1.2.2 // indirect github.com/ulikunitz/xz v0.5.9 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad From 96256ac63e4e5659f8bf4b5921f75bb2455ad33f Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 9 Jan 2021 17:52:27 -0800 Subject: [PATCH 03/30] [security] fix vulnerability when handling remote file redirects Also adds the ability for an admin to just completely disable this service if it is not needed on the node. --- CHANGELOG.md | 7 +++++++ config/config.go | 7 ++++++- router/downloader/downloader.go | 17 ++++++++++++++++- router/middleware.go | 15 +++++++++++++++ router/router.go | 6 +++--- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c4b4fc..a74ef1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## v1.2.3 +### Fixed +* **[Security]** Fixes a remaining security vulnerability in the code handling remote file downloads for servers relating to redirect validation. + +### Added +* Adds a configuration key at `api.disable_remote_download` that can be set to `true` to completely download the remote download system. + ## v1.2.2 ### Fixed * Reverts changes to logic handling blocking until a server process is done running when polling stats. This change exposed a bug in the underlying Docker system causing servers to enter a state in which Wings was unable to terminate the process and Docker commands would hang if executed against the container. diff --git a/config/config.go b/config/config.go index 06c1157..0af323f 100644 --- a/config/config.go +++ b/config/config.go @@ -88,11 +88,16 @@ type ApiConfiguration struct { // SSL configuration for the daemon. Ssl struct { - Enabled bool `default:"false"` + Enabled bool `json:"enabled" yaml:"enabled"` CertificateFile string `json:"cert" yaml:"cert"` KeyFile string `json:"key" yaml:"key"` } + // Determines if functionality for allowing remote download of files into server directories + // is enabled on this instance. If set to "true" remote downloads will not be possible for + // servers. + DisableRemoteDownload bool `json:"disable_remote_download" yaml:"disable_remote_download"` + // The maximum size for files uploaded through the Panel in bytes. UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"` } diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go index a89cca1..20dcba7 100644 --- a/router/downloader/downloader.go +++ b/router/downloader/downloader.go @@ -18,7 +18,22 @@ import ( "time" ) -var client = &http.Client{Timeout: time.Hour * 12} +var client = &http.Client{ + Timeout: time.Hour * 12, + // Disallow any redirect on a HTTP call. This is a security requirement: do not modify + // this logic without first ensuring that the new target location IS NOT within the current + // instance's local network. + // + // This specific error response just causes the client to not follow the redirect and + // returns the actual redirect response to the caller. Not perfect, but simple and most + // people won't be using URLs that redirect anyways hopefully? + // + // We'll re-evaluate this down the road if needed. + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, +} + var instance = &Downloader{ // Tracks all of the active downloads. downloadCache: make(map[string]*Download), diff --git a/router/middleware.go b/router/middleware.go index c3f32fd..7fd52bf 100644 --- a/router/middleware.go +++ b/router/middleware.go @@ -120,6 +120,21 @@ func (m *Middleware) ServerExists() gin.HandlerFunc { } } +// Checks if remote file downloading is enabled on this instance before allowing access +// to the given endpoint. +func (m *Middleware) CheckRemoteDownloadEnabled() gin.HandlerFunc { + disabled := config.Get().Api.DisableRemoteDownload + return func(c *gin.Context) { + if disabled { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ + "error": "This functionality is not currently enabled on this instance.", + }) + return + } + c.Next() + } +} + // Returns the server instance from the gin context. If there is no server set in the // context (e.g. calling from a controller not protected by ServerExists) this function // will panic. diff --git a/router/router.go b/router/router.go index 85a97a7..0eb4075 100644 --- a/router/router.go +++ b/router/router.go @@ -88,9 +88,9 @@ func Configure() *gin.Engine { files.POST("/decompress", postServerDecompressFiles) files.POST("/chmod", postServerChmodFile) - files.GET("/pull", getServerPullingFiles) - files.POST("/pull", postServerPullRemoteFile) - files.DELETE("/pull/:download", deleteServerPullRemoteFile) + files.GET("/pull", m.CheckRemoteDownloadEnabled(), getServerPullingFiles) + files.POST("/pull", m.CheckRemoteDownloadEnabled(), postServerPullRemoteFile) + files.DELETE("/pull/:download", m.CheckRemoteDownloadEnabled(), deleteServerPullRemoteFile) } backup := server.Group("/backup") From c228acaafc474a9b53a66de3ee5907b851dcd243 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 14:25:39 -0800 Subject: [PATCH 04/30] Massive refactor of SFTP system now that it is deeply integrated with Wings --- server/filesystem/errors.go | 6 +- server/filesystem/filesystem.go | 66 ++++-- sftp/errors.go | 19 -- sftp/handler.go | 383 +++++++++++--------------------- sftp/server.go | 66 +++--- sftp/sftp.go | 36 +-- sftp/{lister.go => utils.go} | 17 ++ 7 files changed, 226 insertions(+), 367 deletions(-) delete mode 100644 sftp/errors.go rename sftp/{lister.go => utils.go} (57%) diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index 52a2c20..a5890bd 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -49,10 +49,8 @@ func (e *Error) Code() ErrorCode { // Checks if the given error is one of the Filesystem errors. func IsFilesystemError(err error) (*Error, bool) { - if e := errors.Unwrap(err); e != nil { - err = e - } - if fserr, ok := err.(*Error); ok { + var fserr *Error + if errors.As(err, &fserr) { return fserr, true } return nil, false diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 2e0bd88..a4a49de 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -2,11 +2,6 @@ package filesystem import ( "bufio" - "emperror.dev/errors" - "github.com/gabriel-vasile/mimetype" - "github.com/karrick/godirwalk" - "github.com/pterodactyl/wings/config" - "github.com/pterodactyl/wings/system" "io" "io/ioutil" "os" @@ -17,6 +12,12 @@ import ( "strings" "sync" "time" + + "emperror.dev/errors" + "github.com/gabriel-vasile/mimetype" + "github.com/karrick/godirwalk" + "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/system" ) type Filesystem struct { @@ -71,6 +72,41 @@ func (fs *Filesystem) File(p string) (*os.File, os.FileInfo, error) { return f, st, nil } +// Acts by creating the given file and path on the disk if it is not present already. If +// it is present, the file is opened using the defaults which will truncate the contents. +// The opened file is then returned to the caller. +func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + f, err := os.OpenFile(cleaned, flag, 0644) + if err == nil { + return f, nil + } + // If the error is not because it doesn't exist then we just need to bail at this point. + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + // Create the path leading up to the file we're trying to create, setting the final perms + // on it as we go. + if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil { + return nil, err + } + if err := fs.Chown(filepath.Dir(cleaned)); err != nil { + return nil, err + } + o := &fileOpener{} + // Try to open the file now that we have created the pathing necessary for it, and then + // Chown that file so that the permissions don't mess with things. + f, err = o.open(cleaned, flag, 0644) + if err != nil { + return nil, err + } + _ = fs.Chown(cleaned) + return f, nil +} + // Reads a file on the system and returns it as a byte representation in a file // reader. This is not the most memory efficient usage since it will be reading the // entirety of the file into memory. @@ -112,22 +148,9 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { return err } - // If we were unable to stat the location because it did not exist, go ahead and create - // it now. We do this after checking the disk space so that we do not just create empty - // directories at random. - if err != nil { - if err := os.MkdirAll(filepath.Dir(cleaned), 0755); err != nil { - return err - } - if err := fs.Chown(filepath.Dir(cleaned)); err != nil { - return err - } - } - - o := &fileOpener{} - // This will either create the file if it does not already exist, or open and - // truncate the existing file. - file, err := o.open(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + // Touch the file and return the handle to it at this point. This will create the file + // and any necessary directories as needed. + file, err := fs.Touch(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { return err } @@ -150,7 +173,6 @@ func (fs *Filesystem) CreateDirectory(name string, p string) error { if err != nil { return err } - return os.MkdirAll(cleaned, 0755) } diff --git a/sftp/errors.go b/sftp/errors.go deleted file mode 100644 index 122aa15..0000000 --- a/sftp/errors.go +++ /dev/null @@ -1,19 +0,0 @@ -package sftp - -type fxerr uint32 - -const ( - // Extends the default SFTP server to return a quota exceeded error to the client. - // - // @see https://tools.ietf.org/id/draft-ietf-secsh-filexfer-13.txt - ErrSshQuotaExceeded = fxerr(15) -) - -func (e fxerr) Error() string { - switch e { - case ErrSshQuotaExceeded: - return "Quota Exceeded" - default: - return "Failure" - } -} diff --git a/sftp/handler.go b/sftp/handler.go index 68eed8c..8957583 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -5,31 +5,15 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "sync" + "emperror.dev/errors" "github.com/apex/log" - "github.com/patrickmn/go-cache" "github.com/pkg/sftp" + "github.com/pterodactyl/wings/server/filesystem" ) -type FileSystem struct { - UUID string - Permissions []string - ReadOnly bool - User User - Cache *cache.Cache - - PathValidator func(fs FileSystem, p string) (string, error) - HasDiskSpace func(fs FileSystem) bool - - logger *log.Entry - lock sync.Mutex -} - -func (fs FileSystem) buildPath(p string) (string, error) { - return fs.PathValidator(fs, p) -} - const ( PermissionFileRead = "file.read" PermissionFileReadContent = "file.read-content" @@ -38,343 +22,244 @@ const ( PermissionFileDelete = "file.delete" ) +type Handler struct { + permissions []string + mu sync.Mutex + fs *filesystem.Filesystem + logger *log.Entry + ro bool +} + // Fileread creates a reader for a file on the system and returns the reader back. -func (fs FileSystem) Fileread(request *sftp.Request) (io.ReaderAt, error) { +func (h *Handler) Fileread(request *sftp.Request) (io.ReaderAt, error) { // Check first if the user can actually open and view a file. This permission is named // really poorly, but it is checking if they can read. There is an addition permission, // "save-files" which determines if they can write that file. - if !fs.can(PermissionFileReadContent) { - return nil, sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileReadContent) { + return nil, sftp.ErrSSHFxPermissionDenied } - - p, err := fs.buildPath(request.Filepath) + h.mu.Lock() + defer h.mu.Unlock() + f, _, err := h.fs.File(request.Filepath) if err != nil { - return nil, sftp.ErrSshFxNoSuchFile + if !errors.Is(err, os.ErrNotExist) { + h.logger.WithField("error", err).Error("error processing readfile request") + return nil, sftp.ErrSSHFxFailure + } + return nil, sftp.ErrSSHFxNoSuchFile } - - fs.lock.Lock() - defer fs.lock.Unlock() - - if _, err := os.Stat(p); os.IsNotExist(err) { - return nil, sftp.ErrSshFxNoSuchFile - } else if err != nil { - fs.logger.WithField("error", err).Error("error while processing file stat") - - return nil, sftp.ErrSshFxFailure - } - - file, err := os.Open(p) - if err != nil { - fs.logger.WithField("source", p).WithField("error", err).Error("could not open file for reading") - return nil, sftp.ErrSshFxFailure - } - - return file, nil + return f, nil } // Filewrite handles the write actions for a file on the system. -func (fs FileSystem) Filewrite(request *sftp.Request) (io.WriterAt, error) { - if fs.ReadOnly { - return nil, sftp.ErrSshFxOpUnsupported +func (h *Handler) Filewrite(request *sftp.Request) (io.WriterAt, error) { + if h.ro { + return nil, sftp.ErrSSHFxOpUnsupported } - - p, err := fs.buildPath(request.Filepath) - if err != nil { - return nil, sftp.ErrSshFxNoSuchFile - } - - l := fs.logger.WithField("source", p) - + l := h.logger.WithField("source", request.Filepath) // If the user doesn't have enough space left on the server it should respond with an // error since we won't be letting them write this file to the disk. - if !fs.HasDiskSpace(fs) { - return nil, ErrSshQuotaExceeded + if !h.fs.HasSpaceAvailable(true) { + return nil, ErrSSHQuotaExceeded } - fs.lock.Lock() - defer fs.lock.Unlock() - - stat, statErr := os.Stat(p) - // If the file doesn't exist we need to create it, as well as the directory pathway - // leading up to where that file will be created. - if os.IsNotExist(statErr) { - // This is a different pathway than just editing an existing file. If it doesn't exist already - // we need to determine if this user has permission to create files. - if !fs.can(PermissionFileCreate) { - return nil, sftp.ErrSshFxPermissionDenied + h.mu.Lock() + defer h.mu.Unlock() + // The specific permission required to perform this action. If the file exists on the + // system already it only needs to be an update, otherwise we'll check for a create. + permission := PermissionFileUpdate + _, sterr := h.fs.Stat(request.Filepath) + if sterr != nil { + if !errors.Is(sterr, os.ErrNotExist) { + l.WithField("error", sterr).Error("error while getting file reader") + return nil, sftp.ErrSSHFxFailure } - - // Create all of the directories leading up to the location where this file is being created. - if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { - l.WithFields(log.Fields{ - "path": filepath.Dir(p), - "error": err, - }).Error("error making path for file") - - return nil, sftp.ErrSshFxFailure - } - - file, err := os.Create(p) - if err != nil { - l.WithField("error", err).Error("failed to create file") - - return nil, sftp.ErrSshFxFailure - } - - // Not failing here is intentional. We still made the file, it is just owned incorrectly - // and will likely cause some issues. - if err := os.Chown(p, fs.User.Uid, fs.User.Gid); err != nil { - l.WithField("error", err).Warn("failed to set permissions on file") - } - - return file, nil + permission = PermissionFileCreate } - - // If the stat error isn't about the file not existing, there is some other issue - // at play and we need to go ahead and bail out of the process. - if statErr != nil { - l.WithField("error", statErr).Error("encountered error performing file stat") - - return nil, sftp.ErrSshFxFailure + // Confirm the user has permission to perform this action BEFORE calling Touch, otherwise + // you'll potentially create a file on the system and then fail out because of user + // permission checking after the fact. + if !h.can(permission) { + return nil, sftp.ErrSSHFxPermissionDenied } - - // If we've made it here it means the file already exists and we don't need to do anything - // fancy to handle it. Just pass over the request flags so the system knows what the end - // goal with the file is going to be. - // - // But first, check that the user has permission to save modified files. - if !fs.can(PermissionFileUpdate) { - return nil, sftp.ErrSshFxPermissionDenied - } - - // Not sure this would ever happen, but lets not find out. - if stat.IsDir() { - return nil, sftp.ErrSshFxOpUnsupported - } - - file, err := os.Create(p) + f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { - // Prevent errors if the file is deleted between the stat and this call. - if os.IsNotExist(err) { - return nil, sftp.ErrSSHFxNoSuchFile - } - l.WithField("flags", request.Flags).WithField("error", err).Error("failed to open existing file on system") - return nil, sftp.ErrSshFxFailure + return nil, sftp.ErrSSHFxFailure } - - // Not failing here is intentional. We still made the file, it is just owned incorrectly - // and will likely cause some issues. - if err := os.Chown(p, fs.User.Uid, fs.User.Gid); err != nil { - l.WithField("error", err).Warn("error chowning file") - } - - return file, nil + return f, nil } // Filecmd hander for basic SFTP system calls related to files, but not anything to do with reading // or writing to those files. -func (fs FileSystem) Filecmd(request *sftp.Request) error { - if fs.ReadOnly { - return sftp.ErrSshFxOpUnsupported +func (h *Handler) Filecmd(request *sftp.Request) error { + if h.ro { + return sftp.ErrSSHFxOpUnsupported } - - p, err := fs.buildPath(request.Filepath) - if err != nil { - return sftp.ErrSshFxNoSuchFile - } - - l := fs.logger.WithField("source", p) - - var target string - // If a target is provided in this request validate that it is going to the correct - // location for the server. If it is not, return an operation unsupported error. This - // is maybe not the best error response, but its not wrong either. + l := h.logger.WithField("source", request.Filepath) if request.Target != "" { - target, err = fs.buildPath(request.Target) - if err != nil { - return sftp.ErrSshFxOpUnsupported - } + l = l.WithField("target", request.Target) } switch request.Method { + // Allows a user to make changes to the permissions of a given file or directory + // on their server using their SFTP client. case "Setstat": - if !fs.can(PermissionFileUpdate) { - return sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileUpdate) { + return sftp.ErrSSHFxPermissionDenied } - - mode := os.FileMode(0644) - // If the client passed a valid file permission use that, otherwise use the - // default of 0644 set above. - if request.Attributes().FileMode().Perm() != 0000 { - mode = request.Attributes().FileMode().Perm() + mode := request.Attributes().FileMode().Perm() + // If the client passes an invalid FileMode just use the default 0644. + if mode == 0000 { + mode = os.FileMode(0644) } - - // Force directories to be 0755 + // Force directories to be 0755. if request.Attributes().FileMode().IsDir() { mode = 0755 } - - if err := os.Chmod(p, mode); err != nil { - if os.IsNotExist(err) { + if err := h.fs.Chmod(request.Filepath, mode); err != nil { + if errors.Is(err, os.ErrNotExist) { return sftp.ErrSSHFxNoSuchFile } - l.WithField("error", err).Error("failed to perform setstat on item") return sftp.ErrSSHFxFailure } - return nil + break + // Support renaming a file (aka Move). case "Rename": - if !fs.can(PermissionFileUpdate) { + if !h.can(PermissionFileUpdate) { return sftp.ErrSSHFxPermissionDenied } - - if err := os.Rename(p, target); err != nil { - if os.IsNotExist(err) { + if err := h.fs.Rename(request.Filepath, request.Target); err != nil { + if errors.Is(err, os.ErrNotExist) { return sftp.ErrSSHFxNoSuchFile } - - l.WithField("target", target).WithField("error", err).Error("failed to rename file") - - return sftp.ErrSshFxFailure + l.WithField("error", err).Error("failed to rename file") + return sftp.ErrSSHFxFailure } - break + // Handle deletion of a directory. This will properly delete all of the files and + // folders within that directory if it is not already empty (unlike a lot of SFTP + // clients that must delete each file individually). case "Rmdir": - if !fs.can(PermissionFileDelete) { - return sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileDelete) { + return sftp.ErrSSHFxPermissionDenied } - - if err := os.RemoveAll(p); err != nil { + if err := h.fs.Delete(request.Filepath); err != nil { l.WithField("error", err).Error("failed to remove directory") - - return sftp.ErrSshFxFailure + return sftp.ErrSSHFxFailure } - - return sftp.ErrSshFxOk + return sftp.ErrSSHFxOk + // Handle requests to create a new Directory. case "Mkdir": - if !fs.can(PermissionFileCreate) { - return sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileCreate) { + return sftp.ErrSSHFxPermissionDenied } - - if err := os.MkdirAll(p, 0755); err != nil { + name := strings.Split(filepath.Clean(request.Filepath), "/") + err := h.fs.CreateDirectory(name[len(name)-1], strings.Join(name[0:len(name)-1], "/")) + if err != nil { l.WithField("error", err).Error("failed to create directory") - - return sftp.ErrSshFxFailure + return sftp.ErrSSHFxFailure } - break + // Support creating symlinks between files. The source and target must resolve within + // the server home directory. case "Symlink": - if !fs.can(PermissionFileCreate) { - return sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileCreate) { + return sftp.ErrSSHFxPermissionDenied } - - if err := os.Symlink(p, target); err != nil { + source, err := h.fs.SafePath(request.Filepath) + if err != nil { + return sftp.ErrSSHFxNoSuchFile + } + target, err := h.fs.SafePath(request.Target) + if err != nil { + return sftp.ErrSSHFxNoSuchFile + } + if err := os.Symlink(source, target); err != nil { l.WithField("target", target).WithField("error", err).Error("failed to create symlink") - - return sftp.ErrSshFxFailure + return sftp.ErrSSHFxFailure } - break + // Called when deleting a file. case "Remove": - if !fs.can(PermissionFileDelete) { - return sftp.ErrSshFxPermissionDenied + if !h.can(PermissionFileDelete) { + return sftp.ErrSSHFxPermissionDenied } - - if err := os.Remove(p); err != nil { - if os.IsNotExist(err) { + if err := h.fs.Delete(request.Filepath); err != nil { + if errors.Is(err, os.ErrNotExist) { return sftp.ErrSSHFxNoSuchFile } - l.WithField("error", err).Error("failed to remove a file") - - return sftp.ErrSshFxFailure + return sftp.ErrSSHFxFailure } - - return sftp.ErrSshFxOk + return sftp.ErrSSHFxOk default: - return sftp.ErrSshFxOpUnsupported + return sftp.ErrSSHFxOpUnsupported } - var fileLocation = p - if target != "" { - fileLocation = target + target := request.Filepath + if request.Target != "" { + target = request.Target } - // Not failing here is intentional. We still made the file, it is just owned incorrectly // and will likely cause some issues. There is no logical check for if the file was removed // because both of those cases (Rmdir, Remove) have an explicit return rather than break. - if err := os.Chown(fileLocation, fs.User.Uid, fs.User.Gid); err != nil { + if err := h.fs.Chown(target); err != nil { l.WithField("error", err).Warn("error chowning file") } - return sftp.ErrSshFxOk + return sftp.ErrSSHFxOk } // Filelist is the handler for SFTP filesystem list calls. This will handle calls to list the contents of // a directory as well as perform file/folder stat calls. -func (fs FileSystem) Filelist(request *sftp.Request) (sftp.ListerAt, error) { - p, err := fs.buildPath(request.Filepath) - if err != nil { - return nil, sftp.ErrSshFxNoSuchFile +func (h *Handler) Filelist(request *sftp.Request) (sftp.ListerAt, error) { + if !h.can(PermissionFileRead) { + return nil, sftp.ErrSSHFxPermissionDenied } switch request.Method { case "List": - if !fs.can(PermissionFileRead) { - return nil, sftp.ErrSshFxPermissionDenied + p, err := h.fs.SafePath(request.Filepath) + if err != nil { + return nil, sftp.ErrSSHFxNoSuchFile } - files, err := ioutil.ReadDir(p) if err != nil { - fs.logger.WithField("error", err).Error("error while listing directory") + h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error while listing directory") - return nil, sftp.ErrSshFxFailure + return nil, sftp.ErrSSHFxFailure } - return ListerAt(files), nil case "Stat": - if !fs.can(PermissionFileRead) { - return nil, sftp.ErrSshFxPermissionDenied + st, err := h.fs.Stat(request.Filepath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, sftp.ErrSSHFxNoSuchFile + } + h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error performing stat on file") + return nil, sftp.ErrSSHFxFailure } - - s, err := os.Stat(p) - if os.IsNotExist(err) { - return nil, sftp.ErrSshFxNoSuchFile - } else if err != nil { - fs.logger.WithField("source", p).WithField("error", err).Error("error performing stat on file") - - return nil, sftp.ErrSshFxFailure - } - - return ListerAt([]os.FileInfo{s}), nil + return ListerAt([]os.FileInfo{st.Info}), nil default: - // Before adding readlink support we need to evaluate any potential security risks - // as a result of navigating around to a location that is outside the home directory - // for the logged in user. I don't foresee it being much of a problem, but I do want to - // check it out before slapping some code here. Until then, we'll just return an - // unsupported response code. - return nil, sftp.ErrSshFxOpUnsupported + return nil, sftp.ErrSSHFxOpUnsupported } } // Determines if a user has permission to perform a specific action on the SFTP server. These // permissions are defined and returned by the Panel API. -func (fs FileSystem) can(permission string) bool { - // Server owners and super admins have their permissions returned as '[*]' via the Panel +func (h *Handler) can(permission string) bool { + // SFTPServer owners and super admins have their permissions returned as '[*]' via the Panel // API, so for the sake of speed do an initial check for that before iterating over the // entire array of permissions. - if len(fs.Permissions) == 1 && fs.Permissions[0] == "*" { + if len(h.permissions) == 1 && h.permissions[0] == "*" { return true } - - // Not the owner or an admin, loop over the permissions that were returned to determine - // if they have the passed permission. - for _, p := range fs.Permissions { + for _, p := range h.permissions { if p == permission { return true } } - return false } diff --git a/sftp/server.go b/sftp/server.go index a0ae24b..294122a 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -12,12 +12,12 @@ import ( "os" "path" "strings" - "time" "github.com/apex/log" - "github.com/patrickmn/go-cache" "github.com/pkg/sftp" "github.com/pterodactyl/wings/api" + "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/server" "golang.org/x/crypto/ssh" ) @@ -33,35 +33,23 @@ type User struct { Gid int } -type Server struct { - cache *cache.Cache - +//goland:noinspection GoNameStartsWithPackageName +type SFTPServer struct { Settings Settings User User - - PathValidator func(fs FileSystem, p string) (string, error) - DiskSpaceValidator func(fs FileSystem) bool - // Validator function that is called when a user connects to the server. This should // check against whatever system is desired to confirm if the given username and password // combination is valid. If so, should return an authentication response. - CredentialValidator func(r api.SftpAuthRequest) (*api.SftpAuthResponse, error) -} - -// Create a new server configuration instance. -func New(c *Server) error { - c.cache = cache.New(5*time.Minute, 10*time.Minute) - - return nil + credentialValidator func(r api.SftpAuthRequest) (*api.SftpAuthResponse, error) } // Initialize the SFTP server and add a persistent listener to handle inbound SFTP connections. -func (c *Server) Initialize() error { +func (c *SFTPServer) Initialize() error { serverConfig := &ssh.ServerConfig{ NoClientAuth: false, MaxAuthTries: 6, PasswordCallback: func(conn ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { - resp, err := c.CredentialValidator(api.SftpAuthRequest{ + resp, err := c.credentialValidator(api.SftpAuthRequest{ User: conn.User(), Pass: string(pass), IP: conn.RemoteAddr().String(), @@ -123,7 +111,7 @@ func (c *Server) Initialize() error { // Handles an inbound connection to the instance and determines if we should serve the request // or not. -func (c Server) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) { +func (c SFTPServer) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) { defer conn.Close() // Before beginning a handshake must be performed on the incoming net.Conn @@ -165,19 +153,17 @@ func (c Server) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) } }(requests) - // Configure the user's home folder for the rest of the request cycle. if sconn.Permissions.Extensions["uuid"] == "" { continue } // Create a new handler for the currently logged in user's server. - fs := c.createHandler(sconn) + fs := c.newHandler(sconn) // Create the server instance for the channel using the filesystem we created above. - server := sftp.NewRequestServer(channel, fs) - - if err := server.Serve(); err == io.EOF { - server.Close() + handler := sftp.NewRequestServer(channel, fs) + if err := handler.Serve(); err == io.EOF { + handler.Close() } } } @@ -185,15 +171,15 @@ func (c Server) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) // Creates a new SFTP handler for a given server. The directory argument should // be the base directory for a server. All actions done on the server will be // relative to that directory, and the user will not be able to escape out of it. -func (c Server) createHandler(sc *ssh.ServerConn) sftp.Handlers { - p := FileSystem{ - UUID: sc.Permissions.Extensions["uuid"], - Permissions: strings.Split(sc.Permissions.Extensions["permissions"], ","), - ReadOnly: c.Settings.ReadOnly, - Cache: c.cache, - User: c.User, - HasDiskSpace: c.DiskSpaceValidator, - PathValidator: c.PathValidator, +func (c SFTPServer) newHandler(sc *ssh.ServerConn) sftp.Handlers { + s := server.GetServers().Find(func(s *server.Server) bool { + return s.Id() == sc.Permissions.Extensions["uuid"] + }) + + p := Handler{ + fs: s.Filesystem(), + permissions: strings.Split(sc.Permissions.Extensions["permissions"], ","), + ro: config.Get().System.Sftp.ReadOnly, logger: log.WithFields(log.Fields{ "subsystem": "sftp", "username": sc.User(), @@ -202,15 +188,15 @@ func (c Server) createHandler(sc *ssh.ServerConn) sftp.Handlers { } return sftp.Handlers{ - FileGet: p, - FilePut: p, - FileCmd: p, - FileList: p, + FileGet: &p, + FilePut: &p, + FileCmd: &p, + FileList: &p, } } // Generates a private key that will be used by the SFTP server. -func (c Server) generatePrivateKey() error { +func (c SFTPServer) generatePrivateKey() error { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return err diff --git a/sftp/sftp.go b/sftp/sftp.go index 995db58..edab0a8 100644 --- a/sftp/sftp.go +++ b/sftp/sftp.go @@ -11,7 +11,7 @@ import ( var noMatchingServerError = errors.New("no matching server with that UUID was found") func Initialize(config config.SystemConfiguration) error { - s := &Server{ + s := &SFTPServer{ User: User{ Uid: config.User.Uid, Gid: config.User.Gid, @@ -22,18 +22,12 @@ func Initialize(config config.SystemConfiguration) error { BindAddress: config.Sftp.Address, BindPort: config.Sftp.Port, }, - CredentialValidator: validateCredentials, - PathValidator: validatePath, - DiskSpaceValidator: validateDiskSpace, - } - - if err := New(s); err != nil { - return err + credentialValidator: validateCredentials, } // Initialize the SFTP server in a background thread since this is // a long running operation. - go func(s *Server) { + go func(s *SFTPServer) { if err := s.Initialize(); err != nil { log.WithField("subsystem", "sftp").WithField("error", err).Error("failed to initialize SFTP subsystem") } @@ -42,30 +36,6 @@ func Initialize(config config.SystemConfiguration) error { return nil } -func validatePath(fs FileSystem, p string) (string, error) { - s := server.GetServers().Find(func(server *server.Server) bool { - return server.Id() == fs.UUID - }) - - if s == nil { - return "", noMatchingServerError - } - - return s.Filesystem().SafePath(p) -} - -func validateDiskSpace(fs FileSystem) bool { - s := server.GetServers().Find(func(server *server.Server) bool { - return server.Id() == fs.UUID - }) - - if s == nil { - return false - } - - return s.Filesystem().HasSpaceAvailable(true) -} - // Validates a set of credentials for a SFTP login against Pterodactyl Panel and returns // the server's UUID if the credentials were valid. func validateCredentials(c api.SftpAuthRequest) (*api.SftpAuthResponse, error) { diff --git a/sftp/lister.go b/sftp/utils.go similarity index 57% rename from sftp/lister.go rename to sftp/utils.go index 129020a..3ef6c5a 100644 --- a/sftp/lister.go +++ b/sftp/utils.go @@ -6,6 +6,14 @@ import ( ) type ListerAt []os.FileInfo +type fxerr uint32 + +const ( + // Extends the default SFTP server to return a quota exceeded error to the client. + // + // @see https://tools.ietf.org/id/draft-ietf-secsh-filexfer-13.txt + ErrSSHQuotaExceeded = fxerr(15) +) // Returns the number of entries copied and an io.EOF error if we made it to the end of the file list. // Take a look at the pkg/sftp godoc for more information about how this function should work. @@ -20,3 +28,12 @@ func (l ListerAt) ListAt(f []os.FileInfo, offset int64) (int, error) { return n, nil } } + +func (e fxerr) Error() string { + switch e { + case ErrSSHQuotaExceeded: + return "Quota Exceeded" + default: + return "Failure" + } +} From a48abc92adc3ec4881c3ae90d9ab10d9bd5b7db4 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 14:43:27 -0800 Subject: [PATCH 05/30] More code cleanup and struct simplification --- cmd/root.go | 4 +- sftp/server.go | 110 +++++++++++++++++++++++++++---------------------- sftp/sftp.go | 67 ------------------------------ sftp/utils.go | 7 ++-- 4 files changed, 67 insertions(+), 121 deletions(-) delete mode 100644 sftp/sftp.go diff --git a/cmd/root.go b/cmd/root.go index 4ddf4f2..78c79cf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -272,8 +272,8 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Wait until all of the servers are ready to go before we fire up the SFTP and HTTP servers. pool.StopWait() - // Initialize the SFTP server. - if err := sftp.Initialize(c.System); err != nil { + // Run the SFTP server. + if err := sftp.NewServer().Run(); err != nil { log.WithError(err).Fatal("failed to initialize the sftp server") return } diff --git a/sftp/server.go b/sftp/server.go index 294122a..4697036 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -13,6 +13,7 @@ import ( "path" "strings" + "emperror.dev/errors" "github.com/apex/log" "github.com/pkg/sftp" "github.com/pterodactyl/wings/api" @@ -21,59 +22,35 @@ import ( "golang.org/x/crypto/ssh" ) -type Settings struct { +//goland:noinspection GoNameStartsWithPackageName +type SFTPServer struct { BasePath string ReadOnly bool BindPort int BindAddress string } -type User struct { - Uid int - Gid int +var noMatchingServerError = errors.Sentinel("sftp: no matching server with UUID") + +func NewServer() *SFTPServer { + cfg := config.Get().System + return &SFTPServer{ + BasePath: cfg.Data, + ReadOnly: cfg.Sftp.ReadOnly, + BindAddress: cfg.Sftp.Address, + BindPort: cfg.Sftp.Port, + } } -//goland:noinspection GoNameStartsWithPackageName -type SFTPServer struct { - Settings Settings - User User - // Validator function that is called when a user connects to the server. This should - // check against whatever system is desired to confirm if the given username and password - // combination is valid. If so, should return an authentication response. - credentialValidator func(r api.SftpAuthRequest) (*api.SftpAuthResponse, error) -} - -// Initialize the SFTP server and add a persistent listener to handle inbound SFTP connections. -func (c *SFTPServer) Initialize() error { +// Starts the SFTP server and add a persistent listener to handle inbound SFTP connections. +func (c *SFTPServer) Run() error { serverConfig := &ssh.ServerConfig{ - NoClientAuth: false, - MaxAuthTries: 6, - PasswordCallback: func(conn ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { - resp, err := c.credentialValidator(api.SftpAuthRequest{ - User: conn.User(), - Pass: string(pass), - IP: conn.RemoteAddr().String(), - SessionID: conn.SessionID(), - ClientVersion: conn.ClientVersion(), - }) - - if err != nil { - return nil, err - } - - sshPerm := &ssh.Permissions{ - Extensions: map[string]string{ - "uuid": resp.Server, - "user": conn.User(), - "permissions": strings.Join(resp.Permissions, ","), - }, - } - - return sshPerm, nil - }, + NoClientAuth: false, + MaxAuthTries: 6, + PasswordCallback: c.passwordCallback, } - if _, err := os.Stat(path.Join(c.Settings.BasePath, ".sftp/id_rsa")); os.IsNotExist(err) { + if _, err := os.Stat(path.Join(c.BasePath, ".sftp/id_rsa")); os.IsNotExist(err) { if err := c.generatePrivateKey(); err != nil { return err } @@ -81,7 +58,7 @@ func (c *SFTPServer) Initialize() error { return err } - privateBytes, err := ioutil.ReadFile(path.Join(c.Settings.BasePath, ".sftp/id_rsa")) + privateBytes, err := ioutil.ReadFile(path.Join(c.BasePath, ".sftp/id_rsa")) if err != nil { return err } @@ -94,12 +71,12 @@ func (c *SFTPServer) Initialize() error { // Add our private key to the server configuration. serverConfig.AddHostKey(private) - listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", c.Settings.BindAddress, c.Settings.BindPort)) + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", c.BindAddress, c.BindPort)) if err != nil { return err } - log.WithField("host", c.Settings.BindAddress).WithField("port", c.Settings.BindPort).Info("sftp subsystem listening for connections") + log.WithField("host", c.BindAddress).WithField("port", c.BindPort).Info("sftp subsystem listening for connections") for { conn, _ := listener.Accept() @@ -171,7 +148,7 @@ func (c SFTPServer) AcceptInboundConnection(conn net.Conn, config *ssh.ServerCon // Creates a new SFTP handler for a given server. The directory argument should // be the base directory for a server. All actions done on the server will be // relative to that directory, and the user will not be able to escape out of it. -func (c SFTPServer) newHandler(sc *ssh.ServerConn) sftp.Handlers { +func (c *SFTPServer) newHandler(sc *ssh.ServerConn) sftp.Handlers { s := server.GetServers().Find(func(s *server.Server) bool { return s.Id() == sc.Permissions.Extensions["uuid"] }) @@ -196,17 +173,17 @@ func (c SFTPServer) newHandler(sc *ssh.ServerConn) sftp.Handlers { } // Generates a private key that will be used by the SFTP server. -func (c SFTPServer) generatePrivateKey() error { +func (c *SFTPServer) generatePrivateKey() error { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return err } - if err := os.MkdirAll(path.Join(c.Settings.BasePath, ".sftp"), 0755); err != nil { + if err := os.MkdirAll(path.Join(c.BasePath, ".sftp"), 0755); err != nil { return err } - o, err := os.OpenFile(path.Join(c.Settings.BasePath, ".sftp/id_rsa"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + o, err := os.OpenFile(path.Join(c.BasePath, ".sftp/id_rsa"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return err } @@ -223,3 +200,38 @@ func (c SFTPServer) generatePrivateKey() error { return nil } + +// A function capable of validating user credentials with the Panel API. +func (c *SFTPServer) passwordCallback(conn ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + request := api.SftpAuthRequest{ + User: conn.User(), + Pass: string(pass), + IP: conn.RemoteAddr().String(), + SessionID: conn.SessionID(), + ClientVersion: conn.ClientVersion(), + } + + logger := log.WithFields(log.Fields{"subsystem": "sftp", "username": conn.User(), "ip": conn.RemoteAddr().String()}) + logger.Debug("validating credentials for SFTP connection") + + resp, err := api.New().ValidateSftpCredentials(request) + if err != nil { + if api.IsInvalidCredentialsError(err) { + logger.Warn("failed to validate user credentials (invalid username or password)") + } else { + logger.Error("encountered an error while trying to validate user credentials") + } + return nil, err + } + + logger.WithField("server", resp.Server).Debug("credentials validated and matched to server instance") + sshPerm := &ssh.Permissions{ + Extensions: map[string]string{ + "uuid": resp.Server, + "user": conn.User(), + "permissions": strings.Join(resp.Permissions, ","), + }, + } + + return sshPerm, nil +} diff --git a/sftp/sftp.go b/sftp/sftp.go deleted file mode 100644 index edab0a8..0000000 --- a/sftp/sftp.go +++ /dev/null @@ -1,67 +0,0 @@ -package sftp - -import ( - "emperror.dev/errors" - "github.com/apex/log" - "github.com/pterodactyl/wings/api" - "github.com/pterodactyl/wings/config" - "github.com/pterodactyl/wings/server" -) - -var noMatchingServerError = errors.New("no matching server with that UUID was found") - -func Initialize(config config.SystemConfiguration) error { - s := &SFTPServer{ - User: User{ - Uid: config.User.Uid, - Gid: config.User.Gid, - }, - Settings: Settings{ - BasePath: config.Data, - ReadOnly: config.Sftp.ReadOnly, - BindAddress: config.Sftp.Address, - BindPort: config.Sftp.Port, - }, - credentialValidator: validateCredentials, - } - - // Initialize the SFTP server in a background thread since this is - // a long running operation. - go func(s *SFTPServer) { - if err := s.Initialize(); err != nil { - log.WithField("subsystem", "sftp").WithField("error", err).Error("failed to initialize SFTP subsystem") - } - }(s) - - return nil -} - -// Validates a set of credentials for a SFTP login against Pterodactyl Panel and returns -// the server's UUID if the credentials were valid. -func validateCredentials(c api.SftpAuthRequest) (*api.SftpAuthResponse, error) { - f := log.Fields{"subsystem": "sftp", "username": c.User, "ip": c.IP} - - log.WithFields(f).Debug("validating credentials for SFTP connection") - resp, err := api.New().ValidateSftpCredentials(c) - if err != nil { - if api.IsInvalidCredentialsError(err) { - log.WithFields(f).Warn("failed to validate user credentials (invalid username or password)") - } else { - log.WithFields(f).Error("encountered an error while trying to validate user credentials") - } - - return resp, err - } - - s := server.GetServers().Find(func(server *server.Server) bool { - return server.Id() == resp.Server - }) - - if s == nil { - return resp, noMatchingServerError - } - - s.Log().WithFields(f).Debug("credentials successfully validated and matched user to server instance") - - return resp, err -} diff --git a/sftp/utils.go b/sftp/utils.go index 3ef6c5a..5dad454 100644 --- a/sftp/utils.go +++ b/sftp/utils.go @@ -5,9 +5,6 @@ import ( "os" ) -type ListerAt []os.FileInfo -type fxerr uint32 - const ( // Extends the default SFTP server to return a quota exceeded error to the client. // @@ -15,6 +12,8 @@ const ( ErrSSHQuotaExceeded = fxerr(15) ) +type ListerAt []os.FileInfo + // Returns the number of entries copied and an io.EOF error if we made it to the end of the file list. // Take a look at the pkg/sftp godoc for more information about how this function should work. func (l ListerAt) ListAt(f []os.FileInfo, offset int64) (int, error) { @@ -29,6 +28,8 @@ func (l ListerAt) ListAt(f []os.FileInfo, offset int64) (int, error) { } } +type fxerr uint32 + func (e fxerr) Error() string { switch e { case ErrSSHQuotaExceeded: From 0cb3b815d172752eabc928db7248e2e59b45bd40 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 15:06:06 -0800 Subject: [PATCH 06/30] Finish refactoring SFTP server logic --- cmd/root.go | 2 +- sftp/handler.go | 27 ++++++++ sftp/server.go | 165 ++++++++++++++++++------------------------------ 3 files changed, 90 insertions(+), 104 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 78c79cf..18dccb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -273,7 +273,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { pool.StopWait() // Run the SFTP server. - if err := sftp.NewServer().Run(); err != nil { + if err := sftp.New().Run(); err != nil { log.WithError(err).Fatal("failed to initialize the sftp server") return } diff --git a/sftp/handler.go b/sftp/handler.go index 8957583..1461876 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -11,7 +11,9 @@ import ( "emperror.dev/errors" "github.com/apex/log" "github.com/pkg/sftp" + "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/server/filesystem" + "golang.org/x/crypto/ssh" ) const ( @@ -30,6 +32,31 @@ type Handler struct { ro bool } +// Returns a new connection handler for the SFTP server. This allows a given user +// to access the underlying filesystem. +func NewHandler(sc *ssh.ServerConn, fs *filesystem.Filesystem) *Handler { + return &Handler{ + fs: fs, + ro: config.Get().System.Sftp.ReadOnly, + permissions: strings.Split(sc.Permissions.Extensions["permissions"], ","), + logger: log.WithFields(log.Fields{ + "subsystem": "sftp", + "username": sc.User(), + "ip": sc.RemoteAddr(), + }), + } +} + +// Returns the sftp.Handlers for this struct. +func (h *Handler) Handlers() sftp.Handlers { + return sftp.Handlers{ + FileGet: h, + FilePut: h, + FileCmd: h, + FileList: h, + } +} + // Fileread creates a reader for a file on the system and returns the reader back. func (h *Handler) Fileread(request *sftp.Request) (io.ReaderAt, error) { // Check first if the user can actually open and view a file. This permission is named diff --git a/sftp/server.go b/sftp/server.go index 4697036..3604430 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -5,12 +5,12 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" - "fmt" "io" "io/ioutil" "net" "os" "path" + "strconv" "strings" "emperror.dev/errors" @@ -24,181 +24,140 @@ import ( //goland:noinspection GoNameStartsWithPackageName type SFTPServer struct { - BasePath string - ReadOnly bool - BindPort int - BindAddress string + BasePath string + ReadOnly bool + Listen string } -var noMatchingServerError = errors.Sentinel("sftp: no matching server with UUID") - -func NewServer() *SFTPServer { +func New() *SFTPServer { cfg := config.Get().System return &SFTPServer{ - BasePath: cfg.Data, - ReadOnly: cfg.Sftp.ReadOnly, - BindAddress: cfg.Sftp.Address, - BindPort: cfg.Sftp.Port, + BasePath: cfg.Data, + ReadOnly: cfg.Sftp.ReadOnly, + Listen: cfg.Sftp.Address + ":" + strconv.Itoa(cfg.Sftp.Port), } } // Starts the SFTP server and add a persistent listener to handle inbound SFTP connections. func (c *SFTPServer) Run() error { - serverConfig := &ssh.ServerConfig{ - NoClientAuth: false, - MaxAuthTries: 6, - PasswordCallback: c.passwordCallback, - } - if _, err := os.Stat(path.Join(c.BasePath, ".sftp/id_rsa")); os.IsNotExist(err) { if err := c.generatePrivateKey(); err != nil { return err } } else if err != nil { - return err + return errors.Wrap(err, "sftp/server: could not stat private key file") } - - privateBytes, err := ioutil.ReadFile(path.Join(c.BasePath, ".sftp/id_rsa")) + pb, err := ioutil.ReadFile(path.Join(c.BasePath, ".sftp/id_rsa")) + if err != nil { + return errors.Wrap(err, "sftp/server: could not read private key file") + } + private, err := ssh.ParsePrivateKey(pb) if err != nil { return err } - private, err := ssh.ParsePrivateKey(privateBytes) + conf := &ssh.ServerConfig{ + NoClientAuth: false, + MaxAuthTries: 6, + PasswordCallback: c.passwordCallback, + } + conf.AddHostKey(private) + + listener, err := net.Listen("tcp", c.Listen) if err != nil { return err } - // Add our private key to the server configuration. - serverConfig.AddHostKey(private) - - listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", c.BindAddress, c.BindPort)) - if err != nil { - return err - } - - log.WithField("host", c.BindAddress).WithField("port", c.BindPort).Info("sftp subsystem listening for connections") - + log.WithField("listen", c.Listen).Info("sftp server listening for connections") for { - conn, _ := listener.Accept() - if conn != nil { - go c.AcceptInboundConnection(conn, serverConfig) + if conn, _ := listener.Accept(); conn != nil { + go func(conn net.Conn) { + defer conn.Close() + c.AcceptInbound(conn, conf) + }(conn) } } } -// Handles an inbound connection to the instance and determines if we should serve the request -// or not. -func (c SFTPServer) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) { - defer conn.Close() - +// Handles an inbound connection to the instance and determines if we should serve the +// request or not. +func (c SFTPServer) AcceptInbound(conn net.Conn, config *ssh.ServerConfig) { // Before beginning a handshake must be performed on the incoming net.Conn sconn, chans, reqs, err := ssh.NewServerConn(conn, config) if err != nil { return } defer sconn.Close() - go ssh.DiscardRequests(reqs) - for newChannel := range chans { + for ch := range chans { // If its not a session channel we just move on because its not something we // know how to handle at this point. - if newChannel.ChannelType() != "session" { - newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + if ch.ChannelType() != "session" { + ch.Reject(ssh.UnknownChannelType, "unknown channel type") continue } - channel, requests, err := newChannel.Accept() + channel, requests, err := ch.Accept() if err != nil { continue } - // Channels have a type that is dependent on the protocol. For SFTP this is "subsystem" - // with a payload that (should) be "sftp". Discard anything else we receive ("pty", "shell", etc) go func(in <-chan *ssh.Request) { for req := range in { - ok := false - - switch req.Type { - case "subsystem": - if string(req.Payload[4:]) == "sftp" { - ok = true - } - } - - req.Reply(ok, nil) + // Channels have a type that is dependent on the protocol. For SFTP + // this is "subsystem" with a payload that (should) be "sftp". Discard + // anything else we receive ("pty", "shell", etc) + req.Reply(req.Type == "subsystem" && string(req.Payload[4:]) == "sftp", nil) } }(requests) - if sconn.Permissions.Extensions["uuid"] == "" { + // If no UUID has been set on this inbound request then we can assume we + // have screwed up something in the authentication code. This is a sanity + // check, but should never be encountered (ideally...). + // + // This will also attempt to match a specific server out of the global server + // store and return nil if there is no match. + uuid := sconn.Permissions.Extensions["uuid"] + srv := server.GetServers().Find(func(s *server.Server) bool { + if uuid == "" { + return false + } + return s.Id() == uuid + }) + if srv == nil { continue } - // Create a new handler for the currently logged in user's server. - fs := c.newHandler(sconn) - - // Create the server instance for the channel using the filesystem we created above. - handler := sftp.NewRequestServer(channel, fs) + // Spin up a SFTP server instance for the authenticated user's server allowing + // them access to the underlying filesystem. + handler := sftp.NewRequestServer(channel, NewHandler(sconn, srv.Filesystem()).Handlers()) if err := handler.Serve(); err == io.EOF { handler.Close() } } } -// Creates a new SFTP handler for a given server. The directory argument should -// be the base directory for a server. All actions done on the server will be -// relative to that directory, and the user will not be able to escape out of it. -func (c *SFTPServer) newHandler(sc *ssh.ServerConn) sftp.Handlers { - s := server.GetServers().Find(func(s *server.Server) bool { - return s.Id() == sc.Permissions.Extensions["uuid"] - }) - - p := Handler{ - fs: s.Filesystem(), - permissions: strings.Split(sc.Permissions.Extensions["permissions"], ","), - ro: config.Get().System.Sftp.ReadOnly, - logger: log.WithFields(log.Fields{ - "subsystem": "sftp", - "username": sc.User(), - "ip": sc.RemoteAddr(), - }), - } - - return sftp.Handlers{ - FileGet: &p, - FilePut: &p, - FileCmd: &p, - FileList: &p, - } -} - // Generates a private key that will be used by the SFTP server. func (c *SFTPServer) generatePrivateKey() error { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { - return err + return errors.WithStack(err) } - if err := os.MkdirAll(path.Join(c.BasePath, ".sftp"), 0755); err != nil { - return err + return errors.Wrap(err, "sftp/server: could not create .sftp directory") } - o, err := os.OpenFile(path.Join(c.BasePath, ".sftp/id_rsa"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { - return err + return errors.WithStack(err) } defer o.Close() - pkey := &pem.Block{ + err = pem.Encode(o, &pem.Block{ Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), - } - - if err := pem.Encode(o, pkey); err != nil { - return err - } - - return nil + }) + return errors.WithStack(err) } // A function capable of validating user credentials with the Panel API. From b0fa9619dec3033349195a243efdbaf3382edf35 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 15:09:33 -0800 Subject: [PATCH 07/30] debug --- server/listeners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/listeners.go b/server/listeners.go index 767cbf1..77f6cf6 100644 --- a/server/listeners.go +++ b/server/listeners.go @@ -136,7 +136,7 @@ func (s *Server) StartEventListeners() { } } - s.Log().Info("registering event listeners: console, state, resources...") + s.Log().Debug("registering event listeners: console, state, resources...") s.Environment.Events().On(environment.ConsoleOutputEvent, &console) s.Environment.Events().On(environment.StateChangeEvent, &state) s.Environment.Events().On(environment.ResourceEvent, &stats) From 1e15beb953c1fa169da47be80315f377c02216b2 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 15:12:13 -0800 Subject: [PATCH 08/30] double log, I think not? --- api/sftp_endpoints.go | 6 ------ sftp/server.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/api/sftp_endpoints.go b/api/sftp_endpoints.go index b1e0f39..e222749 100644 --- a/api/sftp_endpoints.go +++ b/api/sftp_endpoints.go @@ -62,12 +62,6 @@ func (r *Request) ValidateSftpCredentials(request SftpAuthRequest) (*SftpAuthRes e := resp.Error() if e != nil { if resp.StatusCode >= 400 && resp.StatusCode < 500 { - log.WithFields(log.Fields{ - "subsystem": "sftp", - "username": request.User, - "ip": request.IP, - }).Warn(e.Error()) - return nil, &sftpInvalidCredentialsError{} } diff --git a/sftp/server.go b/sftp/server.go index 3604430..b50650d 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -178,7 +178,7 @@ func (c *SFTPServer) passwordCallback(conn ssh.ConnMetadata, pass []byte) (*ssh. if api.IsInvalidCredentialsError(err) { logger.Warn("failed to validate user credentials (invalid username or password)") } else { - logger.Error("encountered an error while trying to validate user credentials") + logger.WithField("error", err).Error("encountered an error while trying to validate user credentials") } return nil, err } From 3459c25be08c6cf0e7c4f720157bf8ad934c8052 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 15:59:45 -0800 Subject: [PATCH 09/30] Don't block the proc --- cmd/root.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 18dccb8..0414c79 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -272,11 +272,13 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Wait until all of the servers are ready to go before we fire up the SFTP and HTTP servers. pool.StopWait() - // Run the SFTP server. - if err := sftp.New().Run(); err != nil { - log.WithError(err).Fatal("failed to initialize the sftp server") - return - } + go func() { + // Run the SFTP server. + if err := sftp.New().Run(); err != nil { + log.WithError(err).Fatal("failed to initialize the sftp server") + return + } + }() // Ensure the archive directory exists. if err := os.MkdirAll(c.System.ArchiveDirectory, 0755); err != nil { From 2c1b211280b872db28b59640087ea0a15645aba5 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 16:33:39 -0800 Subject: [PATCH 10/30] Add base idea for denying write access to certain files; ref pterodactyl/panel#569 --- router/router_server_files.go | 24 +++++++++++++++++++----- server/configuration.go | 11 +++++++++++ server/filesystem/decompress.go | 7 +++---- server/filesystem/errors.go | 8 ++++++-- server/filesystem/filesystem.go | 5 ++++- server/filesystem/path.go | 18 +++++++++++++++++- server/loader.go | 2 +- 7 files changed, 61 insertions(+), 14 deletions(-) diff --git a/router/router_server_files.go b/router/router_server_files.go index d0e6da5..bee077d 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -94,8 +94,7 @@ func putServerRenameFiles(c *gin.Context) { return } - g, ctx := errgroup.WithContext(context.Background()) - + g, ctx := errgroup.WithContext(c.Request.Context()) // Loop over the array of files passed in and perform the move or rename action against each. for _, p := range data.Files { pf := path.Join(data.Root, p.From) @@ -106,16 +105,20 @@ func putServerRenameFiles(c *gin.Context) { case <-ctx.Done(): return ctx.Err() default: - if err := s.Filesystem().Rename(pf, pt); err != nil { + fs := s.Filesystem() + // Ignore renames on a file that is on the denylist (both as the rename from or + // the rename to value). + if err := fs.IsIgnored(pf, pt); err != nil { + return err + } + if err := fs.Rename(pf, pt); err != nil { // Return nil if the error is an is not exists. // NOTE: os.IsNotExist() does not work if the error is wrapped. if errors.Is(err, os.ErrNotExist) { return nil } - return err } - return nil } }) @@ -148,6 +151,10 @@ func postServerCopyFile(c *gin.Context) { return } + if err := s.Filesystem().IsIgnored(data.Location); err != nil { + NewServerError(err, s).Abort(c) + return + } if err := s.Filesystem().Copy(data.Location); err != nil { NewServerError(err, s).AbortFilesystemError(c) return @@ -208,6 +215,10 @@ func postServerWriteFile(c *gin.Context) { f := c.Query("file") f = "/" + strings.TrimLeft(f, "/") + if err := s.Filesystem().IsIgnored(f); err != nil { + NewServerError(err, s).Abort(c) + return + } if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil { if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ @@ -557,6 +568,9 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) } defer file.Close() + if err := s.Filesystem().IsIgnored(p); err != nil { + return err + } if err := s.Filesystem().Writefile(p, file); err != nil { return err } diff --git a/server/configuration.go b/server/configuration.go index 5013650..b7f16b7 100644 --- a/server/configuration.go +++ b/server/configuration.go @@ -6,6 +6,16 @@ import ( "github.com/pterodactyl/wings/environment" ) +type EggConfiguration struct { + // The internal UUID of the Egg on the Panel. + ID string + + // Maintains a list of files that are blacklisted for opening/editing/downloading + // or basically any type of access on the server by any user. This is NOT the same + // as a per-user denylist, this is defined at the Egg level. + FileDenylist []string `json:"file_denylist"` +} + type Configuration struct { mu sync.RWMutex @@ -34,6 +44,7 @@ type Configuration struct { CrashDetectionEnabled bool `default:"true" json:"enabled" yaml:"enabled"` Mounts []Mount `json:"mounts"` Resources ResourceUsage `json:"resources"` + Egg EggConfiguration `json:"egg,omitempty"` Container struct { // Defines the Docker image that will be used for this server diff --git a/server/filesystem/decompress.go b/server/filesystem/decompress.go index 7ff4593..66d7901 100644 --- a/server/filesystem/decompress.go +++ b/server/filesystem/decompress.go @@ -91,11 +91,10 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { return errors.New(fmt.Sprintf("could not parse underlying data source with type %s", reflect.TypeOf(s).String())) } - p, err := fs.SafePath(filepath.Join(dir, name)) - if err != nil { - return errors.WithMessage(err, "failed to generate a safe path to server file") + p := filepath.Join(dir, name) + if err := fs.IsIgnored(p); err != nil { + return err } - return errors.WithMessage(fs.Writefile(p, f), "could not extract file from archive") }) if err != nil { diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index a5890bd..bb3f1af 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -1,11 +1,12 @@ package filesystem import ( - "emperror.dev/errors" "fmt" - "github.com/apex/log" "os" "path/filepath" + + "emperror.dev/errors" + "github.com/apex/log" ) type ErrorCode string @@ -15,6 +16,7 @@ const ( ErrCodeDiskSpace ErrorCode = "E_NODISK" ErrCodeUnknownArchive ErrorCode = "E_UNKNFMT" ErrCodePathResolution ErrorCode = "E_BADPATH" + ErrCodeDenylistFile ErrorCode = "E_DENYLIST" ) type Error struct { @@ -32,6 +34,8 @@ func (e *Error) Error() string { return "filesystem: not enough disk space" case ErrCodeUnknownArchive: return "filesystem: unknown archive format" + case ErrCodeDenylistFile: + return "filesystem: file access prohibited: denylist" case ErrCodePathResolution: r := e.resolved if r == "" { diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index a4a49de..4732a22 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -18,6 +18,7 @@ import ( "github.com/karrick/godirwalk" "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/system" + ignore "github.com/sabhiram/go-gitignore" ) type Filesystem struct { @@ -26,6 +27,7 @@ type Filesystem struct { lookupInProgress *system.AtomicBool diskUsed int64 diskCheckInterval time.Duration + denylist *ignore.GitIgnore // The maximum amount of disk space (in bytes) that this Filesystem instance can use. diskLimit int64 @@ -37,13 +39,14 @@ type Filesystem struct { } // Creates a new Filesystem instance for a given server. -func New(root string, size int64) *Filesystem { +func New(root string, size int64, denylist []string) *Filesystem { return &Filesystem{ root: root, diskLimit: size, diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval), lastLookupTime: &usageLookupTime{}, lookupInProgress: system.NewAtomicBool(false), + denylist: ignore.CompileIgnoreLines(denylist...), } } diff --git a/server/filesystem/path.go b/server/filesystem/path.go index 901c40b..e816854 100644 --- a/server/filesystem/path.go +++ b/server/filesystem/path.go @@ -2,13 +2,29 @@ package filesystem import ( "context" - "golang.org/x/sync/errgroup" "os" "path/filepath" "strings" "sync" + + "golang.org/x/sync/errgroup" ) +// Checks if the given file or path is in the server's file denylist. If so, an Error +// is returned, otherwise nil is returned. +func (fs *Filesystem) IsIgnored(paths ...string) error { + for _, p := range paths { + sp, err := fs.SafePath(p) + if err != nil { + return err + } + if fs.denylist.MatchesPath(sp) { + return &Error{code: ErrCodeDenylistFile, path: p, resolved: sp} + } + } + return nil +} + // Normalizes a directory being passed in to ensure the user is not able to escape // from their data directory. After normalization if the directory is still within their home // path it is returned. If they managed to "escape" an error will be returned. diff --git a/server/loader.go b/server/loader.go index c2d401e..1c0e2b8 100644 --- a/server/loader.go +++ b/server/loader.go @@ -96,7 +96,7 @@ func FromConfiguration(data api.ServerConfigurationResponse) (*Server, error) { } s.Archiver = Archiver{Server: s} - s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace()) + s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.Id()), s.DiskSpace(), s.Config().Egg.FileDenylist) // Right now we only support a Docker based environment, so I'm going to hard code // this logic in. When we're ready to support other environment we'll need to make From b10e4dd43754ce447005acd08994b4d779bd2e30 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 16:43:33 -0800 Subject: [PATCH 11/30] Better error handling for access to denylist files --- router/error.go | 18 ++++++++++-------- server/filesystem/errors.go | 6 +++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/router/error.go b/router/error.go index f56876a..7503f89 100644 --- a/router/error.go +++ b/router/error.go @@ -122,20 +122,22 @@ func (e *RequestError) Abort(c *gin.Context) { // Looks at the given RequestError and determines if it is a specific filesystem error that // we can process and return differently for the user. func (e *RequestError) getAsFilesystemError() (int, string) { - err := errors.Unwrap(e.err) - if err == nil { - return 0, "" - } - if errors.Is(err, os.ErrNotExist) || filesystem.IsErrorCode(err, filesystem.ErrCodePathResolution) { + if filesystem.IsErrorCode(e.err, filesystem.ErrCodePathResolution) || errors.Is(e.err, os.ErrNotExist) { return http.StatusNotFound, "The requested resource was not found on the system." } - if filesystem.IsErrorCode(err, filesystem.ErrCodeDiskSpace) { + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDiskSpace) { return http.StatusConflict, "There is not enough disk space available to perform that action." } - if strings.HasSuffix(err.Error(), "file name too long") { + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDenylistFile) { + return http.StatusForbidden, "This file cannot be modified: present in egg denylist." + } + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeIsDirectory) { + return http.StatusBadRequest, "Cannot perform that action: file is a directory." + } + if strings.HasSuffix(e.err.Error(), "file name too long") { return http.StatusBadRequest, "Cannot perform that action: file name is too long." } - if e, ok := err.(*os.SyscallError); ok && e.Syscall == "readdirent" { + if e, ok := e.err.(*os.SyscallError); ok && e.Syscall == "readdirent" { return http.StatusNotFound, "The requested directory does not exist." } return 0, "" diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index bb3f1af..4cc9929 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -35,7 +35,11 @@ func (e *Error) Error() string { case ErrCodeUnknownArchive: return "filesystem: unknown archive format" case ErrCodeDenylistFile: - return "filesystem: file access prohibited: denylist" + r := e.resolved + if r == "" { + r = "" + } + return fmt.Sprintf("filesystem: file access prohibited: [%s] is on the denylist", r) case ErrCodePathResolution: r := e.resolved if r == "" { From e9e70b60817fc58683e1a9a5d508e3e9f2fdc668 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 10 Jan 2021 17:01:41 -0800 Subject: [PATCH 12/30] Better error handling; skip file when unarchiving --- router/error.go | 20 +++++++++++--------- router/router_server_files.go | 13 +++---------- server/filesystem/decompress.go | 3 ++- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/router/error.go b/router/error.go index 7503f89..f7be956 100644 --- a/router/error.go +++ b/router/error.go @@ -77,7 +77,6 @@ func (e *RequestError) AbortWithStatus(status int, c *gin.Context) { // If this error is because the resource does not exist, we likely do not need to log // the error anywhere, just return a 404 and move on with our lives. if errors.Is(e.err, os.ErrNotExist) { - e.logger().Debug("encountered os.IsNotExist error while handling request") c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The requested resource was not found on the system.", }) @@ -122,16 +121,19 @@ func (e *RequestError) Abort(c *gin.Context) { // Looks at the given RequestError and determines if it is a specific filesystem error that // we can process and return differently for the user. func (e *RequestError) getAsFilesystemError() (int, string) { - if filesystem.IsErrorCode(e.err, filesystem.ErrCodePathResolution) || errors.Is(e.err, os.ErrNotExist) { - return http.StatusNotFound, "The requested resource was not found on the system." - } - if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDiskSpace) { - return http.StatusConflict, "There is not enough disk space available to perform that action." - } - if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDenylistFile) { + // Some external things end up calling fmt.Errorf() on our filesystem errors + // which ends up just unleashing chaos on the system. For the sake of this + // fallback to using text checks... + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDenylistFile) || strings.Contains(e.err.Error(), "filesystem: file access prohibited") { return http.StatusForbidden, "This file cannot be modified: present in egg denylist." } - if filesystem.IsErrorCode(e.err, filesystem.ErrCodeIsDirectory) { + if filesystem.IsErrorCode(e.err, filesystem.ErrCodePathResolution) || strings.Contains(e.err.Error(), "resolves to a location outside the server root") { + return http.StatusNotFound, "The requested resource was not found on the system." + } + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeIsDirectory) || strings.Contains(e.err.Error(), "filesystem: is a directory") { + return http.StatusBadRequest, "Cannot perform that action: file is a directory." + } + if filesystem.IsErrorCode(e.err, filesystem.ErrCodeDiskSpace) || strings.Contains(e.err.Error(), "filesystem: not enough disk space") { return http.StatusBadRequest, "Cannot perform that action: file is a directory." } if strings.HasSuffix(e.err.Error(), "file name too long") { diff --git a/router/router_server_files.go b/router/router_server_files.go index bee077d..b6a3af5 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -410,13 +410,6 @@ func postServerDecompressFiles(c *gin.Context) { } if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil { - if errors.Is(err, os.ErrNotExist) { - c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ - "error": "The requested archive was not found.", - }) - return - } - // If the file is busy for some reason just return a nicer error to the user since there is not // much we specifically can do. They'll need to stop the running server process in order to overwrite // a file like this. @@ -429,7 +422,7 @@ func postServerDecompressFiles(c *gin.Context) { return } - NewServerError(err, s).AbortFilesystemError(c) + NewServerError(err, s).Abort(c) return } @@ -548,14 +541,14 @@ func postServerUploadFiles(c *gin.Context) { for _, header := range headers { p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename)) if err != nil { - NewServerError(err, s).AbortFilesystemError(c) + NewServerError(err, s).Abort(c) return } // We run this in a different method so I can use defer without any of // the consequences caused by calling it in a loop. if err := handleFileUpload(p, s, header); err != nil { - NewServerError(err, s).AbortFilesystemError(c) + NewServerError(err, s).Abort(c) return } } diff --git a/server/filesystem/decompress.go b/server/filesystem/decompress.go index 66d7901..4fb2988 100644 --- a/server/filesystem/decompress.go +++ b/server/filesystem/decompress.go @@ -92,8 +92,9 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { } p := filepath.Join(dir, name) + // If it is ignored, just don't do anything with the file and skip over it. if err := fs.IsIgnored(p); err != nil { - return err + return nil } return errors.WithMessage(fs.Writefile(p, f), "could not extract file from archive") }) From d45a159456a3e1e71979c4b787c04ef9ecffc133 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Tue, 12 Jan 2021 20:07:00 -0800 Subject: [PATCH 13/30] Fix tests --- Makefile | 2 +- server/filesystem/filesystem_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index bc18a83..594cc64 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ build: debug: go build -race - ./wings --debug --ignore-certificate-errors --config config.yml + sudo ./wings --debug --ignore-certificate-errors --config config.yml compress: upx --brute build/wings_* diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index 10b0b70..8427738 100644 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -3,8 +3,6 @@ package filesystem import ( "bytes" "errors" - . "github.com/franela/goblin" - "github.com/pterodactyl/wings/config" "io/ioutil" "math/rand" "os" @@ -12,6 +10,9 @@ import ( "sync/atomic" "testing" "unicode/utf8" + + . "github.com/franela/goblin" + "github.com/pterodactyl/wings/config" ) func NewFs() (*Filesystem, *rootFs) { @@ -33,7 +34,7 @@ func NewFs() (*Filesystem, *rootFs) { rfs.reset() - fs := New(filepath.Join(tmpDir, "/server"), 0) + fs := New(filepath.Join(tmpDir, "/server"), 0, []string{}) fs.isTest = true return fs, &rfs From 9480ccdbba88d0bdde0c37ccd13016e3f3d73e86 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Tue, 12 Jan 2021 21:14:57 -0800 Subject: [PATCH 14/30] Initial WIP logic to handle loading configuration from the disk using viper --- .gitignore | 1 + cmd/config_finder.go | 61 ----------------------- cmd/root.go | 96 +++++++++++++++++------------------ config/config.go | 99 +++++++++++++------------------------ config/config_system.go | 107 ++++++++++++++++++++++------------------ environment/docker.go | 105 +++++++++++++++++---------------------- go.mod | 1 + go.sum | 9 ++++ system/utils.go | 21 ++++++++ 9 files changed, 213 insertions(+), 287 deletions(-) delete mode 100644 cmd/config_finder.go diff --git a/.gitignore b/.gitignore index a98e257..4f2bd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ # ignore configuration file /config.yml +/config*.yml # Ignore Vagrant stuff /.vagrant diff --git a/cmd/config_finder.go b/cmd/config_finder.go deleted file mode 100644 index 3e9f42a..0000000 --- a/cmd/config_finder.go +++ /dev/null @@ -1,61 +0,0 @@ -package cmd - -import ( - "os" - "path/filepath" - - "github.com/pterodactyl/wings/config" -) - -// We've gone through a couple of iterations of where the configuration is stored. This -// helpful little function will look through the three areas it might have ended up, and -// return it. -// -// We only run this if the configuration flag for the instance is not actually passed in -// via the command line. Once found, the configuration is moved into the expected default -// location. Only errors are returned from this function, you can safely assume that after -// running this the configuration can be found in the correct default location. -func RelocateConfiguration() error { - var match string - check := []string{ - config.DefaultLocation, - "/var/lib/pterodactyl/config.yml", - "/etc/wings/config.yml", - } - - // Loop over all of the configuration paths, and return which one we found, if - // any. - for _, p := range check { - if s, err := os.Stat(p); err != nil { - if !os.IsNotExist(err) { - return err - } - } else if !s.IsDir() { - match = p - break - } - } - - // Just return a generic not exist error at this point if we didn't have a match, this - // will allow the caller to handle displaying a more friendly error to the user. If we - // did match in the default location, go ahead and return successfully. - if match == "" { - return os.ErrNotExist - } else if match == config.DefaultLocation { - return nil - } - - // The rest of this function simply creates the new default location and moves the - // old configuration file over to the new location, then sets the permissions on the - // file correctly so that only the user running this process can read it. - p, _ := filepath.Split(config.DefaultLocation) - if err := os.MkdirAll(p, 0755); err != nil { - return err - } - - if err := os.Rename(match, config.DefaultLocation); err != nil { - return err - } - - return os.Chmod(config.DefaultLocation, 0600) -} diff --git a/cmd/root.go b/cmd/root.go index 0414c79..c1080b3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,12 +26,13 @@ import ( "github.com/pterodactyl/wings/sftp" "github.com/pterodactyl/wings/system" "github.com/spf13/cobra" + "github.com/spf13/viper" "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" ) var ( - configPath = config.DefaultLocation + configPath = "" debug = false ) @@ -64,7 +65,9 @@ func Execute() { } func init() { - rootCommand.PersistentFlags().StringVar(&configPath, "config", config.DefaultLocation, "set the location for the configuration file") + cobra.OnInitialize(initConfig, initLogging) + + rootCommand.PersistentFlags().StringVar(&configPath, "config", "", "set the location for the configuration file") rootCommand.PersistentFlags().BoolVar(&debug, "debug", false, "pass in order to run wings in debug mode") // Flags specifically used when running the API. @@ -119,17 +122,6 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { defer profile.Start(profile.BlockProfile).Stop() } - // Only attempt configuration file relocation if a custom location has not - // been specified in the command startup. - if configPath == config.DefaultLocation { - if err := RelocateConfiguration(); err != nil { - if errors.Is(err, os.ErrNotExist) { - exitWithConfigurationNotice() - } - panic(err) - } - } - c, err := readConfiguration() if err != nil { panic(err) @@ -140,14 +132,8 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { } printLogo() - if err := configureLogging(c.System.LogDirectory, c.Debug); err != nil { - panic(err) - } - - log.WithField("path", c.GetPath()).Info("loading configuration from path") - if c.Debug { - log.Debug("running in debug mode") - } + log.WithField("path", viper.ConfigFileUsed()).Info("loading configuration from file") + log.Debug("running in debug mode") if ok, _ := cmd.Flags().GetBool("ignore-certificate-errors"); ok { log.Warn("running with --ignore-certificate-errors: TLS certificate host chains and name will not be verified") @@ -158,47 +144,42 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { config.Set(c) config.SetDebugViaFlag(debug) - - if err := c.System.ConfigureTimezone(); err != nil { + if err := config.ConfigureTimezone(); err != nil { log.WithField("error", err).Fatal("failed to detect system timezone or use supplied configuration value") - return } - log.WithField("timezone", c.System.Timezone).Info("configured wings with system timezone") - if err := c.System.ConfigureDirectories(); err != nil { + if err := config.ConfigureDirectories(); err != nil { log.WithField("error", err).Fatal("failed to configure system directories for pterodactyl") return } - if err := c.System.EnableLogRotation(); err != nil { + if err := config.EnableLogRotation(); err != nil { log.WithField("error", err).Fatal("failed to configure log rotation on the system") return } log.WithField("username", c.System.Username).Info("checking for pterodactyl system user") - if su, err := c.EnsurePterodactylUser(); err != nil { + if err := config.EnsurePterodactylUser(); err != nil { log.WithField("error", err).Fatal("failed to create pterodactyl system user") - return - } else { - log.WithFields(log.Fields{ - "username": su.Username, - "uid": su.Uid, - "gid": su.Gid, - }).Info("configured system user successfully") } + log.WithFields(log.Fields{ + "username": viper.GetString("system.username"), + "uid": viper.GetInt("system.user.uid"), + "gid": viper.GetInt("system.user.gid"), + }).Info("configured system user successfully") if err := server.LoadDirectory(); err != nil { log.WithField("error", err).Fatal("failed to load server configurations") return } - if err := environment.ConfigureDocker(&c.Docker); err != nil { + if err := environment.ConfigureDocker(cmd.Context()); err != nil { log.WithField("error", err).Fatal("failed to configure docker environment") return } - if err := c.WriteToDisk(); err != nil { + if err := viper.WriteConfig(); err != nil { log.WithField("error", err).Error("failed to save configuration to disk") } @@ -379,28 +360,44 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { } } +func initConfig() { + if configPath != "" { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath("/etc/pterodactyl") + viper.AddConfigPath("$HOME/.pterodactyl") + viper.AddConfigPath(".") + } else { + viper.SetConfigFile(configPath) + } + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(*viper.ConfigFileNotFoundError); ok { + exitWithConfigurationNotice() + } + log2.Fatalf("cmd/root: failed to read configuration: %s", err) + } +} + // Configures the global logger for Zap so that we can call it from any location // in the code without having to pass around a logger instance. -func configureLogging(logDir string, debug bool) error { - if err := os.MkdirAll(path.Join(logDir, "/install"), 0700); err != nil { - return err +func initLogging() { + dir := viper.GetString("system.log_directory") + if err := os.MkdirAll(path.Join(dir, "/install"), 0700); err != nil { + log2.Fatalf("cmd/root: failed to create install directory path: %s", err) } - - p := filepath.Join(logDir, "/wings.log") + p := filepath.Join(dir, "/wings.log") w, err := logrotate.NewFile(p) if err != nil { - return err + log2.Fatalf("cmd/root: failed to create wings log: %s", err) } log.SetLevel(log.InfoLevel) - if debug { + if viper.GetBool("debug") { log.SetLevel(log.DebugLevel) } log.SetHandler(multi.New(cli.Default, cli.New(w.File, false))) log.WithField("path", p).Info("writing log files to disk") - - return nil } // Prints the wings logo, nothing special here! @@ -429,11 +426,8 @@ func exitWithConfigurationNotice() { [_red_][white][bold]Error: Configuration File Not Found[reset] Wings was not able to locate your configuration file, and therefore is not -able to complete its boot process. - -Please ensure you have copied your instance configuration file into -the default location, or have provided the --config flag to use a -custom location. +able to complete its boot process. Please ensure you have copied your instance +configuration file into the default location below. Default Location: /etc/pterodactyl/config.yml diff --git a/config/config.go b/config/config.go index 0af323f..8d2d3d9 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "os/user" - "strconv" "strings" "sync" @@ -14,6 +13,8 @@ import ( "github.com/cobaugh/osrelease" "github.com/creasty/defaults" "github.com/gbrlsnchs/jwt/v3" + "github.com/pterodactyl/wings/system" + "github.com/spf13/viper" "gopkg.in/yaml.v2" ) @@ -223,98 +224,64 @@ func (c *Configuration) GetPath() string { return c.path } -// Ensures that the Pterodactyl core user exists on the system. This user will be the -// owner of all data in the root data directory and is used as the user within containers. +// EnsurePterodactylUser ensures that the Pterodactyl core user exists on the +// system. This user will be the owner of all data in the root data directory +// and is used as the user within containers. // -// If files are not owned by this user there will be issues with permissions on Docker -// mount points. -func (c *Configuration) EnsurePterodactylUser() (*user.User, error) { +// If files are not owned by this user there will be issues with permissions on +// Docker mount points. +func EnsurePterodactylUser() error { sysName, err := getSystemName() if err != nil { - return nil, err + return err } // Our way of detecting if wings is running inside of Docker. if sysName == "busybox" { - uid := os.Getenv("WINGS_UID") - if uid == "" { - uid = "988" - } - - gid := os.Getenv("WINGS_GID") - if gid == "" { - gid = "988" - } - - username := os.Getenv("WINGS_USERNAME") - if username == "" { - username = "pterodactyl" - } - - u := &user.User{ - Uid: uid, - Gid: gid, - Username: username, - } - return u, c.setSystemUser(u) + viper.Set("system.username", system.FirstNotEmpty(os.Getenv("WINGS_USERNAME"), "pterodactyl")) + viper.Set("system.user.uid", system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_UID"), "988"))) + viper.Set("system.user.gid", system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_GID"), "988"))) + return nil } - u, err := user.Lookup(c.System.Username) - + username := viper.GetString("system.username") + u, err := user.Lookup(username) // If an error is returned but it isn't the unknown user error just abort // the process entirely. If we did find a user, return it immediately. - if err == nil { - return u, c.setSystemUser(u) - } else if _, ok := err.(user.UnknownUserError); !ok { - return nil, err + if err != nil { + if _, ok := err.(user.UnknownUserError); !ok { + return err + } + } else { + viper.Set("system.user.uid", system.MustInt(u.Uid)) + viper.Set("system.user.gid", system.MustInt(u.Gid)) + return nil } - command := fmt.Sprintf("useradd --system --no-create-home --shell /usr/sbin/nologin %s", c.System.Username) - - // Alpine Linux is the only OS we currently support that doesn't work with the useradd command, so - // in those cases we just modify the command a bit to work as expected. + command := fmt.Sprintf("useradd --system --no-create-home --shell /usr/sbin/nologin %s", username) + // Alpine Linux is the only OS we currently support that doesn't work with the useradd + // command, so in those cases we just modify the command a bit to work as expected. if strings.HasPrefix(sysName, "alpine") { - command = fmt.Sprintf("adduser -S -D -H -G %[1]s -s /sbin/nologin %[1]s", c.System.Username) - + command = fmt.Sprintf("adduser -S -D -H -G %[1]s -s /sbin/nologin %[1]s", username) // We have to create the group first on Alpine, so do that here before continuing on // to the user creation process. - if _, err := exec.Command("addgroup", "-S", c.System.Username).Output(); err != nil { - return nil, err + if _, err := exec.Command("addgroup", "-S", username).Output(); err != nil { + return err } } split := strings.Split(command, " ") if _, err := exec.Command(split[0], split[1:]...).Output(); err != nil { - return nil, err - } - - if u, err := user.Lookup(c.System.Username); err != nil { - return nil, err - } else { - return u, c.setSystemUser(u) - } -} - -// Set the system user into the configuration and then write it to the disk so that -// it is persisted on boot. -func (c *Configuration) setSystemUser(u *user.User) error { - uid, err := strconv.Atoi(u.Uid) - if err != nil { return err } - gid, err := strconv.Atoi(u.Gid) + u, err = user.Lookup(username) if err != nil { return err } - - c.Lock() - c.System.Username = u.Username - c.System.User.Uid = uid - c.System.User.Gid = gid - c.Unlock() - - return c.WriteToDisk() + viper.Set("system.user.uid", system.MustInt(u.Uid)) + viper.Set("system.user.gid", system.MustInt(u.Gid)) + return nil } // Writes the configuration to the disk as a blocking operation by obtaining an exclusive diff --git a/config/config_system.go b/config/config_system.go index c87dd21..0b2fd9a 100644 --- a/config/config_system.go +++ b/config/config_system.go @@ -14,6 +14,7 @@ import ( "emperror.dev/errors" "github.com/apex/log" + "github.com/spf13/viper" ) // Defines basic system configuration settings. @@ -116,11 +117,13 @@ type Transfers struct { DownloadLimit int `default:"0" yaml:"download_limit"` } -// Ensures that all of the system directories exist on the system. These directories are -// created so that only the owner can read the data, and no other users. -func (sc *SystemConfiguration) ConfigureDirectories() error { - log.WithField("path", sc.RootDirectory).Debug("ensuring root data directory exists") - if err := os.MkdirAll(sc.RootDirectory, 0700); err != nil { +// ConfigureDirectories ensures that all of the system directories exist on the +// system. These directories are created so that only the owner can read the data, +// and no other users. +func ConfigureDirectories() error { + root := viper.GetString("system.root_directory") + log.WithField("path", root).Debug("ensuring root data directory exists") + if err := os.MkdirAll(root, 0700); err != nil { return err } @@ -132,40 +135,42 @@ func (sc *SystemConfiguration) ConfigureDirectories() error { // For the sake of automating away as much of this as possible, see if the data directory is a // symlink, and if so resolve to its final real path, and then update the configuration to use // that. - if d, err := filepath.EvalSymlinks(sc.Data); err != nil { + data := viper.GetString("system.data") + if d, err := filepath.EvalSymlinks(data); err != nil { if !os.IsNotExist(err) { return err } - } else if d != sc.Data { - sc.Data = d + } else if d != data { + data = d + viper.Set("system.data", d) } - log.WithField("path", sc.Data).Debug("ensuring server data directory exists") - if err := os.MkdirAll(sc.Data, 0700); err != nil { + log.WithField("path", data).Debug("ensuring server data directory exists") + if err := os.MkdirAll(data, 0700); err != nil { return err } - log.WithField("path", sc.ArchiveDirectory).Debug("ensuring archive data directory exists") - if err := os.MkdirAll(sc.ArchiveDirectory, 0700); err != nil { + log.WithField("path", viper.GetString("system.archive_directory")).Debug("ensuring archive data directory exists") + if err := os.MkdirAll(viper.GetString("system.archive_directory"), 0700); err != nil { return err } - log.WithField("path", sc.BackupDirectory).Debug("ensuring backup data directory exists") - if err := os.MkdirAll(sc.BackupDirectory, 0700); err != nil { + log.WithField("path", viper.GetString("system.backup_directory")).Debug("ensuring backup data directory exists") + if err := os.MkdirAll(viper.GetString("system.backup_directory"), 0700); err != nil { return err } return nil } -// Writes a logrotate file for wings to the system logrotate configuration directory if one -// exists and a logrotate file is not found. This allows us to basically automate away the log -// rotation for most installs, but also enable users to make modifications on their own. -func (sc *SystemConfiguration) EnableLogRotation() error { +// EnableLogRotation writes a logrotate file for wings to the system logrotate +// configuration directory if one exists and a logrotate file is not found. This +// allows us to basically automate away the log rotation for most installs, but +// also enable users to make modifications on their own. +func EnableLogRotation() error { // Do nothing if not enabled. - if sc.EnableLogRotate == false { + if !viper.GetBool("system.enable_log_rotate") { log.Info("skipping log rotate configuration, disabled in wings config file") - return nil } @@ -174,14 +179,11 @@ func (sc *SystemConfiguration) EnableLogRotation() error { } else if (err != nil && os.IsNotExist(err)) || !st.IsDir() { return nil } - - if _, err := os.Stat("/etc/logrotate.d/wings"); err != nil && !os.IsNotExist(err) { + if _, err := os.Stat("/etc/logrotate.d/wings"); err == nil || !os.IsNotExist(err) { return err - } else if err == nil { - return nil } - log.Info("no log rotation configuration found, system is configured to support it, adding file now") + log.Info("no log rotation configuration found: adding file now") // If we've gotten to this point it means the logrotate directory exists on the system // but there is not a file for wings already. In that case, let us write a new file to // it so files can be rotated easily. @@ -191,8 +193,14 @@ func (sc *SystemConfiguration) EnableLogRotation() error { } defer f.Close() + type logrotateConfig struct { + Directory string + UserID int + GroupID int + } + t, err := template.New("logrotate").Parse(` -{{.LogDirectory}}/wings.log { +{{.Directory}}/wings.log { size 10M compress delaycompress @@ -200,17 +208,21 @@ func (sc *SystemConfiguration) EnableLogRotation() error { maxage 7 missingok notifempty - create 0640 {{.User.Uid}} {{.User.Gid}} + create 0640 {{.UserID}} {{.GroupID}} postrotate killall -SIGHUP wings endscript }`) - if err != nil { return err } - return errors.WithMessage(t.Execute(f, sc), "failed to write logrotate file to disk") + err = t.Execute(f, logrotateConfig{ + Directory: viper.GetString("system.log_directory"), + UserID: viper.GetInt("system.user.uid"), + GroupID: viper.GetInt("system.user.gid"), + }) + return errors.Wrap(err, "config: failed to write logrotate to disk") } // Returns the location of the JSON file that tracks server states. @@ -223,25 +235,28 @@ func (sc *SystemConfiguration) GetInstallLogPath() string { return path.Join(sc.LogDirectory, "install/") } -// Configures the timezone data for the configuration if it is currently missing. If -// a value has been set, this functionality will only run to validate that the timezone -// being used is valid. -func (sc *SystemConfiguration) ConfigureTimezone() error { - if sc.Timezone == "" { - if b, err := ioutil.ReadFile("/etc/timezone"); err != nil { +// ConfigureTimezone sets the timezone data for the configuration if it is +// currently missing. If a value has been set, this functionality will only run +// to validate that the timezone being used is valid. +func ConfigureTimezone() error { + tz := viper.GetString("system.timezone") + defer viper.Set("system.timezone", tz) + if tz == "" { + b, err := ioutil.ReadFile("/etc/timezone") + if err != nil { if !os.IsNotExist(err) { - return errors.WithMessage(err, "failed to open /etc/timezone for automatic server timezone calibration") + return errors.WithMessage(err, "config: failed to open timezone file") } - ctx, _ := context.WithTimeout(context.Background(), time.Second*5) + tz = "UTC" + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() // Okay, file isn't found on this OS, we will try using timedatectl to handle this. If this // command fails, exit, but if it returns a value use that. If no value is returned we will // fall through to UTC to get Wings booted at least. out, err := exec.CommandContext(ctx, "timedatectl").Output() if err != nil { log.WithField("error", err).Warn("failed to execute \"timedatectl\" to determine system timezone, falling back to UTC") - - sc.Timezone = "UTC" return nil } @@ -249,20 +264,16 @@ func (sc *SystemConfiguration) ConfigureTimezone() error { matches := r.FindSubmatch(out) if len(matches) != 2 || string(matches[1]) == "" { log.Warn("failed to parse timezone from \"timedatectl\" output, falling back to UTC") - - sc.Timezone = "UTC" return nil } - - sc.Timezone = string(matches[1]) + tz = string(matches[1]) } else { - sc.Timezone = string(b) + tz = string(b) } } - sc.Timezone = regexp.MustCompile(`(?i)[^a-z_/]+`).ReplaceAllString(sc.Timezone, "") + tz = regexp.MustCompile(`(?i)[^a-z_/]+`).ReplaceAllString(tz, "") + _, err := time.LoadLocation(tz) - _, err := time.LoadLocation(sc.Timezone) - - return errors.WithMessage(err, fmt.Sprintf("the supplied timezone %s is invalid", sc.Timezone)) + return errors.WithMessage(err, fmt.Sprintf("the supplied timezone %s is invalid", tz)) } diff --git a/environment/docker.go b/environment/docker.go index fb59b92..8b8b9c3 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -6,112 +6,95 @@ import ( "sync" "github.com/apex/log" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/pterodactyl/wings/config" + "github.com/spf13/viper" ) -var _cmu sync.Mutex +var _conce sync.Once var _client *client.Client -// Return a Docker client to be used throughout the codebase. Once a client has been created it -// will be returned for all subsequent calls to this function. +// DockerClient returns a Docker client to be used throughout the codebase. Once +// a client has been created it will be returned for all subsequent calls to this +// function. func DockerClient() (*client.Client, error) { - _cmu.Lock() - defer _cmu.Unlock() - - if _client != nil { - return _client, nil - } - - _client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - + var err error + _conce.Do(func() { + _client, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + }) return _client, err } -// Configures the required network for the docker environment. -func ConfigureDocker(c *config.DockerConfiguration) error { +// ConfigureDocker configures the required network for the docker environment. +func ConfigureDocker(ctx context.Context) error { // Ensure the required docker network exists on the system. cli, err := DockerClient() if err != nil { return err } - resource, err := cli.NetworkInspect(context.Background(), c.Network.Name, types.NetworkInspectOptions{}) - if err != nil && client.IsErrNotFound(err) { - log.Info("creating missing pterodactyl0 interface, this could take a few seconds...") - return createDockerNetwork(cli, c) - } else if err != nil { - log.WithField("error", err).Fatal("failed to create required docker network for containers") + nw := viper.Sub("docker.network") + resource, err := cli.NetworkInspect(ctx, nw.GetString("name"), types.NetworkInspectOptions{}) + if err != nil { + if client.IsErrNotFound(err) { + log.Info("creating missing pterodactyl0 interface, this could take a few seconds...") + if err := createDockerNetwork(ctx, cli); err != nil { + return err + } + } + return err + } else { + nw.Set("driver", resource.Driver) } - switch resource.Driver { + switch nw.GetString("driver") { case "host": - c.Network.Interface = "127.0.0.1" - c.Network.ISPN = false - return nil + nw.Set("interface", "127.0.0.1") + nw.Set("ispn", false) case "overlay": + fallthrough case "weavemesh": - c.Network.Interface = "" - c.Network.ISPN = true - return nil + nw.Set("interface", "") + nw.Set("ispn", true) default: - c.Network.ISPN = false + nw.Set("ispn", false) } - return nil } // Creates a new network on the machine if one does not exist already. -func createDockerNetwork(cli *client.Client, c *config.DockerConfiguration) error { - _, err := cli.NetworkCreate(context.Background(), c.Network.Name, types.NetworkCreate{ - Driver: c.Network.Driver, +func createDockerNetwork(ctx context.Context, cli *client.Client) error { + nw := viper.Sub("docker.network") + _, err := cli.NetworkCreate(ctx, nw.GetString("name"), types.NetworkCreate{ + Driver: nw.GetString("driver"), EnableIPv6: true, - Internal: c.Network.IsInternal, + Internal: nw.GetBool("is_internal"), IPAM: &network.IPAM{ Config: []network.IPAMConfig{ { - Subnet: c.Network.Interfaces.V4.Subnet, - Gateway: c.Network.Interfaces.V4.Gateway, + Subnet: nw.GetString("interfaces.v4.subnet"), + Gateway: nw.GetString("interfaces.v4.gateway"), }, { - Subnet: c.Network.Interfaces.V6.Subnet, - Gateway: c.Network.Interfaces.V6.Gateway, + Subnet: nw.GetString("interfaces.v6.subnet"), + Gateway: nw.GetString("interfaces.v6.gateway"), }, }, }, Options: map[string]string{ "encryption": "false", "com.docker.network.bridge.default_bridge": "false", - "com.docker.network.bridge.enable_icc": strconv.FormatBool(c.Network.EnableICC), + "com.docker.network.bridge.enable_icc": strconv.FormatBool(nw.GetBool("enable_icc")), "com.docker.network.bridge.enable_ip_masquerade": "true", "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", "com.docker.network.bridge.name": "pterodactyl0", "com.docker.network.driver.mtu": "1500", }, }) - - if err != nil { - return err + driver := nw.GetString("driver") + if driver != "host" && driver != "overlay" && driver != "weavemesh" { + nw.Set("interface", nw.GetString("interfaces.v4.gateway")) } - - switch c.Network.Driver { - case "host": - c.Network.Interface = "127.0.0.1" - c.Network.ISPN = false - break - case "overlay": - case "weavemesh": - c.Network.Interface = "" - c.Network.ISPN = true - break - default: - c.Network.Interface = c.Network.Interfaces.V4.Gateway - c.Network.ISPN = false - break - } - - return nil + return err } diff --git a/go.mod b/go.mod index 079a014..b25e755 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.7.1 github.com/ugorji/go v1.2.2 // indirect github.com/ulikunitz/xz v0.5.9 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad diff --git a/go.sum b/go.sum index 9e9b174..ca88e19 100644 --- a/go.sum +++ b/go.sum @@ -285,6 +285,7 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -401,6 +402,7 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -454,6 +456,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -550,11 +553,14 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= @@ -562,6 +568,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -575,6 +583,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= diff --git a/system/utils.go b/system/utils.go index e0e8d72..681ac2a 100644 --- a/system/utils.go +++ b/system/utils.go @@ -7,14 +7,35 @@ import ( "encoding/json" "fmt" "io" + "strconv" "strings" "sync" "time" + + "emperror.dev/errors" ) var cr = []byte(" \r") var crr = []byte("\r\n") +// FirstNotEmpty returns the first string passed in that is not an empty value. +func FirstNotEmpty(v ...string) string { + for _, val := range v { + if val != "" { + return val + } + } + return "" +} + +func MustInt(v string) int { + i, err := strconv.Atoi(v) + if err != nil { + panic(errors.Wrap(err, "system/utils: could not parse int")) + } + return i +} + func ScanReader(r io.Reader, callback func(line string)) error { br := bufio.NewReader(r) // Avoid constantly re-allocating memory when we're flooding lines through this From 80faea328670f7baa9f192df279ff0f462e94294 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 14 Jan 2021 20:11:01 -0800 Subject: [PATCH 15/30] yoink viper back out of code, simplify some config logic --- cmd/configure.go | 4 +- cmd/diagnostics.go | 2 +- cmd/root.go | 165 ++++------ config/config.go | 626 +++++++++++++++++++++++++++---------- config/config_docker.go | 18 +- config/config_system.go | 279 ----------------- config/config_throttles.go | 27 -- environment/docker.go | 1 - go.mod | 2 - go.sum | 3 +- server/install.go | 2 +- 11 files changed, 527 insertions(+), 602 deletions(-) delete mode 100644 config/config_system.go delete mode 100644 config/config_throttles.go diff --git a/cmd/configure.go b/cmd/configure.go index 93a9341..eab150f 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -147,7 +147,7 @@ func configureCmdRun(cmd *cobra.Command, args []string) { b, err := ioutil.ReadAll(res.Body) - cfg, err := config.NewFromPath(configPath) + cfg, err := config.NewAtPath(configPath) if err != nil { panic(err) } @@ -156,7 +156,7 @@ func configureCmdRun(cmd *cobra.Command, args []string) { panic(err) } - if err = cfg.WriteToDisk(); err != nil { + if err = config.WriteToDisk(cfg); err != nil { panic(err) } diff --git a/cmd/diagnostics.go b/cmd/diagnostics.go index 5514723..d67185f 100644 --- a/cmd/diagnostics.go +++ b/cmd/diagnostics.go @@ -102,7 +102,7 @@ func diagnosticsCmdRun(cmd *cobra.Command, args []string) { } printHeader(output, "Wings Configuration") - cfg, err := config.ReadConfiguration(config.DefaultLocation) + cfg, err := config.FromFile(config.DefaultLocation) if cfg != nil { fmt.Fprintln(output, " Panel Location:", redact(cfg.PanelLocation)) fmt.Fprintln(output, "") diff --git a/cmd/root.go b/cmd/root.go index c1080b3..2d4e775 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" "emperror.dev/errors" @@ -26,13 +27,12 @@ import ( "github.com/pterodactyl/wings/sftp" "github.com/pterodactyl/wings/system" "github.com/spf13/cobra" - "github.com/spf13/viper" "golang.org/x/crypto/acme" "golang.org/x/crypto/acme/autocert" ) var ( - configPath = "" + configPath = config.DefaultLocation debug = false ) @@ -40,6 +40,8 @@ var rootCommand = &cobra.Command{ Use: "wings", Short: "Runs the API server allowing programatic control of game servers for Pterodactyl Panel.", PreRun: func(cmd *cobra.Command, args []string) { + initConfig() + initLogging() if tls, _ := cmd.Flags().GetBool("auto-tls"); tls { if host, _ := cmd.Flags().GetString("tls-hostname"); host == "" { fmt.Println("A TLS hostname must be provided when running wings with automatic TLS, e.g.:\n\n ./wings --auto-tls --tls-hostname my.example.com") @@ -65,9 +67,7 @@ func Execute() { } func init() { - cobra.OnInitialize(initConfig, initLogging) - - rootCommand.PersistentFlags().StringVar(&configPath, "config", "", "set the location for the configuration file") + rootCommand.PersistentFlags().StringVar(&configPath, "config", config.DefaultLocation, "set the location for the configuration file") rootCommand.PersistentFlags().BoolVar(&debug, "debug", false, "pass in order to run wings in debug mode") // Flags specifically used when running the API. @@ -81,27 +81,6 @@ func init() { rootCommand.AddCommand(diagnosticsCmd) } -// Get the configuration path based on the arguments provided. -func readConfiguration() (*config.Configuration, error) { - p := configPath - if !strings.HasPrefix(p, "/") { - d, err := os.Getwd() - if err != nil { - return nil, err - } - - p = path.Clean(path.Join(d, configPath)) - } - - if s, err := os.Stat(p); err != nil { - return nil, err - } else if s.IsDir() { - return nil, errors.New("cannot use directory as configuration file path") - } - - return config.ReadConfiguration(p) -} - func rootCmdRun(cmd *cobra.Command, _ []string) { switch cmd.Flag("profiler").Value.String() { case "cpu": @@ -122,18 +101,9 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { defer profile.Start(profile.BlockProfile).Stop() } - c, err := readConfiguration() - if err != nil { - panic(err) - } - - if debug { - c.Debug = true - } - printLogo() - log.WithField("path", viper.ConfigFileUsed()).Info("loading configuration from file") log.Debug("running in debug mode") + log.WithField("config_file", configPath).Info("loading configuration from file") if ok, _ := cmd.Flags().GetBool("ignore-certificate-errors"); ok { log.Warn("running with --ignore-certificate-errors: TLS certificate host chains and name will not be verified") @@ -142,45 +112,39 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { } } - config.Set(c) - config.SetDebugViaFlag(debug) if err := config.ConfigureTimezone(); err != nil { log.WithField("error", err).Fatal("failed to detect system timezone or use supplied configuration value") } - log.WithField("timezone", c.System.Timezone).Info("configured wings with system timezone") - + log.WithField("timezone", config.Get().System.Timezone).Info("configured wings with system timezone") if err := config.ConfigureDirectories(); err != nil { log.WithField("error", err).Fatal("failed to configure system directories for pterodactyl") return } - if err := config.EnableLogRotation(); err != nil { log.WithField("error", err).Fatal("failed to configure log rotation on the system") return } - log.WithField("username", c.System.Username).Info("checking for pterodactyl system user") + log.WithField("username", config.Get().System.User).Info("checking for pterodactyl system user") if err := config.EnsurePterodactylUser(); err != nil { log.WithField("error", err).Fatal("failed to create pterodactyl system user") } log.WithFields(log.Fields{ - "username": viper.GetString("system.username"), - "uid": viper.GetInt("system.user.uid"), - "gid": viper.GetInt("system.user.gid"), + "username": config.Get().System.Username, + "uid": config.Get().System.User.Uid, + "gid": config.Get().System.User.Gid, }).Info("configured system user successfully") if err := server.LoadDirectory(); err != nil { log.WithField("error", err).Fatal("failed to load server configurations") - return } if err := environment.ConfigureDocker(cmd.Context()); err != nil { log.WithField("error", err).Fatal("failed to configure docker environment") - return } - if err := viper.WriteConfig(); err != nil { - log.WithField("error", err).Error("failed to save configuration to disk") + if err := config.WriteToDisk(config.Get()); err != nil { + log.WithField("error", err).Fatal("failed to write configuration to disk") } // Just for some nice log output. @@ -197,7 +161,6 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // on Wings. This allows us to ensure the environment exists, write configurations, // and reboot processes without causing a slow-down due to sequential booting. pool := workerpool.New(4) - for _, serv := range server.GetServers().All() { s := serv @@ -252,6 +215,13 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // Wait until all of the servers are ready to go before we fire up the SFTP and HTTP servers. pool.StopWait() + defer func() { + // Cancel the context on all of the running servers at this point, even though the + // program is just shutting down. + for _, s := range server.GetServers().All() { + s.CtxCancel() + } + }() go func() { // Run the SFTP server. @@ -261,13 +231,14 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { } }() + sys := config.Get().System // Ensure the archive directory exists. - if err := os.MkdirAll(c.System.ArchiveDirectory, 0755); err != nil { + if err := os.MkdirAll(sys.ArchiveDirectory, 0755); err != nil { log.WithField("error", err).Error("failed to create archive directory") } // Ensure the backup directory exists. - if err := os.MkdirAll(c.System.BackupDirectory, 0755); err != nil { + if err := os.MkdirAll(sys.BackupDirectory, 0755); err != nil { log.WithField("error", err).Error("failed to create backup directory") } @@ -277,47 +248,31 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { autotls = false } + api := config.Get().Api log.WithFields(log.Fields{ - "use_ssl": c.Api.Ssl.Enabled, + "use_ssl": api.Ssl.Enabled, "use_auto_tls": autotls, - "host_address": c.Api.Host, - "host_port": c.Api.Port, + "host_address": api.Host, + "host_port": api.Port, }).Info("configuring internal webserver") - // Configure the router. - r := router.Configure() - + // Create a new HTTP server instance to handle inbound requests from the Panel + // and external clients. s := &http.Server{ - Addr: fmt.Sprintf("%s:%d", c.Api.Host, c.Api.Port), - Handler: r, - TLSConfig: &tls.Config{ - NextProtos: []string{"h2", "http/1.1"}, - // @see https://blog.cloudflare.com/exposing-go-on-the-internet - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, - }, - PreferServerCipherSuites: true, - MinVersion: tls.VersionTLS12, - MaxVersion: tls.VersionTLS13, - CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256}, - }, + Addr: api.Host + ":" + strconv.Itoa(api.Port), + Handler: router.Configure(), + TLSConfig: config.DefaultTLSConfig, } // Check if the server should run with TLS but using autocert. if autotls { m := autocert.Manager{ Prompt: autocert.AcceptTOS, - Cache: autocert.DirCache(path.Join(c.System.RootDirectory, "/.tls-cache")), + Cache: autocert.DirCache(path.Join(sys.RootDirectory, "/.tls-cache")), HostPolicy: autocert.HostWhitelist(tlshostname), } - log.WithField("hostname", tlshostname). - Info("webserver is now listening with auto-TLS enabled; certificates will be automatically generated by Let's Encrypt") + log.WithField("hostname", tlshostname).Info("webserver is now listening with auto-TLS enabled; certificates will be automatically generated by Let's Encrypt") // Hook autocert into the main http server. s.TLSConfig.GetCertificate = m.GetCertificate @@ -329,59 +284,53 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { log.WithError(err).Error("failed to serve autocert http server") } }() - // Start the main http server with TLS using autocert. if err := s.ListenAndServeTLS("", ""); err != nil { - log.WithFields(log.Fields{"auto_tls": true, "tls_hostname": tlshostname, "error": err}). - Fatal("failed to configure HTTP server using auto-tls") + log.WithFields(log.Fields{"auto_tls": true, "tls_hostname": tlshostname, "error": err}).Fatal("failed to configure HTTP server using auto-tls") } - return } - // Check if main http server should run with TLS. - if c.Api.Ssl.Enabled { - if err := s.ListenAndServeTLS(strings.ToLower(c.Api.Ssl.CertificateFile), strings.ToLower(c.Api.Ssl.KeyFile)); err != nil { + // Check if main http server should run with TLS. Otherwise reset the TLS + // config on the server and then serve it over normal HTTP. + if api.Ssl.Enabled { + if err := s.ListenAndServeTLS(strings.ToLower(api.Ssl.CertificateFile), strings.ToLower(api.Ssl.KeyFile)); err != nil { log.WithFields(log.Fields{"auto_tls": false, "error": err}).Fatal("failed to configure HTTPS server") } return } - - // Run the main http server without TLS. s.TLSConfig = nil if err := s.ListenAndServe(); err != nil { log.WithField("error", err).Fatal("failed to configure HTTP server") } - - // Cancel the context on all of the running servers at this point, even though the - // program is just shutting down. - for _, s := range server.GetServers().All() { - s.CtxCancel() - } } +// Reads the configuration from the disk and then sets up the global singleton +// with all of the configuration values. func initConfig() { - if configPath != "" { - viper.SetConfigName("config") - viper.SetConfigType("yaml") - viper.AddConfigPath("/etc/pterodactyl") - viper.AddConfigPath("$HOME/.pterodactyl") - viper.AddConfigPath(".") - } else { - viper.SetConfigFile(configPath) + if !strings.HasPrefix(configPath, "/") { + d, err := os.Getwd() + if err != nil { + log2.Fatalf("cmd/root: could not determine directory: %s", err) + } + configPath = path.Clean(path.Join(d, configPath)) } - if err := viper.ReadInConfig(); err != nil { - if _, ok := err.(*viper.ConfigFileNotFoundError); ok { + err := config.FromFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { exitWithConfigurationNotice() } - log2.Fatalf("cmd/root: failed to read configuration: %s", err) + log2.Fatalf("cmd/root: error while reading configuration file: %s", err) + } + if debug && !config.Get().Debug { + config.SetDebugViaFlag(debug) } } // Configures the global logger for Zap so that we can call it from any location // in the code without having to pass around a logger instance. func initLogging() { - dir := viper.GetString("system.log_directory") + dir := config.Get().System.LogDirectory if err := os.MkdirAll(path.Join(dir, "/install"), 0700); err != nil { log2.Fatalf("cmd/root: failed to create install directory path: %s", err) } @@ -390,12 +339,10 @@ func initLogging() { if err != nil { log2.Fatalf("cmd/root: failed to create wings log: %s", err) } - log.SetLevel(log.InfoLevel) - if viper.GetBool("debug") { + if config.Get().Debug { log.SetLevel(log.DebugLevel) } - log.SetHandler(multi.New(cli.Default, cli.New(w.File, false))) log.WithField("path", p).Info("writing log files to disk") } diff --git a/config/config.go b/config/config.go index 8d2d3d9..7138a38 100644 --- a/config/config.go +++ b/config/config.go @@ -1,35 +1,248 @@ package config import ( + "context" + "crypto/tls" "fmt" "io/ioutil" "os" "os/exec" "os/user" + "path" + "path/filepath" + "regexp" "strings" "sync" + "text/template" + "time" "emperror.dev/errors" + "github.com/apex/log" "github.com/cobaugh/osrelease" "github.com/creasty/defaults" "github.com/gbrlsnchs/jwt/v3" "github.com/pterodactyl/wings/system" - "github.com/spf13/viper" "gopkg.in/yaml.v2" ) const DefaultLocation = "/etc/pterodactyl/config.yml" -type Configuration struct { - sync.RWMutex `json:"-" yaml:"-"` +// DefaultTLSConfig sets sane defaults to use when configuring the internal +// webserver to listen for public connections. +// +// @see https://blog.cloudflare.com/exposing-go-on-the-internet +var DefaultTLSConfig = &tls.Config{ + NextProtos: []string{"h2", "http/1.1"}, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + }, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + CurvePreferences: []tls.CurveID{tls.X25519, tls.CurveP256}, +} +var mu sync.RWMutex +var _config *Configuration +var _jwtAlgo *jwt.HMACSHA +var _debugViaFlag bool + +// Locker specific to writing the configuration to the disk, this happens +// in areas that might already be locked so we don't want to crash the process. +var _writeLock sync.Mutex + +// SftpConfiguration defines the configuration of the internal SFTP server. +type SftpConfiguration struct { + // The bind address of the SFTP server. + Address string `default:"0.0.0.0" json:"bind_address" yaml:"bind_address"` + // The bind port of the SFTP server. + Port int `default:"2022" json:"bind_port" yaml:"bind_port"` + // If set to true, no write actions will be allowed on the SFTP server. + ReadOnly bool `default:"false" yaml:"read_only"` +} + +// ApiConfiguration defines the configuration for the internal API that is +// exposed by the Wings webserver. +type ApiConfiguration struct { + // The interface that the internal webserver should bind to. + Host string `default:"0.0.0.0" yaml:"host"` + + // The port that the internal webserver should bind to. + Port int `default:"8080" yaml:"port"` + + // SSL configuration for the daemon. + Ssl struct { + Enabled bool `json:"enabled" yaml:"enabled"` + CertificateFile string `json:"cert" yaml:"cert"` + KeyFile string `json:"key" yaml:"key"` + } + + // Determines if functionality for allowing remote download of files into server directories + // is enabled on this instance. If set to "true" remote downloads will not be possible for + // servers. + DisableRemoteDownload bool `json:"disable_remote_download" yaml:"disable_remote_download"` + + // The maximum size for files uploaded through the Panel in bytes. + UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"` +} + +// RemoteQueryConfiguration defines the configuration settings for remote requests +// from Wings to the Panel. +type RemoteQueryConfiguration struct { + // The amount of time in seconds that Wings should allow for a request to the Panel API + // to complete. If this time passes the request will be marked as failed. If your requests + // are taking longer than 30 seconds to complete it is likely a performance issue that + // should be resolved on the Panel, and not something that should be resolved by upping this + // number. + Timeout uint `default:"30" yaml:"timeout"` + + // The number of servers to load in a single request to the Panel API when booting the + // Wings instance. A single request is initially made to the Panel to get this number + // of servers, and then the pagination status is checked and additional requests are + // fired off in parallel to request the remaining pages. + // + // It is not recommended to change this from the default as you will likely encounter + // memory limits on your Panel instance. In the grand scheme of things 4 requests for + // 50 servers is likely just as quick as two for 100 or one for 400, and will certainly + // be less likely to cause performance issues on the Panel. + BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"` +} + +// SystemConfiguration defines basic system configuration settings. +type SystemConfiguration struct { + // The root directory where all of the pterodactyl data is stored at. + RootDirectory string `default:"/var/lib/pterodactyl" yaml:"root_directory"` + + // Directory where logs for server installations and other wings events are logged. + LogDirectory string `default:"/var/log/pterodactyl" yaml:"log_directory"` + + // Directory where the server data is stored at. + Data string `default:"/var/lib/pterodactyl/volumes" yaml:"data"` + + // Directory where server archives for transferring will be stored. + ArchiveDirectory string `default:"/var/lib/pterodactyl/archives" yaml:"archive_directory"` + + // Directory where local backups will be stored on the machine. + BackupDirectory string `default:"/var/lib/pterodactyl/backups" yaml:"backup_directory"` + + // The user that should own all of the server files, and be used for containers. + Username string `default:"pterodactyl" yaml:"username"` + + // The timezone for this Wings instance. This is detected by Wings automatically if possible, + // and falls back to UTC if not able to be detected. If you need to set this manually, that + // can also be done. + // + // This timezone value is passed into all containers created by Wings. + Timezone string `yaml:"timezone"` + + // Definitions for the user that gets created to ensure that we can quickly access + // this information without constantly having to do a system lookup. + User struct { + Uid int + Gid int + } + + // The amount of time in seconds that can elapse before a server's disk space calculation is + // considered stale and a re-check should occur. DANGER: setting this value too low can seriously + // impact system performance and cause massive I/O bottlenecks and high CPU usage for the Wings + // process. + // + // Set to 0 to disable disk checking entirely. This will always return 0 for the disk space used + // by a server and should only be set in extreme scenarios where performance is critical and + // disk usage is not a concern. + DiskCheckInterval int64 `default:"150" yaml:"disk_check_interval"` + + // If set to true, file permissions for a server will be checked when the process is + // booted. This can cause boot delays if the server has a large amount of files. In most + // cases disabling this should not have any major impact unless external processes are + // frequently modifying a servers' files. + CheckPermissionsOnBoot bool `default:"true" yaml:"check_permissions_on_boot"` + + // If set to false Wings will not attempt to write a log rotate configuration to the disk + // when it boots and one is not detected. + EnableLogRotate bool `default:"true" yaml:"enable_log_rotate"` + + // The number of lines to send when a server connects to the websocket. + WebsocketLogCount int `default:"150" yaml:"websocket_log_count"` + + Sftp SftpConfiguration `yaml:"sftp"` + + CrashDetection CrashDetection `yaml:"crash_detection"` + + Backups Backups `yaml:"backups"` + + Transfers Transfers `yaml:"transfers"` +} + +type CrashDetection struct { + // Determines if Wings should detect a server that stops with a normal exit code of + // "0" as being crashed if the process stopped without any Wings interaction. E.g. + // the user did not press the stop button, but the process stopped cleanly. + DetectCleanExitAsCrash bool `default:"true" yaml:"detect_clean_exit_as_crash"` + + // Timeout specifies the timeout between crashes that will not cause the server + // to be automatically restarted, this value is used to prevent servers from + // becoming stuck in a boot-loop after multiple consecutive crashes. + Timeout int `default:"60" json:"timeout"` +} + +type Backups struct { + // WriteLimit imposes a Disk I/O write limit on backups to the disk, this affects all + // backup drivers as the archiver must first write the file to the disk in order to + // upload it to any external storage provider. + // + // If the value is less than 1, the write speed is unlimited, + // if the value is greater than 0, the write speed is the value in MiB/s. + // + // Defaults to 0 (unlimited) + WriteLimit int `default:"0" yaml:"write_limit"` +} + +type Transfers struct { + // DownloadLimit imposes a Network I/O read limit when downloading a transfer archive. + // + // If the value is less than 1, the write speed is unlimited, + // if the value is greater than 0, the write speed is the value in MiB/s. + // + // Defaults to 0 (unlimited) + DownloadLimit int `default:"0" yaml:"download_limit"` +} + +type ConsoleThrottles struct { + // Whether or not the throttler is enabled for this instance. + Enabled bool `json:"enabled" yaml:"enabled" default:"true"` + + // The total number of lines that can be output in a given LineResetInterval period before + // a warning is triggered and counted against the server. + Lines uint64 `json:"lines" yaml:"lines" default:"2000"` + + // The total number of throttle activations that can accumulate before a server is considered + // to be breaching and will be stopped. This value is decremented by one every DecayInterval. + MaximumTriggerCount uint64 `json:"maximum_trigger_count" yaml:"maximum_trigger_count" default:"5"` + + // The amount of time after which the number of lines processed is reset to 0. This runs in + // a constant loop and is not affected by the current console output volumes. By default, this + // will reset the processed line count back to 0 every 100ms. + LineResetInterval uint64 `json:"line_reset_interval" yaml:"line_reset_interval" default:"100"` + + // The amount of time in milliseconds that must pass without an output warning being triggered + // before a throttle activation is decremented. + DecayInterval uint64 `json:"decay_interval" yaml:"decay_interval" default:"10000"` + + // The amount of time that a server is allowed to be stopping for before it is terminated + // forcefully if it triggers output throttles. + StopGracePeriod uint `json:"stop_grace_period" yaml:"stop_grace_period" default:"15"` +} + +type Configuration struct { // The location from which this configuration instance was instantiated. path string - // Locker specific to writing the configuration to the disk, this happens - // in areas that might already be locked so we don't want to crash the process. - writeLock sync.Mutex - // Determines if wings should be running in debug mode. This value is ignored // if the debug flag is passed through the command line arguments. Debug bool @@ -68,168 +281,110 @@ type Configuration struct { AllowedOrigins []string `json:"allowed_origins" yaml:"allowed_origins"` } -// Defines the configuration of the internal SFTP server. -type SftpConfiguration struct { - // The bind address of the SFTP server. - Address string `default:"0.0.0.0" json:"bind_address" yaml:"bind_address"` - // The bind port of the SFTP server. - Port int `default:"2022" json:"bind_port" yaml:"bind_port"` - // If set to true, no write actions will be allowed on the SFTP server. - ReadOnly bool `default:"false" yaml:"read_only"` -} - -// Defines the configuration for the internal API that is exposed by the -// daemon webserver. -type ApiConfiguration struct { - // The interface that the internal webserver should bind to. - Host string `default:"0.0.0.0" yaml:"host"` - - // The port that the internal webserver should bind to. - Port int `default:"8080" yaml:"port"` - - // SSL configuration for the daemon. - Ssl struct { - Enabled bool `json:"enabled" yaml:"enabled"` - CertificateFile string `json:"cert" yaml:"cert"` - KeyFile string `json:"key" yaml:"key"` - } - - // Determines if functionality for allowing remote download of files into server directories - // is enabled on this instance. If set to "true" remote downloads will not be possible for - // servers. - DisableRemoteDownload bool `json:"disable_remote_download" yaml:"disable_remote_download"` - - // The maximum size for files uploaded through the Panel in bytes. - UploadLimit int `default:"100" json:"upload_limit" yaml:"upload_limit"` -} - -// Defines the configuration settings for remote requests from Wings to the Panel. -type RemoteQueryConfiguration struct { - // The amount of time in seconds that Wings should allow for a request to the Panel API - // to complete. If this time passes the request will be marked as failed. If your requests - // are taking longer than 30 seconds to complete it is likely a performance issue that - // should be resolved on the Panel, and not something that should be resolved by upping this - // number. - Timeout uint `default:"30" yaml:"timeout"` - - // The number of servers to load in a single request to the Panel API when booting the - // Wings instance. A single request is initially made to the Panel to get this number - // of servers, and then the pagination status is checked and additional requests are - // fired off in parallel to request the remaining pages. - // - // It is not recommended to change this from the default as you will likely encounter - // memory limits on your Panel instance. In the grand scheme of things 4 requests for - // 50 servers is likely just as quick as two for 100 or one for 400, and will certainly - // be less likely to cause performance issues on the Panel. - BootServersPerPage uint `default:"50" yaml:"boot_servers_per_page"` -} - -// Reads the configuration from the provided file and returns the configuration -// object that can then be used. -func ReadConfiguration(path string) (*Configuration, error) { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - c := new(Configuration) +// NewAtPath creates a new struct and set the path where it should be stored. +// This function does not modify the currently stored global configuration. +func NewAtPath(path string) (*Configuration, error) { + var c Configuration // Configures the default values for many of the configuration options present // in the structs. Values set in the configuration file take priority over the // default values. - if err := defaults.Set(c); err != nil { + if err := defaults.Set(&c); err != nil { return nil, err } - // Track the location where we created this configuration. - c.unsafeSetPath(path) - - // Replace environment variables within the configuration file with their - // values from the host system. - b = []byte(os.ExpandEnv(string(b))) - - if err := yaml.Unmarshal(b, c); err != nil { - return nil, err - } - - return c, nil + c.path = path + return &c, nil } -var mu sync.RWMutex - -var _config *Configuration -var _jwtAlgo *jwt.HMACSHA -var _debugViaFlag bool - // Set the global configuration instance. This is a blocking operation such that // anything trying to set a different configuration value, or read the configuration // will be paused until it is complete. func Set(c *Configuration) { mu.Lock() - if _config == nil || _config.AuthenticationToken != c.AuthenticationToken { _jwtAlgo = jwt.NewHS256([]byte(c.AuthenticationToken)) } - _config = c mu.Unlock() } +// SetDebugViaFlag tracks if the application is running in debug mode because of +// a command line flag argument. If so we do not want to store that configuration +// change to the disk. func SetDebugViaFlag(d bool) { + mu.Lock() + _config.Debug = d _debugViaFlag = d + mu.Unlock() } -// Get the global configuration instance. This is a read-safe operation that will block -// if the configuration is presently being modified. +// Get returns the global configuration instance. This is a thread-safe operation +// that will block if the configuration is presently being modified. +// +// Be aware that you CANNOT make modifications to the currently stored configuration +// by modifying the struct returned by this function. The only way to make +// modifications is by using the Update() function and passing data through in +// the callback. func Get() *Configuration { mu.RLock() - defer mu.RUnlock() - - return _config + // Create a copy of the struct so that all modifications made beyond this + // point are immutable. + //goland:noinspection GoVetCopyLock + c := *_config + mu.RUnlock() + return &c } -// Returns the in-memory JWT algorithm. +// Update performs an in-situ update of the global configuration object using +// a thread-safe mutex lock. This is the correct way to make modifications to +// the global configuration. +func Update(callback func(c *Configuration)) { + mu.Lock() + defer mu.Unlock() + callback(_config) +} + +// GetJwtAlgorithm returns the in-memory JWT algorithm. func GetJwtAlgorithm() *jwt.HMACSHA { mu.RLock() defer mu.RUnlock() - return _jwtAlgo } -// Create a new struct and set the path where it should be stored. -func NewFromPath(path string) (*Configuration, error) { - c := new(Configuration) - if err := defaults.Set(c); err != nil { - return c, err +// WriteToDisk writes the configuration to the disk. This is a thread safe operation +// and will only allow one write at a time. Additional calls while writing are +// queued up. +func WriteToDisk(c *Configuration) error { + _writeLock.Lock() + defer _writeLock.Unlock() + + //goland:noinspection GoVetCopyLock + ccopy := *c + // If debugging is set with the flag, don't save that to the configuration file, + // otherwise you'll always end up in debug mode. + if _debugViaFlag { + ccopy.Debug = false } - - c.unsafeSetPath(path) - - return c, nil -} - -// Sets the path where the configuration file is located on the server. This function should -// not be called except by processes that are generating the configuration such as the configuration -// command shipped with this software. -func (c *Configuration) unsafeSetPath(path string) { - c.Lock() - c.path = path - c.Unlock() -} - -// Returns the path for this configuration file. -func (c *Configuration) GetPath() string { - c.RLock() - defer c.RUnlock() - - return c.path + if c.path == "" { + return errors.New("cannot write configuration, no path defined in struct") + } + b, err := yaml.Marshal(&ccopy) + if err != nil { + return err + } + if err := ioutil.WriteFile(c.path, b, 0600); err != nil { + return err + } + return nil } // EnsurePterodactylUser ensures that the Pterodactyl core user exists on the // system. This user will be the owner of all data in the root data directory -// and is used as the user within containers. +// and is used as the user within containers. If files are not owned by this +// user there will be issues with permissions on Docker mount points. // -// If files are not owned by this user there will be issues with permissions on -// Docker mount points. +// This function IS NOT thread safe and should only be called in the main thread +// when the application is booting. func EnsurePterodactylUser() error { sysName, err := getSystemName() if err != nil { @@ -238,14 +393,13 @@ func EnsurePterodactylUser() error { // Our way of detecting if wings is running inside of Docker. if sysName == "busybox" { - viper.Set("system.username", system.FirstNotEmpty(os.Getenv("WINGS_USERNAME"), "pterodactyl")) - viper.Set("system.user.uid", system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_UID"), "988"))) - viper.Set("system.user.gid", system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_GID"), "988"))) + _config.System.Username = system.FirstNotEmpty(os.Getenv("WINGS_USERNAME"), "pterodactyl") + _config.System.User.Uid = system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_UID"), "988")) + _config.System.User.Gid = system.MustInt(system.FirstNotEmpty(os.Getenv("WINGS_UID"), "988")) return nil } - username := viper.GetString("system.username") - u, err := user.Lookup(username) + u, err := user.Lookup(_config.System.Username) // If an error is returned but it isn't the unknown user error just abort // the process entirely. If we did find a user, return it immediately. if err != nil { @@ -253,19 +407,19 @@ func EnsurePterodactylUser() error { return err } } else { - viper.Set("system.user.uid", system.MustInt(u.Uid)) - viper.Set("system.user.gid", system.MustInt(u.Gid)) + _config.System.User.Uid = system.MustInt(u.Uid) + _config.System.User.Gid = system.MustInt(u.Gid) return nil } - command := fmt.Sprintf("useradd --system --no-create-home --shell /usr/sbin/nologin %s", username) + command := fmt.Sprintf("useradd --system --no-create-home --shell /usr/sbin/nologin %s", _config.System.Username) // Alpine Linux is the only OS we currently support that doesn't work with the useradd // command, so in those cases we just modify the command a bit to work as expected. if strings.HasPrefix(sysName, "alpine") { - command = fmt.Sprintf("adduser -S -D -H -G %[1]s -s /sbin/nologin %[1]s", username) + command = fmt.Sprintf("adduser -S -D -H -G %[1]s -s /sbin/nologin %[1]s", _config.System.Username) // We have to create the group first on Alpine, so do that here before continuing on // to the user creation process. - if _, err := exec.Command("addgroup", "-S", username).Output(); err != nil { + if _, err := exec.Command("addgroup", "-S", _config.System.Username).Output(); err != nil { return err } } @@ -274,53 +428,189 @@ func EnsurePterodactylUser() error { if _, err := exec.Command(split[0], split[1:]...).Output(); err != nil { return err } - - u, err = user.Lookup(username) + u, err = user.Lookup(_config.System.Username) if err != nil { return err } - viper.Set("system.user.uid", system.MustInt(u.Uid)) - viper.Set("system.user.gid", system.MustInt(u.Gid)) + _config.System.User.Uid = system.MustInt(u.Uid) + _config.System.User.Gid = system.MustInt(u.Gid) return nil } -// Writes the configuration to the disk as a blocking operation by obtaining an exclusive -// lock on the file. This prevents something else from writing at the exact same time and -// leading to bad data conditions. -func (c *Configuration) WriteToDisk() error { - // Obtain an exclusive write against the configuration file. - c.writeLock.Lock() - defer c.writeLock.Unlock() - - ccopy := *c - // If debugging is set with the flag, don't save that to the configuration file, otherwise - // you'll always end up in debug mode. - if _debugViaFlag { - ccopy.Debug = false - } - - if c.path == "" { - return errors.New("cannot write configuration, no path defined in struct") - } - - b, err := yaml.Marshal(&ccopy) +// FromFile reads the configuration from the provided file and stores it in the +// global singleton for this instance. +func FromFile(path string) error { + b, err := ioutil.ReadFile(path) if err != nil { return err } + c, err := NewAtPath(path) + if err != nil { + return err + } + // Replace environment variables within the configuration file with their + // values from the host system. + b = []byte(os.ExpandEnv(string(b))) + if err := yaml.Unmarshal(b, c); err != nil { + return err + } + // Store this configuration in the global state. + Set(c) + return nil +} - if err := ioutil.WriteFile(c.GetPath(), b, 0644); err != nil { +// ConfigureDirectories ensures that all of the system directories exist on the +// system. These directories are created so that only the owner can read the data, +// and no other users. +// +// This function IS NOT thread-safe. +func ConfigureDirectories() error { + root := _config.System.RootDirectory + log.WithField("path", root).Debug("ensuring root data directory exists") + if err := os.MkdirAll(root, 0700); err != nil { + return err + } + + // There are a non-trivial number of users out there whose data directories are actually a + // symlink to another location on the disk. If we do not resolve that final destination at this + // point things will appear to work, but endless errors will be encountered when we try to + // verify accessed paths since they will all end up resolving outside the expected data directory. + // + // For the sake of automating away as much of this as possible, see if the data directory is a + // symlink, and if so resolve to its final real path, and then update the configuration to use + // that. + if d, err := filepath.EvalSymlinks(_config.System.Data); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if d != _config.System.Data { + _config.System.Data = d + } + + log.WithField("path", _config.System.Data).Debug("ensuring server data directory exists") + if err := os.MkdirAll(_config.System.Data, 0700); err != nil { + return err + } + + log.WithField("path", _config.System.ArchiveDirectory).Debug("ensuring archive data directory exists") + if err := os.MkdirAll(_config.System.ArchiveDirectory, 0700); err != nil { + return err + } + + log.WithField("path", _config.System.BackupDirectory).Debug("ensuring backup data directory exists") + if err := os.MkdirAll(_config.System.BackupDirectory, 0700); err != nil { return err } return nil } +// EnableLogRotation writes a logrotate file for wings to the system logrotate +// configuration directory if one exists and a logrotate file is not found. This +// allows us to basically automate away the log rotation for most installs, but +// also enable users to make modifications on their own. +// +// This function IS NOT thread-safe. +func EnableLogRotation() error { + if !_config.System.EnableLogRotate { + log.Info("skipping log rotate configuration, disabled in wings config file") + return nil + } + + if st, err := os.Stat("/etc/logrotate.d"); err != nil && !os.IsNotExist(err) { + return err + } else if (err != nil && os.IsNotExist(err)) || !st.IsDir() { + return nil + } + if _, err := os.Stat("/etc/logrotate.d/wings"); err == nil || !os.IsNotExist(err) { + return err + } + + log.Info("no log rotation configuration found: adding file now") + // If we've gotten to this point it means the logrotate directory exists on the system + // but there is not a file for wings already. In that case, let us write a new file to + // it so files can be rotated easily. + f, err := os.Create("/etc/logrotate.d/wings") + if err != nil { + return err + } + defer f.Close() + + t, err := template.New("logrotate").Parse(` +{{.LogDirectory}}/wings.log { + size 10M + compress + delaycompress + dateext + maxage 7 + missingok + notifempty + create 0640 {{.User.Uid}} {{.User.Gid}} + postrotate + killall -SIGHUP wings + endscript +}`) + if err != nil { + return err + } + + return errors.Wrap(t.Execute(f, _config.System), "config: failed to write logrotate to disk") +} + +// GetStatesPath returns the location of the JSON file that tracks server states. +func (sc *SystemConfiguration) GetStatesPath() string { + return path.Join(sc.RootDirectory, "/states.json") +} + +// ConfigureTimezone sets the timezone data for the configuration if it is +// currently missing. If a value has been set, this functionality will only run +// to validate that the timezone being used is valid. +// +// This function IS NOT thread-safe. +func ConfigureTimezone() error { + if _config.System.Timezone == "" { + b, err := ioutil.ReadFile("/etc/timezone") + if err != nil { + if !os.IsNotExist(err) { + return errors.WithMessage(err, "config: failed to open timezone file") + } + + _config.System.Timezone = "UTC" + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + // Okay, file isn't found on this OS, we will try using timedatectl to handle this. If this + // command fails, exit, but if it returns a value use that. If no value is returned we will + // fall through to UTC to get Wings booted at least. + out, err := exec.CommandContext(ctx, "timedatectl").Output() + if err != nil { + log.WithField("error", err).Warn("failed to execute \"timedatectl\" to determine system timezone, falling back to UTC") + return nil + } + + r := regexp.MustCompile(`Time zone: ([\w/]+)`) + matches := r.FindSubmatch(out) + if len(matches) != 2 || string(matches[1]) == "" { + log.Warn("failed to parse timezone from \"timedatectl\" output, falling back to UTC") + return nil + } + _config.System.Timezone = string(matches[1]) + } else { + _config.System.Timezone = string(b) + } + } + + _config.System.Timezone = regexp.MustCompile(`(?i)[^a-z_/]+`).ReplaceAllString(_config.System.Timezone, "") + _, err := time.LoadLocation(_config.System.Timezone) + + return errors.WithMessage(err, fmt.Sprintf("the supplied timezone %s is invalid", _config.System.Timezone)) +} + // Gets the system release name. func getSystemName() (string, error) { // use osrelease to get release version and ID - if release, err := osrelease.Read(); err != nil { + release, err := osrelease.Read() + if err != nil { return "", err - } else { - return release["ID"], nil } -} + return release["ID"], nil +} \ No newline at end of file diff --git a/config/config_docker.go b/config/config_docker.go index f538173..3db5a56 100644 --- a/config/config_docker.go +++ b/config/config_docker.go @@ -12,7 +12,6 @@ type dockerNetworkInterfaces struct { Subnet string `default:"172.18.0.0/16"` Gateway string `default:"172.18.0.1"` } - V6 struct { Subnet string `default:"fdba:17c8:6c94::/64"` Gateway string `default:"fdba:17c8:6c94::1011"` @@ -39,8 +38,8 @@ type DockerNetworkConfiguration struct { Interfaces dockerNetworkInterfaces `yaml:"interfaces"` } -// Defines the docker configuration used by the daemon when interacting with -// containers and networks on the system. +// DockerConfiguration defines the docker configuration used by the daemon when +// interacting with containers and networks on the system. type DockerConfiguration struct { // Network configuration that should be used when creating a new network // for containers run through the daemon. @@ -58,23 +57,22 @@ type DockerConfiguration struct { TmpfsSize uint `default:"100" json:"tmpfs_size" yaml:"tmpfs_size"` } -// RegistryConfiguration . +// RegistryConfiguration defines the authentication credentials for a given +// Docker registry. type RegistryConfiguration struct { Username string `yaml:"username"` Password string `yaml:"password"` } -// Base64 . +// Base64 returns the authentication for a given registry as a base64 encoded +// string value. func (c RegistryConfiguration) Base64() (string, error) { - authConfig := types.AuthConfig{ + b, err := json.Marshal(types.AuthConfig{ Username: c.Username, Password: c.Password, - } - - b, err := json.Marshal(authConfig) + }) if err != nil { return "", err } - return base64.URLEncoding.EncodeToString(b), nil } diff --git a/config/config_system.go b/config/config_system.go deleted file mode 100644 index 0b2fd9a..0000000 --- a/config/config_system.go +++ /dev/null @@ -1,279 +0,0 @@ -package config - -import ( - "context" - "fmt" - "html/template" - "io/ioutil" - "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "time" - - "emperror.dev/errors" - "github.com/apex/log" - "github.com/spf13/viper" -) - -// Defines basic system configuration settings. -type SystemConfiguration struct { - // The root directory where all of the pterodactyl data is stored at. - RootDirectory string `default:"/var/lib/pterodactyl" yaml:"root_directory"` - - // Directory where logs for server installations and other wings events are logged. - LogDirectory string `default:"/var/log/pterodactyl" yaml:"log_directory"` - - // Directory where the server data is stored at. - Data string `default:"/var/lib/pterodactyl/volumes" yaml:"data"` - - // Directory where server archives for transferring will be stored. - ArchiveDirectory string `default:"/var/lib/pterodactyl/archives" yaml:"archive_directory"` - - // Directory where local backups will be stored on the machine. - BackupDirectory string `default:"/var/lib/pterodactyl/backups" yaml:"backup_directory"` - - // The user that should own all of the server files, and be used for containers. - Username string `default:"pterodactyl" yaml:"username"` - - // The timezone for this Wings instance. This is detected by Wings automatically if possible, - // and falls back to UTC if not able to be detected. If you need to set this manually, that - // can also be done. - // - // This timezone value is passed into all containers created by Wings. - Timezone string `yaml:"timezone"` - - // Definitions for the user that gets created to ensure that we can quickly access - // this information without constantly having to do a system lookup. - User struct { - Uid int - Gid int - } - - // The amount of time in seconds that can elapse before a server's disk space calculation is - // considered stale and a re-check should occur. DANGER: setting this value too low can seriously - // impact system performance and cause massive I/O bottlenecks and high CPU usage for the Wings - // process. - // - // Set to 0 to disable disk checking entirely. This will always return 0 for the disk space used - // by a server and should only be set in extreme scenarios where performance is critical and - // disk usage is not a concern. - DiskCheckInterval int64 `default:"150" yaml:"disk_check_interval"` - - // If set to true, file permissions for a server will be checked when the process is - // booted. This can cause boot delays if the server has a large amount of files. In most - // cases disabling this should not have any major impact unless external processes are - // frequently modifying a servers' files. - CheckPermissionsOnBoot bool `default:"true" yaml:"check_permissions_on_boot"` - - // If set to false Wings will not attempt to write a log rotate configuration to the disk - // when it boots and one is not detected. - EnableLogRotate bool `default:"true" yaml:"enable_log_rotate"` - - // The number of lines to send when a server connects to the websocket. - WebsocketLogCount int `default:"150" yaml:"websocket_log_count"` - - Sftp SftpConfiguration `yaml:"sftp"` - - CrashDetection CrashDetection `yaml:"crash_detection"` - - Backups Backups `yaml:"backups"` - - Transfers Transfers `yaml:"transfers"` -} - -type CrashDetection struct { - // Determines if Wings should detect a server that stops with a normal exit code of - // "0" as being crashed if the process stopped without any Wings interaction. E.g. - // the user did not press the stop button, but the process stopped cleanly. - DetectCleanExitAsCrash bool `default:"true" yaml:"detect_clean_exit_as_crash"` - - // Timeout specifies the timeout between crashes that will not cause the server - // to be automatically restarted, this value is used to prevent servers from - // becoming stuck in a boot-loop after multiple consecutive crashes. - Timeout int `default:"60" json:"timeout"` -} - -type Backups struct { - // WriteLimit imposes a Disk I/O write limit on backups to the disk, this affects all - // backup drivers as the archiver must first write the file to the disk in order to - // upload it to any external storage provider. - // - // If the value is less than 1, the write speed is unlimited, - // if the value is greater than 0, the write speed is the value in MiB/s. - // - // Defaults to 0 (unlimited) - WriteLimit int `default:"0" yaml:"write_limit"` -} - -type Transfers struct { - // DownloadLimit imposes a Network I/O read limit when downloading a transfer archive. - // - // If the value is less than 1, the write speed is unlimited, - // if the value is greater than 0, the write speed is the value in MiB/s. - // - // Defaults to 0 (unlimited) - DownloadLimit int `default:"0" yaml:"download_limit"` -} - -// ConfigureDirectories ensures that all of the system directories exist on the -// system. These directories are created so that only the owner can read the data, -// and no other users. -func ConfigureDirectories() error { - root := viper.GetString("system.root_directory") - log.WithField("path", root).Debug("ensuring root data directory exists") - if err := os.MkdirAll(root, 0700); err != nil { - return err - } - - // There are a non-trivial number of users out there whose data directories are actually a - // symlink to another location on the disk. If we do not resolve that final destination at this - // point things will appear to work, but endless errors will be encountered when we try to - // verify accessed paths since they will all end up resolving outside the expected data directory. - // - // For the sake of automating away as much of this as possible, see if the data directory is a - // symlink, and if so resolve to its final real path, and then update the configuration to use - // that. - data := viper.GetString("system.data") - if d, err := filepath.EvalSymlinks(data); err != nil { - if !os.IsNotExist(err) { - return err - } - } else if d != data { - data = d - viper.Set("system.data", d) - } - - log.WithField("path", data).Debug("ensuring server data directory exists") - if err := os.MkdirAll(data, 0700); err != nil { - return err - } - - log.WithField("path", viper.GetString("system.archive_directory")).Debug("ensuring archive data directory exists") - if err := os.MkdirAll(viper.GetString("system.archive_directory"), 0700); err != nil { - return err - } - - log.WithField("path", viper.GetString("system.backup_directory")).Debug("ensuring backup data directory exists") - if err := os.MkdirAll(viper.GetString("system.backup_directory"), 0700); err != nil { - return err - } - - return nil -} - -// EnableLogRotation writes a logrotate file for wings to the system logrotate -// configuration directory if one exists and a logrotate file is not found. This -// allows us to basically automate away the log rotation for most installs, but -// also enable users to make modifications on their own. -func EnableLogRotation() error { - // Do nothing if not enabled. - if !viper.GetBool("system.enable_log_rotate") { - log.Info("skipping log rotate configuration, disabled in wings config file") - return nil - } - - if st, err := os.Stat("/etc/logrotate.d"); err != nil && !os.IsNotExist(err) { - return err - } else if (err != nil && os.IsNotExist(err)) || !st.IsDir() { - return nil - } - if _, err := os.Stat("/etc/logrotate.d/wings"); err == nil || !os.IsNotExist(err) { - return err - } - - log.Info("no log rotation configuration found: adding file now") - // If we've gotten to this point it means the logrotate directory exists on the system - // but there is not a file for wings already. In that case, let us write a new file to - // it so files can be rotated easily. - f, err := os.Create("/etc/logrotate.d/wings") - if err != nil { - return err - } - defer f.Close() - - type logrotateConfig struct { - Directory string - UserID int - GroupID int - } - - t, err := template.New("logrotate").Parse(` -{{.Directory}}/wings.log { - size 10M - compress - delaycompress - dateext - maxage 7 - missingok - notifempty - create 0640 {{.UserID}} {{.GroupID}} - postrotate - killall -SIGHUP wings - endscript -}`) - if err != nil { - return err - } - - err = t.Execute(f, logrotateConfig{ - Directory: viper.GetString("system.log_directory"), - UserID: viper.GetInt("system.user.uid"), - GroupID: viper.GetInt("system.user.gid"), - }) - return errors.Wrap(err, "config: failed to write logrotate to disk") -} - -// Returns the location of the JSON file that tracks server states. -func (sc *SystemConfiguration) GetStatesPath() string { - return path.Join(sc.RootDirectory, "states.json") -} - -// Returns the location of the JSON file that tracks server states. -func (sc *SystemConfiguration) GetInstallLogPath() string { - return path.Join(sc.LogDirectory, "install/") -} - -// ConfigureTimezone sets the timezone data for the configuration if it is -// currently missing. If a value has been set, this functionality will only run -// to validate that the timezone being used is valid. -func ConfigureTimezone() error { - tz := viper.GetString("system.timezone") - defer viper.Set("system.timezone", tz) - if tz == "" { - b, err := ioutil.ReadFile("/etc/timezone") - if err != nil { - if !os.IsNotExist(err) { - return errors.WithMessage(err, "config: failed to open timezone file") - } - - tz = "UTC" - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - // Okay, file isn't found on this OS, we will try using timedatectl to handle this. If this - // command fails, exit, but if it returns a value use that. If no value is returned we will - // fall through to UTC to get Wings booted at least. - out, err := exec.CommandContext(ctx, "timedatectl").Output() - if err != nil { - log.WithField("error", err).Warn("failed to execute \"timedatectl\" to determine system timezone, falling back to UTC") - return nil - } - - r := regexp.MustCompile(`Time zone: ([\w/]+)`) - matches := r.FindSubmatch(out) - if len(matches) != 2 || string(matches[1]) == "" { - log.Warn("failed to parse timezone from \"timedatectl\" output, falling back to UTC") - return nil - } - tz = string(matches[1]) - } else { - tz = string(b) - } - } - - tz = regexp.MustCompile(`(?i)[^a-z_/]+`).ReplaceAllString(tz, "") - _, err := time.LoadLocation(tz) - - return errors.WithMessage(err, fmt.Sprintf("the supplied timezone %s is invalid", tz)) -} diff --git a/config/config_throttles.go b/config/config_throttles.go deleted file mode 100644 index e9c5531..0000000 --- a/config/config_throttles.go +++ /dev/null @@ -1,27 +0,0 @@ -package config - -type ConsoleThrottles struct { - // Whether or not the throttler is enabled for this instance. - Enabled bool `json:"enabled" yaml:"enabled" default:"true"` - - // The total number of lines that can be output in a given LineResetInterval period before - // a warning is triggered and counted against the server. - Lines uint64 `json:"lines" yaml:"lines" default:"2000"` - - // The total number of throttle activations that can accumulate before a server is considered - // to be breaching and will be stopped. This value is decremented by one every DecayInterval. - MaximumTriggerCount uint64 `json:"maximum_trigger_count" yaml:"maximum_trigger_count" default:"5"` - - // The amount of time after which the number of lines processed is reset to 0. This runs in - // a constant loop and is not affected by the current console output volumes. By default, this - // will reset the processed line count back to 0 every 100ms. - LineResetInterval uint64 `json:"line_reset_interval" yaml:"line_reset_interval" default:"100"` - - // The amount of time in milliseconds that must pass without an output warning being triggered - // before a throttle activation is decremented. - DecayInterval uint64 `json:"decay_interval" yaml:"decay_interval" default:"10000"` - - // The amount of time that a server is allowed to be stopping for before it is terminated - // forcefully if it triggers output throttles. - StopGracePeriod uint `json:"stop_grace_period" yaml:"stop_grace_period" default:"15"` -} diff --git a/environment/docker.go b/environment/docker.go index 8b8b9c3..edb7b81 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -9,7 +9,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/spf13/viper" ) var _conce sync.Once diff --git a/go.mod b/go.mod index b25e755..4659f61 100644 --- a/go.mod +++ b/go.mod @@ -63,8 +63,6 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/cobra v1.1.1 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.7.1 github.com/ugorji/go v1.2.2 // indirect github.com/ulikunitz/xz v0.5.9 // indirect golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad diff --git a/go.sum b/go.sum index ca88e19..ff4afb8 100644 --- a/go.sum +++ b/go.sum @@ -567,9 +567,8 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= diff --git a/server/install.go b/server/install.go index 7faf69d..d6d392d 100644 --- a/server/install.go +++ b/server/install.go @@ -326,7 +326,7 @@ func (ip *InstallationProcess) BeforeExecute() error { // Returns the log path for the installation process. func (ip *InstallationProcess) GetLogPath() string { - return filepath.Join(config.Get().System.GetInstallLogPath(), ip.Server.Id()+".log") + return filepath.Join(config.Get().System.LogDirectory, "/install", ip.Server.Id()+".log") } // Cleans up after the execution of the installation process. This grabs the logs from the From 05c04c4350c37a3f0238c4a31690b0d1002ac36d Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 14 Jan 2021 20:19:28 -0800 Subject: [PATCH 16/30] Update remaining logic using viper --- cmd/diagnostics.go | 2 +- config/config.go | 2 +- environment/docker.go | 81 ++++++++++++++++--------------- environment/docker/environment.go | 2 +- server/install.go | 2 +- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/cmd/diagnostics.go b/cmd/diagnostics.go index d67185f..e726d71 100644 --- a/cmd/diagnostics.go +++ b/cmd/diagnostics.go @@ -188,7 +188,7 @@ func diagnosticsCmdRun(cmd *cobra.Command, args []string) { } func getDockerInfo() (types.Version, types.Info, error) { - cli, err := environment.DockerClient() + cli, err := environment.Docker() if err != nil { return types.Version{}, types.Info{}, err } diff --git a/config/config.go b/config/config.go index 7138a38..0a7ec8a 100644 --- a/config/config.go +++ b/config/config.go @@ -340,8 +340,8 @@ func Get() *Configuration { // the global configuration. func Update(callback func(c *Configuration)) { mu.Lock() - defer mu.Unlock() callback(_config) + mu.Unlock() } // GetJwtAlgorithm returns the in-memory JWT algorithm. diff --git a/environment/docker.go b/environment/docker.go index edb7b81..68ef38d 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -9,15 +9,16 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" + "github.com/pterodactyl/wings/config" ) var _conce sync.Once var _client *client.Client -// DockerClient returns a Docker client to be used throughout the codebase. Once -// a client has been created it will be returned for all subsequent calls to this +// Docker returns a docker client to be used throughout the codebase. Once a +// client has been created it will be returned for all subsequent calls to this // function. -func DockerClient() (*client.Client, error) { +func Docker() (*client.Client, error) { var err error _conce.Do(func() { _client, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) @@ -28,13 +29,13 @@ func DockerClient() (*client.Client, error) { // ConfigureDocker configures the required network for the docker environment. func ConfigureDocker(ctx context.Context) error { // Ensure the required docker network exists on the system. - cli, err := DockerClient() + cli, err := Docker() if err != nil { return err } - nw := viper.Sub("docker.network") - resource, err := cli.NetworkInspect(ctx, nw.GetString("name"), types.NetworkInspectOptions{}) + nw := config.Get().Docker.Network + resource, err := cli.NetworkInspect(ctx, nw.Name, types.NetworkInspectOptions{}) if err != nil { if client.IsErrNotFound(err) { log.Info("creating missing pterodactyl0 interface, this could take a few seconds...") @@ -43,57 +44,59 @@ func ConfigureDocker(ctx context.Context) error { } } return err - } else { - nw.Set("driver", resource.Driver) } - switch nw.GetString("driver") { - case "host": - nw.Set("interface", "127.0.0.1") - nw.Set("ispn", false) - case "overlay": - fallthrough - case "weavemesh": - nw.Set("interface", "") - nw.Set("ispn", true) - default: - nw.Set("ispn", false) - } + config.Update(func(c *config.Configuration) { + c.Docker.Network.Driver = resource.Driver + switch c.Docker.Network.Driver { + case "host": + c.Docker.Network.Interface = "127.0.0.1" + c.Docker.Network.ISPN = false + case "overlay": + fallthrough + case "weavemesh": + c.Docker.Network.Interface = "" + c.Docker.Network.ISPN = true + default: + c.Docker.Network.ISPN = false + } + }) return nil } // Creates a new network on the machine if one does not exist already. func createDockerNetwork(ctx context.Context, cli *client.Client) error { - nw := viper.Sub("docker.network") - _, err := cli.NetworkCreate(ctx, nw.GetString("name"), types.NetworkCreate{ - Driver: nw.GetString("driver"), + nw := config.Get().Docker.Network + _, err := cli.NetworkCreate(ctx, nw.Name, types.NetworkCreate{ + Driver: nw.Driver, EnableIPv6: true, - Internal: nw.GetBool("is_internal"), + Internal: nw.IsInternal, IPAM: &network.IPAM{ - Config: []network.IPAMConfig{ - { - Subnet: nw.GetString("interfaces.v4.subnet"), - Gateway: nw.GetString("interfaces.v4.gateway"), - }, - { - Subnet: nw.GetString("interfaces.v6.subnet"), - Gateway: nw.GetString("interfaces.v6.gateway"), - }, - }, + Config: []network.IPAMConfig{{ + Subnet: nw.Interfaces.V4.Subnet, + Gateway: nw.Interfaces.V4.Gateway, + }, { + Subnet: nw.Interfaces.V6.Subnet, + Gateway: nw.Interfaces.V6.Gateway, + }}, }, Options: map[string]string{ "encryption": "false", "com.docker.network.bridge.default_bridge": "false", - "com.docker.network.bridge.enable_icc": strconv.FormatBool(nw.GetBool("enable_icc")), + "com.docker.network.bridge.enable_icc": strconv.FormatBool(nw.EnableICC), "com.docker.network.bridge.enable_ip_masquerade": "true", "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", "com.docker.network.bridge.name": "pterodactyl0", "com.docker.network.driver.mtu": "1500", }, }) - driver := nw.GetString("driver") - if driver != "host" && driver != "overlay" && driver != "weavemesh" { - nw.Set("interface", nw.GetString("interfaces.v4.gateway")) + if err != nil { + return err } - return err + if nw.Driver != "host" && nw.Driver != "overlay" && nw.Driver != "weavemesh" { + config.Update(func(c *config.Configuration) { + c.Docker.Network.Interface = c.Docker.Network.Interfaces.V4.Gateway + }) + } + return nil } diff --git a/environment/docker/environment.go b/environment/docker/environment.go index 6b1762d..296fe6d 100644 --- a/environment/docker/environment.go +++ b/environment/docker/environment.go @@ -55,7 +55,7 @@ type Environment struct { // reference the container from here on out. This should be unique per-server (we use the UUID // by default). The container does not need to exist at this point. func New(id string, m *Metadata, c *environment.Configuration) (*Environment, error) { - cli, err := environment.DockerClient() + cli, err := environment.Docker() if err != nil { return nil, err } diff --git a/server/install.go b/server/install.go index d6d392d..e72555e 100644 --- a/server/install.go +++ b/server/install.go @@ -127,7 +127,7 @@ func NewInstallationProcess(s *Server, script *api.InstallationScript) (*Install Server: s, } - if c, err := environment.DockerClient(); err != nil { + if c, err := environment.Docker(); err != nil { return nil, err } else { proc.client = c From c2cfaf44b5176c44abed47d920a7ba903be8b681 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 14 Jan 2021 20:32:38 -0800 Subject: [PATCH 17/30] Get wings booting again --- cmd/diagnostics.go | 117 +++++++++++++++++++++------------------- cmd/root.go | 2 +- router/router_system.go | 30 ++++------- 3 files changed, 75 insertions(+), 74 deletions(-) diff --git a/cmd/diagnostics.go b/cmd/diagnostics.go index e726d71..b487c9f 100644 --- a/cmd/diagnostics.go +++ b/cmd/diagnostics.go @@ -15,14 +15,15 @@ import ( "strings" "time" - "github.com/pterodactyl/wings/environment" - "github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2/terminal" + "github.com/apex/log" "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/parsers/operatingsystem" "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/environment" + "github.com/pterodactyl/wings/loggers/cli" "github.com/pterodactyl/wings/system" "github.com/spf13/cobra" ) @@ -40,15 +41,21 @@ var ( } ) -var diagnosticsCmd = &cobra.Command{ - Use: "diagnostics", - Short: "Collect diagnostics information.", - Run: diagnosticsCmdRun, -} +func newDiagnosticsCommand() *cobra.Command { + command := &cobra.Command{ + Use: "diagnostics", + Short: "Collect and report information about this Wings instance to assist in debugging.", + PreRun: func(cmd *cobra.Command, args []string) { + initConfig() + log.SetHandler(cli.Default) + }, + Run: diagnosticsCmdRun, + } -func init() { - diagnosticsCmd.PersistentFlags().StringVar(&diagnosticsArgs.HastebinURL, "hastebin-url", DefaultHastebinUrl, "The url of the hastebin instance to use.") - diagnosticsCmd.PersistentFlags().IntVar(&diagnosticsArgs.LogLines, "log-lines", DefaultLogLines, "The number of log lines to include in the report") + command.Flags().StringVar(&diagnosticsArgs.HastebinURL, "hastebin-url", DefaultHastebinUrl, "the url of the hastebin instance to use") + command.Flags().IntVar(&diagnosticsArgs.LogLines, "log-lines", DefaultLogLines, "the number of log lines to include in the report") + + return command } // diagnosticsCmdRun collects diagnostics about wings, it's configuration and the node. @@ -85,7 +92,6 @@ func diagnosticsCmdRun(cmd *cobra.Command, args []string) { } dockerVersion, dockerInfo, dockerErr := getDockerInfo() - _ = dockerInfo output := &strings.Builder{} fmt.Fprintln(output, "Pterodactyl Wings - Diagnostics Report") @@ -102,50 +108,53 @@ func diagnosticsCmdRun(cmd *cobra.Command, args []string) { } printHeader(output, "Wings Configuration") - cfg, err := config.FromFile(config.DefaultLocation) - if cfg != nil { - fmt.Fprintln(output, " Panel Location:", redact(cfg.PanelLocation)) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Internal Webserver:", redact(cfg.Api.Host), ":", cfg.Api.Port) - fmt.Fprintln(output, " SSL Enabled:", cfg.Api.Ssl.Enabled) - fmt.Fprintln(output, " SSL Certificate:", redact(cfg.Api.Ssl.CertificateFile)) - fmt.Fprintln(output, " SSL Key:", redact(cfg.Api.Ssl.KeyFile)) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " SFTP Server:", redact(cfg.System.Sftp.Address), ":", cfg.System.Sftp.Port) - fmt.Fprintln(output, " SFTP Read-Only:", cfg.System.Sftp.ReadOnly) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Root Directory:", cfg.System.RootDirectory) - fmt.Fprintln(output, " Logs Directory:", cfg.System.LogDirectory) - fmt.Fprintln(output, " Data Directory:", cfg.System.Data) - fmt.Fprintln(output, " Archive Directory:", cfg.System.ArchiveDirectory) - fmt.Fprintln(output, " Backup Directory:", cfg.System.BackupDirectory) - fmt.Fprintln(output, "") - fmt.Fprintln(output, " Username:", cfg.System.Username) - fmt.Fprintln(output, " Server Time:", time.Now().Format(time.RFC1123Z)) - fmt.Fprintln(output, " Debug Mode:", cfg.Debug) - } else { - fmt.Println("Failed to load configuration.", err) + if err := config.FromFile(config.DefaultLocation); err != nil { + } + cfg := config.Get() + fmt.Fprintln(output, " Panel Location:", redact(cfg.PanelLocation)) + fmt.Fprintln(output, "") + fmt.Fprintln(output, " Internal Webserver:", redact(cfg.Api.Host), ":", cfg.Api.Port) + fmt.Fprintln(output, " SSL Enabled:", cfg.Api.Ssl.Enabled) + fmt.Fprintln(output, " SSL Certificate:", redact(cfg.Api.Ssl.CertificateFile)) + fmt.Fprintln(output, " SSL Key:", redact(cfg.Api.Ssl.KeyFile)) + fmt.Fprintln(output, "") + fmt.Fprintln(output, " SFTP Server:", redact(cfg.System.Sftp.Address), ":", cfg.System.Sftp.Port) + fmt.Fprintln(output, " SFTP Read-Only:", cfg.System.Sftp.ReadOnly) + fmt.Fprintln(output, "") + fmt.Fprintln(output, " Root Directory:", cfg.System.RootDirectory) + fmt.Fprintln(output, " Logs Directory:", cfg.System.LogDirectory) + fmt.Fprintln(output, " Data Directory:", cfg.System.Data) + fmt.Fprintln(output, " Archive Directory:", cfg.System.ArchiveDirectory) + fmt.Fprintln(output, " Backup Directory:", cfg.System.BackupDirectory) + fmt.Fprintln(output, "") + fmt.Fprintln(output, " Username:", cfg.System.Username) + fmt.Fprintln(output, " Server Time:", time.Now().Format(time.RFC1123Z)) + fmt.Fprintln(output, " Debug Mode:", cfg.Debug) printHeader(output, "Docker: Info") - fmt.Fprintln(output, "Server Version:", dockerInfo.ServerVersion) - fmt.Fprintln(output, "Storage Driver:", dockerInfo.Driver) - if dockerInfo.DriverStatus != nil { - for _, pair := range dockerInfo.DriverStatus { - fmt.Fprintf(output, " %s: %s\n", pair[0], pair[1]) + if dockerErr == nil { + fmt.Fprintln(output, "Server Version:", dockerInfo.ServerVersion) + fmt.Fprintln(output, "Storage Driver:", dockerInfo.Driver) + if dockerInfo.DriverStatus != nil { + for _, pair := range dockerInfo.DriverStatus { + fmt.Fprintf(output, " %s: %s\n", pair[0], pair[1]) + } } - } - if dockerInfo.SystemStatus != nil { - for _, pair := range dockerInfo.SystemStatus { - fmt.Fprintf(output, " %s: %s\n", pair[0], pair[1]) + if dockerInfo.SystemStatus != nil { + for _, pair := range dockerInfo.SystemStatus { + fmt.Fprintf(output, " %s: %s\n", pair[0], pair[1]) + } } - } - fmt.Fprintln(output, "LoggingDriver:", dockerInfo.LoggingDriver) - fmt.Fprintln(output, " CgroupDriver:", dockerInfo.CgroupDriver) - if len(dockerInfo.Warnings) > 0 { - for _, w := range dockerInfo.Warnings { - fmt.Fprintln(output, w) + fmt.Fprintln(output, "LoggingDriver:", dockerInfo.LoggingDriver) + fmt.Fprintln(output, " CgroupDriver:", dockerInfo.CgroupDriver) + if len(dockerInfo.Warnings) > 0 { + for _, w := range dockerInfo.Warnings { + fmt.Fprintln(output, w) + } } + } else { + fmt.Fprintln(output, dockerErr.Error()) } printHeader(output, "Docker: Running Containers") @@ -180,23 +189,23 @@ func diagnosticsCmdRun(cmd *cobra.Command, args []string) { survey.AskOne(&survey.Confirm{Message: "Upload to " + diagnosticsArgs.HastebinURL + "?", Default: false}, &upload) } if upload { - url, err := uploadToHastebin(diagnosticsArgs.HastebinURL, output.String()) + u, err := uploadToHastebin(diagnosticsArgs.HastebinURL, output.String()) if err == nil { - fmt.Println("Your report is available here: ", url) + fmt.Println("Your report is available here: ", u) } } } func getDockerInfo() (types.Version, types.Info, error) { - cli, err := environment.Docker() + client, err := environment.Docker() if err != nil { return types.Version{}, types.Info{}, err } - dockerVersion, err := cli.ServerVersion(context.Background()) + dockerVersion, err := client.ServerVersion(context.Background()) if err != nil { return types.Version{}, types.Info{}, err } - dockerInfo, err := cli.Info(context.Background()) + dockerInfo, err := client.Info(context.Background()) if err != nil { return types.Version{}, types.Info{}, err } diff --git a/cmd/root.go b/cmd/root.go index 2d4e775..efc5b6a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,7 +78,7 @@ func init() { rootCommand.AddCommand(versionCommand) rootCommand.AddCommand(configureCmd) - rootCommand.AddCommand(diagnosticsCmd) + rootCommand.AddCommand(newDiagnosticsCommand()) } func rootCmdRun(cmd *cobra.Command, _ []string) { diff --git a/router/router_system.go b/router/router_system.go index 140decc..a3bad4d 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -72,37 +72,29 @@ func postCreateServer(c *gin.Context) { c.Status(http.StatusAccepted) } -// Updates the running configuration for this daemon instance. +// Updates the running configuration for this Wings instance. func postUpdateConfiguration(c *gin.Context) { - // A backup of the configuration for error purposes. - ccopy := *config.Get() - // A copy of the configuration we're using to bind the data received into. - cfg := *config.Get() - - // BindJSON sends 400 if the request fails, all we need to do is return + cfg := config.Get() if err := c.BindJSON(&cfg); err != nil { return } - // Keep the SSL certificates the same since the Panel will send through Lets Encrypt // default locations. However, if we picked a different location manually we don't // want to override that. // // If you pass through manual locations in the API call this logic will be skipped. if strings.HasPrefix(cfg.Api.Ssl.KeyFile, "/etc/letsencrypt/live/") { - cfg.Api.Ssl.KeyFile = strings.ToLower(ccopy.Api.Ssl.KeyFile) - cfg.Api.Ssl.CertificateFile = strings.ToLower(ccopy.Api.Ssl.CertificateFile) + cfg.Api.Ssl.KeyFile = strings.ToLower(config.Get().Api.Ssl.KeyFile) + cfg.Api.Ssl.CertificateFile = strings.ToLower(config.Get().Api.Ssl.CertificateFile) } - - config.Set(&cfg) - if err := config.Get().WriteToDisk(); err != nil { - // If there was an error writing to the disk, revert back to the configuration we had - // before this code was run. - config.Set(&ccopy) - - NewTrackedError(err).Abort(c) + // Try to write this new configuration to the disk before updating our global + // state with it. + if err := config.WriteToDisk(cfg); err != nil { + WithError(c, err) return } - + // Since we wrote it to the disk successfully now update the global configuration + // state to use this new configuration struct. + config.Set(cfg) c.Status(http.StatusNoContent) } From f6669213e89f18807be7ac0e8a5b6df3750e2356 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 11:02:57 -0800 Subject: [PATCH 18/30] Begin refactoring and improving the middleware --- router/middleware.go | 58 +------- router/middleware/middleware.go | 230 ++++++++++++++++++++++++++++++++ router/router.go | 6 +- router/router_server_files.go | 20 +-- 4 files changed, 248 insertions(+), 66 deletions(-) create mode 100644 router/middleware/middleware.go diff --git a/router/middleware.go b/router/middleware.go index 7fd52bf..69fd9df 100644 --- a/router/middleware.go +++ b/router/middleware.go @@ -1,7 +1,6 @@ package router import ( - "io" "net/http" "strings" @@ -14,59 +13,10 @@ import ( type Middleware struct{} -// A custom handler function allowing for errors bubbled up by c.Error() to be returned in a -// standardized format with tracking UUIDs on them for easier log searching. -func (m *Middleware) ErrorHandler() gin.HandlerFunc { - return func(c *gin.Context) { - c.Next() - err := c.Errors.Last() - if err == nil || err.Err == nil { - return - } - tracked := NewTrackedError(err.Err) - // If there is a server in the context for this request pull it out so that we can - // track the error specifically for that server. - if s, ok := c.Get("server"); ok { - tracked = NewServerError(err.Err, s.(*server.Server)) - } - // This error occurs if you submit invalid JSON data to an endpoint. - if err.Err.Error() == io.EOF.Error() { - c.JSON(c.Writer.Status(), gin.H{"error": "A JSON formatted body is required for this endpoint."}) - return - } - tracked.Abort(c) - return - } -} - -// Set the access request control headers on all of the requests. -func (m *Middleware) SetAccessControlHeaders() gin.HandlerFunc { - origins := config.Get().AllowedOrigins - location := config.Get().PanelLocation - return func(c *gin.Context) { - c.Header("Access-Control-Allow-Credentials", "true") - c.Header("Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS") - c.Header("Access-Control-Allow-Headers", "Accept, Accept-Encoding, Authorization, Cache-Control, Content-Type, Content-Length, Origin, X-Real-IP, X-CSRF-Token") - - o := c.GetHeader("Origin") - if o != location { - for _, origin := range origins { - if origin != "*" && o != origin { - continue - } - c.Header("Access-Control-Allow-Origin", origin) - c.Next() - return - } - } - c.Header("Access-Control-Allow-Origin", location) - c.Next() - } -} - -// Authenticates the request token against the given permission string, ensuring that -// if it is a server permission, the token has control over that server. If it is a global -// token, this will ensure that the request is using a properly signed global token. +// RequireAuthorization authenticates the request token against the given +// permission string, ensuring that if it is a server permission, the token has +// control over that server. If it is a global token, this will ensure that the +// request is using a properly signed global token. func (m *Middleware) RequireAuthorization() gin.HandlerFunc { token := config.Get().AuthenticationToken return func(c *gin.Context) { diff --git a/router/middleware/middleware.go b/router/middleware/middleware.go new file mode 100644 index 0000000..b366bc7 --- /dev/null +++ b/router/middleware/middleware.go @@ -0,0 +1,230 @@ +package middleware + +import ( + "context" + "io" + "net/http" + "os" + "strings" + + "emperror.dev/errors" + "github.com/apex/log" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/server" + "github.com/pterodactyl/wings/server/filesystem" +) + +// RequestError is a custom error type returned when something goes wrong with +// any of the HTTP endpoints. +type RequestError struct { + err error + status int + msg string +} + +// NewError returns a new RequestError for the provided error. +func NewError(err error) *RequestError { + return &RequestError{ + // Attach a stacktrace to the error if it is missing at this point and mark it + // as originating from the location where NewError was called, rather than this + // specific point in the code. + err: errors.WithStackDepthIf(err, 1), + } +} + +// SetMessage allows for a custom error message to be set on an existing +// RequestError instance. +func (re *RequestError) SetMessage(m string) { + re.msg = m +} + +// SetStatus sets the HTTP status code for the error response. By default this +// is a HTTP-500 error. +func (re *RequestError) SetStatus(s int) { + re.status = s +} + +// Abort aborts the given HTTP request with the specified status code and then +// logs the event into the logs. The error that is output will include the unique +// request ID if it is present. +func (re *RequestError) Abort(c *gin.Context, status int) { + reqId := c.Writer.Header().Get("X-Request-Id") + + // Generate the base logger instance, attaching the unique request ID and + // the URL that was requested. + event := log.WithField("request_id", reqId).WithField("url", c.Request.URL.String()) + // If there is a server present in the gin.Context stack go ahead and pull it + // and attach that server UUID to the logs as well so that we can see what specific + // server triggered this error. + if s, ok := c.Get("server"); ok { + if s, ok := s.(*server.Server); ok { + event = event.WithField("server_id", s.Id()) + } + } + + if c.Writer.Status() == 200 { + // Handle context deadlines being exceeded a little differently since we want + // to report a more user-friendly error and a proper error code. The "context + // canceled" error is generally when a request is terminated before all of the + // logic is finished running. + if errors.Is(re.err, context.DeadlineExceeded) { + re.SetStatus(http.StatusGatewayTimeout) + re.SetMessage("The server could not process this request in time, please try again.") + } else if strings.Contains(re.Cause().Error(), "context canceled") { + re.SetStatus(http.StatusBadRequest) + re.SetMessage("Request aborted by client.") + } + } + + // c.Writer.Status() will be a non-200 value if the headers have already been sent + // to the requester but an error is encountered. This can happen if there is an issue + // marshaling a struct placed into a c.JSON() call (or c.AbortWithJSON() call). + if status >= 500 || c.Writer.Status() != 200 { + event.WithField("status", status).WithField("error", re.err).Error("error while handling HTTP request") + } else { + event.WithField("status", status).WithField("error", re.err).Debug("error handling HTTP request (not a server error)") + } + if re.msg == "" { + re.msg = "An unexpected error was encountered while processing this request" + } + // Now abort the request with the error message and include the unique request + // ID that was present to make things super easy on people who don't know how + // or cannot view the response headers (where X-Request-Id would be present). + c.AbortWithStatusJSON(status, gin.H{"error": re.msg, "request_id": reqId}) +} + +// Cause returns the underlying error. +func (re *RequestError) Cause() error { + return re.err +} + +// Error returns the underlying error message for this request. +func (re *RequestError) Error() string { + return re.err.Error() +} + +// Looks at the given RequestError and determines if it is a specific filesystem +// error that we can process and return differently for the user. +// +// Some external things end up calling fmt.Errorf() on our filesystem errors +// which ends up just unleashing chaos on the system. For the sake of this, +// fallback to using text checks. +// +// If the error passed into this call is nil or does not match empty values will +// be returned to the caller. +func (re *RequestError) asFilesystemError() (int, string) { + err := re.Cause() + if err == nil { + return 0, "" + } + if filesystem.IsErrorCode(err, filesystem.ErrCodeDenylistFile) || strings.Contains(err.Error(), "filesystem: file access prohibited") { + return http.StatusForbidden, "This file cannot be modified: present in egg denylist." + } + if filesystem.IsErrorCode(err, filesystem.ErrCodePathResolution) || strings.Contains(err.Error(), "resolves to a location outside the server root") { + return http.StatusNotFound, "The requested resource was not found on the system." + } + if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) || strings.Contains(err.Error(), "filesystem: is a directory") { + return http.StatusBadRequest, "Cannot perform that action: file is a directory." + } + if filesystem.IsErrorCode(err, filesystem.ErrCodeDiskSpace) || strings.Contains(err.Error(), "filesystem: not enough disk space") { + return http.StatusBadRequest, "Cannot perform that action: file is a directory." + } + if strings.HasSuffix(err.Error(), "file name too long") { + return http.StatusBadRequest, "Cannot perform that action: file name is too long." + } + if e, ok := err.(*os.SyscallError); ok && e.Syscall == "readdirent" { + return http.StatusNotFound, "The requested directory does not exist." + } + return 0, "" +} + +// AttachRequestID attaches a unique ID to the incoming HTTP request so that any +// errors that are generated or returned to the client will include this reference +// allowing for an easier time identifying the specific request that failed for +// the user. +// +// If you are using a tool such as Sentry or Bugsnag for error reporting this is +// a great location to also attach this request ID to your error handling logic +// so that you can easily cross-reference the errors. +func AttachRequestID() gin.HandlerFunc { + return func(c *gin.Context) { + c.Header("X-Request-Id", uuid.New().String()) + c.Next() + } +} + +// CaptureAndAbort aborts the request and attaches the provided error to the gin +// context so it can be reported properly. If the error is missing a stacktrace +// at the time it is called the stack will be attached. +func CaptureAndAbort(c *gin.Context, err error) { + c.Abort() + c.Error(errors.WithStackDepthIf(err, 1)) +} + +// CaptureErrors is custom handler function allowing for errors bubbled up by +// c.Error() to be returned in a standardized format with tracking UUIDs on them +// for easier log searching. +func CaptureErrors() gin.HandlerFunc { + return func(c *gin.Context) { + c.Next() + err := c.Errors.Last() + if err == nil || err.Err == nil { + return + } + + status := http.StatusInternalServerError + if c.Writer.Status() != 200 { + status = c.Writer.Status() + } + if err.Error() == io.EOF.Error() { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "The data passed in the request was not in a parsable format. Please try again."}) + return + } + captured := NewError(err.Err) + if status, msg := captured.asFilesystemError(); msg != "" { + c.AbortWithStatusJSON(status, gin.H{"error": msg, "request_id": c.Writer.Header().Get("X-Request-Id")}) + return + } + captured.Abort(c, status) + } +} + +// SetAccessControlHeaders sets the access request control headers on all of +// the requests. +func SetAccessControlHeaders() gin.HandlerFunc { + origins := config.Get().AllowedOrigins + location := config.Get().PanelLocation + + return func(c *gin.Context) { + c.Header("Access-Control-Allow-Credentials", "true") + c.Header("Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS") + // Maximum age allowable under Chromium v76 is 2 hours, so just use that since + // anything higher will be ignored (even if other browsers do allow higher values). + // + // @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Max-Age#Directives + c.Header("Access-Control-Max-Age", "7200") + c.Header("Access-Control-Allow-Origin", location) + c.Header("Access-Control-Allow-Headers", "Accept, Accept-Encoding, Authorization, Cache-Control, Content-Type, Content-Length, Origin, X-Real-IP, X-CSRF-Token") + // Validate that the request origin is coming from an allowed origin. Because you + // cannot set multiple values here we need to see if the origin is one of the ones + // that we allow, and if so return it explicitly. Otherwise, just return the default + // origin which is the same URL that the Panel is located at. + origin := c.GetHeader("Origin") + if origin != location { + for _, o := range origins { + if o != "*" && o != origin { + continue + } + c.Header("Access-Control-Allow-Origin", o) + break + } + } + if c.Request.Method == http.MethodOptions { + c.AbortWithStatus(http.StatusNoContent) + return + } + c.Next() + } +} diff --git a/router/router.go b/router/router.go index 0eb4075..3073b4b 100644 --- a/router/router.go +++ b/router/router.go @@ -3,15 +3,17 @@ package router import ( "github.com/apex/log" "github.com/gin-gonic/gin" + "github.com/pterodactyl/wings/router/middleware" ) -// Configures the routing infrastructure for this daemon instance. +// Configure configures the routing infrastructure for this daemon instance. func Configure() *gin.Engine { gin.SetMode("release") m := Middleware{} router := gin.New() - router.Use(gin.Recovery(), m.ErrorHandler(), m.SetAccessControlHeaders()) + router.Use(gin.Recovery()) + router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders()) // @todo log this into a different file so you can setup IP blocking for abusive requests and such. // This should still dump requests in debug mode since it does help with understanding the request // lifecycle and quickly seeing what was called leading to the logs. However, it isn't feasible to mix diff --git a/router/router_server_files.go b/router/router_server_files.go index b6a3af5..7833e72 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -15,6 +15,7 @@ import ( "github.com/apex/log" "github.com/gin-gonic/gin" "github.com/pterodactyl/wings/router/downloader" + "github.com/pterodactyl/wings/router/middleware" "github.com/pterodactyl/wings/router/tokens" "github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/server/filesystem" @@ -375,30 +376,29 @@ func postServerCompressFiles(c *gin.Context) { }) } +// postServerDecompressFiles receives the HTTP request and starts the process +// of unpacking an archive that exists on the server into the provided RootPath +// for the server. func postServerDecompressFiles(c *gin.Context) { - s := GetServer(c.Param("server")) - + s := ExtractServer(c) var data struct { RootPath string `json:"root"` File string `json:"file"` } - if err := c.BindJSON(&data); err != nil { return } + lg := s.Log().WithFields(log.Fields{"root_path": data.RootPath, "file": data.File}) + lg.Debug("checking if space is available for decompression") hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File) if err != nil { - // Handle an unknown format error. if filesystem.IsErrorCode(err, filesystem.ErrCodeUnknownArchive) { - s.Log().WithField("error", err).Warn("failed to decompress file due to unknown format") - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "unknown archive format", - }) + s.Log().WithField("path", data.RootPath).WithField("file", data.File).WithField("error", err).Warn("failed to decompress file due to unknown format") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "The archive provided is in a format Wings does not understand."}) return } - - NewServerError(err, s).Abort(c) + middleware.CaptureAndAbort(c, err) return } From b17cf5b93dd3a5875e4d4027fa857a3021027640 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 11:19:33 -0800 Subject: [PATCH 19/30] More middleware cleanup and movement --- router/middleware.go | 91 ++++----------------------------- router/middleware/middleware.go | 86 ++++++++++++++++++++++++++++++- router/router.go | 15 +++--- 3 files changed, 102 insertions(+), 90 deletions(-) diff --git a/router/middleware.go b/router/middleware.go index 69fd9df..e199c9e 100644 --- a/router/middleware.go +++ b/router/middleware.go @@ -1,96 +1,25 @@ package router import ( - "net/http" - "strings" - - "emperror.dev/errors" "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/router/middleware" "github.com/pterodactyl/wings/server" ) -type Middleware struct{} - -// RequireAuthorization authenticates the request token against the given -// permission string, ensuring that if it is a server permission, the token has -// control over that server. If it is a global token, this will ensure that the -// request is using a properly signed global token. -func (m *Middleware) RequireAuthorization() gin.HandlerFunc { - token := config.Get().AuthenticationToken - return func(c *gin.Context) { - auth := strings.SplitN(c.GetHeader("Authorization"), " ", 2) - if len(auth) != 2 || auth[0] != "Bearer" { - c.Header("WWW-Authenticate", "Bearer") - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "The required authorization heads were not present in the request.", - }) - return - } - - // All requests to Wings must be authorized with the authentication token present in - // the Wings configuration file. Remeber, all requests to Wings come from the Panel - // backend, or using a signed JWT for temporary authentication. - if auth[1] == token { - c.Next() - return - } - c.AbortWithStatusJSON(http.StatusForbidden, gin.H{ - "error": "You are not authorized to access this endpoint.", - }) - } -} - -// Helper function to fetch a server out of the servers collection stored in memory. -// -// This function should not be used in new controllers, prefer ExtractServer where -// possible. +// GetServer is a helper function to fetch a server out of the servers +// collection stored in memory. This function should not be used in new +// controllers, prefer ExtractServer where possible. +// Deprecated func GetServer(uuid string) *server.Server { return server.GetServers().Find(func(s *server.Server) bool { return uuid == s.Id() }) } -// Ensure that the requested server exists in this setup. Returns a 404 if we cannot -// locate it. -func (m *Middleware) ServerExists() gin.HandlerFunc { - return func(c *gin.Context) { - u, err := uuid.Parse(c.Param("server")) - if err == nil { - if s := GetServer(u.String()); s != nil { - c.Set("server", s) - c.Next() - return - } - } - c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ - "error": "The resource you requested does not exist.", - }) - } -} - -// Checks if remote file downloading is enabled on this instance before allowing access -// to the given endpoint. -func (m *Middleware) CheckRemoteDownloadEnabled() gin.HandlerFunc { - disabled := config.Get().Api.DisableRemoteDownload - return func(c *gin.Context) { - if disabled { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ - "error": "This functionality is not currently enabled on this instance.", - }) - return - } - c.Next() - } -} - -// Returns the server instance from the gin context. If there is no server set in the -// context (e.g. calling from a controller not protected by ServerExists) this function -// will panic. +// ExtractServer returns the server instance from the gin context. If there is +// no server set in the context (e.g. calling from a controller not protected by +// ServerExists) this function will panic. +// Deprecated func ExtractServer(c *gin.Context) *server.Server { - if s, ok := c.Get("server"); ok { - return s.(*server.Server) - } - panic(errors.New("cannot extract server, missing on gin context")) + return middleware.ExtractServer(c) } diff --git a/router/middleware/middleware.go b/router/middleware/middleware.go index b366bc7..f659282 100644 --- a/router/middleware/middleware.go +++ b/router/middleware/middleware.go @@ -2,6 +2,7 @@ package middleware import ( "context" + "crypto/subtle" "io" "net/http" "os" @@ -150,7 +151,9 @@ func (re *RequestError) asFilesystemError() (int, string) { // so that you can easily cross-reference the errors. func AttachRequestID() gin.HandlerFunc { return func(c *gin.Context) { - c.Header("X-Request-Id", uuid.New().String()) + id := uuid.New().String() + c.Header("X-Request-Id", id) + c.Set("logger", log.WithField("request_id", id)) c.Next() } } @@ -228,3 +231,84 @@ func SetAccessControlHeaders() gin.HandlerFunc { c.Next() } } + +// ServerExists will ensure that the requested server exists in this setup. +// Returns a 404 if we cannot locate it. If the server is found it is set into +// the request context, and the logger for the context is also updated to include +// the server ID in the fields list. +func ServerExists() gin.HandlerFunc { + return func(c *gin.Context) { + s := server.GetServers().Find(func(s *server.Server) bool { + return c.Param("server") == s.Id() + }) + if s == nil { + c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": "The requested resource does not exist on this instance."}) + return + } + c.Set("logger", ExtractLogger(c).WithField("server_id", s.Id())) + c.Set("server", s) + c.Next() + } +} + +// RequireAuthorization authenticates the request token against the given +// permission string, ensuring that if it is a server permission, the token has +// control over that server. If it is a global token, this will ensure that the +// request is using a properly signed global token. +func RequireAuthorization() gin.HandlerFunc { + return func(c *gin.Context) { + // We don't put this value outside this function since the node's authentication + // token can be changed on the fly and the config.Get() call returns a copy, so + // if it is rotated this value will never properly get updated. + token := config.Get().AuthenticationToken + auth := strings.SplitN(c.GetHeader("Authorization"), " ", 2) + if len(auth) != 2 || auth[0] != "Bearer" { + c.Header("WWW-Authenticate", "Bearer") + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "The required authorization heads were not present in the request."}) + return + } + + // All requests to Wings must be authorized with the authentication token present in + // the Wings configuration file. Remeber, all requests to Wings come from the Panel + // backend, or using a signed JWT for temporary authentication. + if subtle.ConstantTimeCompare([]byte(auth[1]), []byte(token)) != 1 { + c.AbortWithStatusJSON(http.StatusForbidden, gin.H{"error": "You are not authorized to access this endpoint."}) + return + } + c.Next() + } +} + +// RemoteDownloadEnabled checks if remote downloads are enabled for this instance +// and if not aborts the request. +func RemoteDownloadEnabled() gin.HandlerFunc { + disabled := config.Get().Api.DisableRemoteDownload + return func(c *gin.Context) { + if disabled { + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "This functionality is not currently enabled on this instance."}) + return + } + c.Next() + } +} + +// ExtractLogger pulls the logger out of the request context and returns it. By +// default this will include the request ID, but may also include the server ID +// if that middleware has been used in the chain by the time it is called. +func ExtractLogger(c *gin.Context) *log.Entry { + v, ok := c.Get("logger") + if !ok { + panic("middleware/middleware: cannot extract logger: not present in request context") + } + return v.(*log.Entry) +} + +// ExtractServer will return the server from the gin.Context or panic if it is +// not present. +func ExtractServer(c *gin.Context) *server.Server { + v, ok := c.Get("server") + if !ok { + panic("middleware/middleware: cannot extract server: not present in request context") + } + return v.(*server.Server) +} \ No newline at end of file diff --git a/router/router.go b/router/router.go index 3073b4b..cce9f71 100644 --- a/router/router.go +++ b/router/router.go @@ -10,7 +10,6 @@ import ( func Configure() *gin.Engine { gin.SetMode("release") - m := Middleware{} router := gin.New() router.Use(gin.Recovery()) router.Use(middleware.AttachRequestID(), middleware.CaptureErrors(), middleware.SetAccessControlHeaders()) @@ -41,16 +40,16 @@ func Configure() *gin.Engine { // This route is special it sits above all of the other requests because we are // using a JWT to authorize access to it, therefore it needs to be publicly // accessible. - router.GET("/api/servers/:server/ws", m.ServerExists(), getServerWebsocket) + router.GET("/api/servers/:server/ws", middleware.ServerExists(), getServerWebsocket) // This request is called by another daemon when a server is going to be transferred out. // This request does not need the AuthorizationMiddleware as the panel should never call it // and requests are authenticated through a JWT the panel issues to the other daemon. - router.GET("/api/servers/:server/archive", m.ServerExists(), getServerArchive) + router.GET("/api/servers/:server/archive", middleware.ServerExists(), getServerArchive) // All of the routes beyond this mount will use an authorization middleware // and will not be accessible without the correct Authorization header provided. - protected := router.Use(m.RequireAuthorization()) + protected := router.Use(middleware.RequireAuthorization()) protected.POST("/api/update", postUpdateConfiguration) protected.GET("/api/system", getSystemInformation) protected.GET("/api/servers", getAllServers) @@ -60,7 +59,7 @@ func Configure() *gin.Engine { // These are server specific routes, and require that the request be authorized, and // that the server exist on the Daemon. server := router.Group("/api/servers/:server") - server.Use(m.RequireAuthorization(), m.ServerExists()) + server.Use(middleware.RequireAuthorization(), middleware.ServerExists()) { server.GET("", getServer) server.PATCH("", patchServer) @@ -90,9 +89,9 @@ func Configure() *gin.Engine { files.POST("/decompress", postServerDecompressFiles) files.POST("/chmod", postServerChmodFile) - files.GET("/pull", m.CheckRemoteDownloadEnabled(), getServerPullingFiles) - files.POST("/pull", m.CheckRemoteDownloadEnabled(), postServerPullRemoteFile) - files.DELETE("/pull/:download", m.CheckRemoteDownloadEnabled(), deleteServerPullRemoteFile) + files.GET("/pull", middleware.RemoteDownloadEnabled(), getServerPullingFiles) + files.POST("/pull", middleware.RemoteDownloadEnabled(), postServerPullRemoteFile) + files.DELETE("/pull/:download", middleware.RemoteDownloadEnabled(), deleteServerPullRemoteFile) } backup := server.Group("/backup") From 67ecbd667ad2b13e0af6d6c29e3f7440ce5150f9 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 11:48:30 -0800 Subject: [PATCH 20/30] Minor improvements to logic around decompression --- router/middleware/middleware.go | 2 +- router/router_server_files.go | 26 ++++----- server/filesystem/decompress.go | 51 +++++++++--------- server/filesystem/errors.go | 95 ++++++++++++++++++++++----------- server/filesystem/filesystem.go | 5 +- 5 files changed, 102 insertions(+), 77 deletions(-) diff --git a/router/middleware/middleware.go b/router/middleware/middleware.go index f659282..b63d49f 100644 --- a/router/middleware/middleware.go +++ b/router/middleware/middleware.go @@ -130,7 +130,7 @@ func (re *RequestError) asFilesystemError() (int, string) { return http.StatusBadRequest, "Cannot perform that action: file is a directory." } if filesystem.IsErrorCode(err, filesystem.ErrCodeDiskSpace) || strings.Contains(err.Error(), "filesystem: not enough disk space") { - return http.StatusBadRequest, "Cannot perform that action: file is a directory." + return http.StatusBadRequest, "There is not enough disk space available to perform that action." } if strings.HasSuffix(err.Error(), "file name too long") { return http.StatusBadRequest, "Cannot perform that action: file name is too long." diff --git a/router/router_server_files.go b/router/router_server_files.go index 7833e72..864eb1a 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -380,7 +380,8 @@ func postServerCompressFiles(c *gin.Context) { // of unpacking an archive that exists on the server into the provided RootPath // for the server. func postServerDecompressFiles(c *gin.Context) { - s := ExtractServer(c) + s := middleware.ExtractServer(c) + lg := middleware.ExtractLogger(c) var data struct { RootPath string `json:"root"` File string `json:"file"` @@ -389,12 +390,12 @@ func postServerDecompressFiles(c *gin.Context) { return } - lg := s.Log().WithFields(log.Fields{"root_path": data.RootPath, "file": data.File}) - lg.Debug("checking if space is available for decompression") - hasSpace, err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File) + lg = lg.WithFields(log.Fields{"root_path": data.RootPath, "file": data.File}) + lg.Debug("checking if space is available for file decompression") + err := s.Filesystem().SpaceAvailableForDecompression(data.RootPath, data.File) if err != nil { if filesystem.IsErrorCode(err, filesystem.ErrCodeUnknownArchive) { - s.Log().WithField("path", data.RootPath).WithField("file", data.File).WithField("error", err).Warn("failed to decompress file due to unknown format") + lg.WithField("error", err).Warn("failed to decompress file: unknown archive format") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "The archive provided is in a format Wings does not understand."}) return } @@ -402,30 +403,21 @@ func postServerDecompressFiles(c *gin.Context) { return } - if !hasSpace { - c.AbortWithStatusJSON(http.StatusConflict, gin.H{ - "error": "This server does not have enough available disk space to decompress this archive.", - }) - return - } - + lg.Info("starting file decompression") if err := s.Filesystem().DecompressFile(data.RootPath, data.File); err != nil { // If the file is busy for some reason just return a nicer error to the user since there is not // much we specifically can do. They'll need to stop the running server process in order to overwrite // a file like this. if strings.Contains(err.Error(), "text file busy") { - s.Log().WithField("error", err).Warn("failed to decompress file due to busy text file") - + lg.WithField("error", err).Warn("failed to decompress file: text file busy") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "One or more files this archive is attempting to overwrite are currently in use by another process. Please try again.", }) return } - - NewServerError(err, s).Abort(c) + middleware.CaptureAndAbort(c, err) return } - c.Status(http.StatusNoContent) } diff --git a/server/filesystem/decompress.go b/server/filesystem/decompress.go index 4fb2988..51c3bff 100644 --- a/server/filesystem/decompress.go +++ b/server/filesystem/decompress.go @@ -4,28 +4,29 @@ import ( "archive/tar" "archive/zip" "compress/gzip" - "emperror.dev/errors" "fmt" - "github.com/mholt/archiver/v3" "os" "path/filepath" "reflect" "strings" "sync/atomic" + + "emperror.dev/errors" + "github.com/mholt/archiver/v3" ) -// Look through a given archive and determine if decompressing it would put the server over -// its allocated disk space limit. -func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (bool, error) { +// SpaceAvailableForDecompression looks through a given archive and determines +// if decompressing it would put the server over its allocated disk space limit. +func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) error { // Don't waste time trying to determine this if we know the server will have the space for // it since there is no limit. if fs.MaxDisk() <= 0 { - return true, nil + return nil } source, err := fs.SafePath(filepath.Join(dir, file)) if err != nil { - return false, err + return err } // Get the cached size in a parallel process so that if it is not cached we are not @@ -38,32 +39,28 @@ func (fs *Filesystem) SpaceAvailableForDecompression(dir string, file string) (b if atomic.AddInt64(&size, f.Size())+dirSize > fs.MaxDisk() { return &Error{code: ErrCodeDiskSpace} } - return nil }) - if err != nil { if strings.HasPrefix(err.Error(), "format ") { - return false, &Error{code: ErrCodeUnknownArchive} + return &Error{code: ErrCodeUnknownArchive} } - - return false, err + return err } - - return true, err + return err } -// Decompress a file in a given directory by using the archiver tool to infer the file -// type and go from there. This will walk over all of the files within the given archive -// and ensure that there is not a zip-slip attack being attempted by validating that the -// final path is within the server data directory. +// DecompressFile will decompress a file in a given directory by using the +// archiver tool to infer the file type and go from there. This will walk over +// all of the files within the given archive and ensure that there is not a +// zip-slip attack being attempted by validating that the final path is within +// the server data directory. func (fs *Filesystem) DecompressFile(dir string, file string) error { source, err := fs.SafePath(filepath.Join(dir, file)) if err != nil { return err } - - // Make sure the file exists basically. + // Ensure that the source archive actually exists on the system. if _, err := os.Stat(source); err != nil { return err } @@ -79,7 +76,6 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { } var name string - switch s := f.Sys().(type) { case *tar.Header: name = s.Name @@ -88,7 +84,11 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { case *zip.FileHeader: name = s.Name default: - return errors.New(fmt.Sprintf("could not parse underlying data source with type %s", reflect.TypeOf(s).String())) + return &Error{ + code: ErrCodeUnknownError, + resolved: filepath.Join(dir, f.Name()), + err: errors.New(fmt.Sprintf("could not parse underlying data source with type: %s", reflect.TypeOf(s).String())), + } } p := filepath.Join(dir, name) @@ -96,15 +96,16 @@ func (fs *Filesystem) DecompressFile(dir string, file string) error { if err := fs.IsIgnored(p); err != nil { return nil } - return errors.WithMessage(fs.Writefile(p, f), "could not extract file from archive") + if err := fs.Writefile(p, f); err != nil { + return &Error{code: ErrCodeUnknownError, err: err, resolved: source} + } + return nil }) if err != nil { if strings.HasPrefix(err.Error(), "format ") { return &Error{code: ErrCodeUnknownArchive} } - return err } - return nil } diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index 4cc9929..ac8159e 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -17,19 +17,33 @@ const ( ErrCodeUnknownArchive ErrorCode = "E_UNKNFMT" ErrCodePathResolution ErrorCode = "E_BADPATH" ErrCodeDenylistFile ErrorCode = "E_DENYLIST" + ErrCodeUnknownError ErrorCode = "E_UNKNOWN" ) type Error struct { - code ErrorCode - path string + code ErrorCode + // Contains the underlying error leading to this. This value may or may not be + // present, it is entirely dependent on how this error was triggered. + err error + // This contains the value of the final destination that triggered this specific + // error event. resolved string + // This value is generally only present on errors stemming from a path resolution + // error. For everything else you should be setting and reading the resolved path + // value which will be far more useful. + path string +} + +// Code returns the ErrorCode for this specific error instance. +func (e *Error) Code() ErrorCode { + return e.code } // Returns a human-readable error string to identify the Error by. func (e *Error) Error() string { switch e.code { case ErrCodeIsDirectory: - return "filesystem: is a directory" + return fmt.Sprintf("filesystem: cannot perform action: [%s] is a directory", e.resolved) case ErrCodeDiskSpace: return "filesystem: not enough disk space" case ErrCodeUnknownArchive: @@ -46,36 +60,17 @@ func (e *Error) Error() string { r = "" } return fmt.Sprintf("filesystem: server path [%s] resolves to a location outside the server root: %s", e.path, r) + case ErrCodeUnknownError: + fallthrough + default: + return fmt.Sprintf("filesystem: an error occurred: %s", e.Cause()) } - return "filesystem: unhandled error type" } -// Returns the ErrorCode for this specific error instance. -func (e *Error) Code() ErrorCode { - return e.code -} - -// Checks if the given error is one of the Filesystem errors. -func IsFilesystemError(err error) (*Error, bool) { - var fserr *Error - if errors.As(err, &fserr) { - return fserr, true - } - return nil, false -} - -// Checks if "err" is a filesystem Error type. If so, it will then drop in and check -// that the error code is the same as the provided ErrorCode passed in "code". -func IsErrorCode(err error, code ErrorCode) bool { - if e, ok := IsFilesystemError(err); ok { - return e.code == code - } - return false -} - -// Returns a new BadPathResolution error. -func NewBadPathResolution(path string, resolved string) *Error { - return &Error{code: ErrCodePathResolution, path: path, resolved: resolved} +// Cause returns the underlying cause of this filesystem error. In some causes +// there may not be a cause present, in which case nil will be returned. +func (e *Error) Cause() error { + return e.err } // Generates an error logger instance with some basic information. @@ -92,10 +87,46 @@ func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error { if !IsErrorCode(err, ErrCodePathResolution) { return err } - if f != nil && f.IsDir() { return filepath.SkipDir } - return nil } + +// IsFilesystemError checks if the given error is one of the Filesystem errors. +func IsFilesystemError(err error) bool { + var fserr *Error + if err != nil && errors.As(err, &fserr) { + return true + } + return false +} + +// IsErrorCode checks if "err" is a filesystem Error type. If so, it will then +// drop in and check that the error code is the same as the provided ErrorCode +// passed in "code". +func IsErrorCode(err error, code ErrorCode) bool { + var fserr *Error + if err != nil && errors.As(err, &fserr) { + return fserr.code == code + } + return false +} + +// NewBadPathResolution returns a new BadPathResolution error. +func NewBadPathResolution(path string, resolved string) *Error { + return &Error{code: ErrCodePathResolution, path: path, resolved: resolved} +} + +// WrapError wraps the provided error as a Filesystem error and attaches the +// provided resolved source to it. If the error is already a Filesystem error +// no action is taken. +func WrapError(err error, resolved string) *Error { + if err == nil { + return nil + } + if IsFilesystemError(err) { + return err.(*Error) + } + return &Error{code: ErrCodeUnknownError, err: err, resolved: resolved} +} \ No newline at end of file diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 4732a22..c5ebc1e 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -123,7 +123,8 @@ func (fs *Filesystem) Readfile(p string, w io.Writer) error { return err } -// Writes a file to the system. If the file does not already exist one will be created. +// Writefile writes a file to the system. If the file does not already exist one +// will be created. func (fs *Filesystem) Writefile(p string, r io.Reader) error { cleaned, err := fs.SafePath(p) if err != nil { @@ -138,7 +139,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { return err } else if err == nil { if stat.IsDir() { - return &Error{code: ErrCodeIsDirectory} + return &Error{code: ErrCodeIsDirectory, resolved: cleaned} } currentSize = stat.Size() } From 2968ea3498e2b41be8500f8ab3ce3bfce108ea96 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 12:03:55 -0800 Subject: [PATCH 21/30] Modify stat to embed os.FileInfo differently and update file content reader --- router/router_server_files.go | 42 +++++++++++++------------------ router/router_transfer.go | 2 +- server/archiver.go | 2 +- server/filesystem/filesystem.go | 38 ++++++++++++---------------- server/filesystem/stat.go | 41 +++++++++++++++--------------- server/filesystem/stat_darwin.go | 4 +-- server/filesystem/stat_linux.go | 2 +- server/filesystem/stat_windows.go | 2 +- sftp/handler.go | 2 +- 9 files changed, 60 insertions(+), 75 deletions(-) diff --git a/router/router_server_files.go b/router/router_server_files.go index 864eb1a..9679e17 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -1,6 +1,7 @@ package router import ( + "bufio" "context" "mime/multipart" "net/http" @@ -22,41 +23,32 @@ import ( "golang.org/x/sync/errgroup" ) -// Returns the contents of a file on the server. +// getServerFileContents returns the contents of a file on the server. func getServerFileContents(c *gin.Context) { - s := ExtractServer(c) - f := c.Query("file") - p := "/" + strings.TrimLeft(f, "/") - st, err := s.Filesystem().Stat(p) + s := middleware.ExtractServer(c) + p := "/" + strings.TrimLeft(c.Query("file"), "/") + f, st, err := s.Filesystem().File(p) if err != nil { - WithError(c, err) - return + middleware.CaptureAndAbort(c, err) } + defer f.Close() c.Header("X-Mime-Type", st.Mimetype) - c.Header("Content-Length", strconv.Itoa(int(st.Info.Size()))) - + c.Header("Content-Length", strconv.Itoa(int(st.Size()))) // If a download parameter is included in the URL go ahead and attach the necessary headers // so that the file can be downloaded. if c.Query("download") != "" { - c.Header("Content-Disposition", "attachment; filename="+st.Info.Name()) + c.Header("Content-Disposition", "attachment; filename="+st.Name()) c.Header("Content-Type", "application/octet-stream") } - - // TODO(dane): should probably come up with a different approach here. If an error is encountered - // by this Readfile call you'll end up causing a (recovered) panic in the program because so many - // headers have already been set. We should probably add a RawReadfile that just returns the file - // to be read and then we can stream from that safely without error. - // - // Until that becomes a problem though I'm just going to leave this how it is. The panic is recovered - // and a normal 500 error is returned to the client to my knowledge. It is also very unlikely to - // happen since we're doing so much before this point that would normally throw an error if there - // was a problem with the file. - if err := s.Filesystem().Readfile(p, c.Writer); err != nil { - WithError(c, err) - return + defer c.Writer.Flush() + _, err = bufio.NewReader(f).WriteTo(c.Writer) + if err != nil { + // Pretty sure this will unleash chaos on the response, but its a risk we can + // take since a panic will at least be recovered and this should be incredibly + // rare? + middleware.CaptureAndAbort(c, err) } - c.Writer.Flush() } // Returns the contents of a directory for a server. @@ -371,7 +363,7 @@ func postServerCompressFiles(c *gin.Context) { } c.JSON(http.StatusOK, &filesystem.Stat{ - Info: f, + FileInfo: f, Mimetype: "application/tar+gzip", }) } diff --git a/router/router_transfer.go b/router/router_transfer.go index ee13c97..0a1c479 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -100,7 +100,7 @@ func getServerArchive(c *gin.Context) { c.Header("X-Checksum", checksum) c.Header("X-Mime-Type", st.Mimetype) - c.Header("Content-Length", strconv.Itoa(int(st.Info.Size()))) + c.Header("Content-Length", strconv.Itoa(int(st.Size()))) c.Header("Content-Disposition", "attachment; filename="+s.Archiver.Name()) c.Header("Content-Type", "application/octet-stream") diff --git a/server/archiver.go b/server/archiver.go index f799fca..e221db4 100644 --- a/server/archiver.go +++ b/server/archiver.go @@ -46,7 +46,7 @@ func (a *Archiver) Stat() (*filesystem.Stat, error) { } return &filesystem.Stat{ - Info: s, + FileInfo: s, Mimetype: "application/tar+gzip", }, nil } diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index c5ebc1e..52da443 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -38,7 +38,7 @@ type Filesystem struct { isTest bool } -// Creates a new Filesystem instance for a given server. +// New creates a new Filesystem instance for a given server. func New(root string, size int64, denylist []string) *Filesystem { return &Filesystem{ root: root, @@ -50,27 +50,27 @@ func New(root string, size int64, denylist []string) *Filesystem { } } -// Returns the root path for the Filesystem instance. +// Path returns the root path for the Filesystem instance. func (fs *Filesystem) Path() string { return fs.root } -// Returns a reader for a file instance. -func (fs *Filesystem) File(p string) (*os.File, os.FileInfo, error) { +// File returns a reader for a file instance as well as the stat information. +func (fs *Filesystem) File(p string) (*os.File, Stat, error) { cleaned, err := fs.SafePath(p) if err != nil { - return nil, nil, err + return nil, Stat{}, err } - st, err := os.Stat(cleaned) + st, err := fs.Stat(cleaned) if err != nil { - return nil, nil, err + return nil, Stat{}, err } if st.IsDir() { - return nil, nil, &Error{code: ErrCodeIsDirectory} + return nil, Stat{}, &Error{code: ErrCodeIsDirectory} } f, err := os.Open(cleaned) if err != nil { - return nil, nil, err + return nil, Stat{}, err } return f, st, nil } @@ -437,9 +437,9 @@ func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, } } -// Lists the contents of a given directory and returns stat information about each -// file and folder within it. -func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { +// ListDirectory lists the contents of a given directory and returns stat +// information about each file and folder within it. +func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { cleaned, err := fs.SafePath(p) if err != nil { return nil, err @@ -455,7 +455,7 @@ func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { // You must initialize the output of this directory as a non-nil value otherwise // when it is marshaled into a JSON object you'll just get 'null' back, which will // break the panel badly. - out := make([]*Stat, len(files)) + out := make([]Stat, len(files)) // Iterate over all of the files and directories returned and perform an async process // to get the mime-type for them all. @@ -482,15 +482,10 @@ func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { } } - st := &Stat{ - Info: f, - Mimetype: d, - } - + st := Stat{FileInfo: f, Mimetype: d} if m != nil { st.Mimetype = m.String() } - out[idx] = st }(i, file) } @@ -500,17 +495,16 @@ func (fs *Filesystem) ListDirectory(p string) ([]*Stat, error) { // Sort the output alphabetically to begin with since we've run the output // through an asynchronous process and the order is gonna be very random. sort.SliceStable(out, func(i, j int) bool { - if out[i].Info.Name() == out[j].Info.Name() || out[i].Info.Name() > out[j].Info.Name() { + if out[i].Name() == out[j].Name() || out[i].Name() > out[j].Name() { return true } - return false }) // Then, sort it so that directories are listed first in the output. Everything // will continue to be alphabetized at this point. sort.SliceStable(out, func(i, j int) bool { - return out[i].Info.IsDir() + return out[i].IsDir() }) return out, nil diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go index a6c014d..a255ff6 100644 --- a/server/filesystem/stat.go +++ b/server/filesystem/stat.go @@ -2,14 +2,15 @@ package filesystem import ( "encoding/json" - "github.com/gabriel-vasile/mimetype" "os" "strconv" "time" + + "github.com/gabriel-vasile/mimetype" ) type Stat struct { - Info os.FileInfo + os.FileInfo Mimetype string } @@ -26,50 +27,48 @@ func (s *Stat) MarshalJSON() ([]byte, error) { Symlink bool `json:"symlink"` Mime string `json:"mime"` }{ - Name: s.Info.Name(), + Name: s.Name(), Created: s.CTime().Format(time.RFC3339), - Modified: s.Info.ModTime().Format(time.RFC3339), - Mode: s.Info.Mode().String(), + Modified: s.ModTime().Format(time.RFC3339), + Mode: s.Mode().String(), // Using `&os.ModePerm` on the file's mode will cause the mode to only have the permission values, and nothing else. - ModeBits: strconv.FormatUint(uint64(s.Info.Mode()&os.ModePerm), 8), - Size: s.Info.Size(), - Directory: s.Info.IsDir(), - File: !s.Info.IsDir(), - Symlink: s.Info.Mode().Perm()&os.ModeSymlink != 0, + ModeBits: strconv.FormatUint(uint64(s.Mode()&os.ModePerm), 8), + Size: s.Size(), + Directory: s.IsDir(), + File: !s.IsDir(), + Symlink: s.Mode().Perm()&os.ModeSymlink != 0, Mime: s.Mimetype, }) } -// Stats a file or folder and returns the base stat object from go along with the -// MIME data that can be used for editing files. -func (fs *Filesystem) Stat(p string) (*Stat, error) { +// Stat stats a file or folder and returns the base stat object from go along +// with the MIME data that can be used for editing files. +func (fs *Filesystem) Stat(p string) (Stat, error) { cleaned, err := fs.SafePath(p) if err != nil { - return nil, err + return Stat{}, err } - return fs.unsafeStat(cleaned) } -func (fs *Filesystem) unsafeStat(p string) (*Stat, error) { +func (fs *Filesystem) unsafeStat(p string) (Stat, error) { s, err := os.Stat(p) if err != nil { - return nil, err + return Stat{}, err } var m *mimetype.MIME if !s.IsDir() { m, err = mimetype.DetectFile(p) if err != nil { - return nil, err + return Stat{}, err } } - st := &Stat{ - Info: s, + st := Stat{ + FileInfo: s, Mimetype: "inode/directory", } - if m != nil { st.Mimetype = m.String() } diff --git a/server/filesystem/stat_darwin.go b/server/filesystem/stat_darwin.go index 4bc6abd..6d0cff3 100644 --- a/server/filesystem/stat_darwin.go +++ b/server/filesystem/stat_darwin.go @@ -5,9 +5,9 @@ import ( "time" ) -// Returns the time that the file/folder was created. +// CTime returns the time that the file/folder was created. func (s *Stat) CTime() time.Time { - st := s.Info.Sys().(*syscall.Stat_t) + st := s.Sys().(*syscall.Stat_t) return time.Unix(st.Ctimespec.Sec, st.Ctimespec.Nsec) } diff --git a/server/filesystem/stat_linux.go b/server/filesystem/stat_linux.go index e166740..a9c7fb3 100644 --- a/server/filesystem/stat_linux.go +++ b/server/filesystem/stat_linux.go @@ -7,7 +7,7 @@ import ( // Returns the time that the file/folder was created. func (s *Stat) CTime() time.Time { - st := s.Info.Sys().(*syscall.Stat_t) + st := s.Sys().(*syscall.Stat_t) // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) diff --git a/server/filesystem/stat_windows.go b/server/filesystem/stat_windows.go index 4cd6dff..3652677 100644 --- a/server/filesystem/stat_windows.go +++ b/server/filesystem/stat_windows.go @@ -8,5 +8,5 @@ import ( // However, I have no idea how to do this on windows, so we're skipping it // for right now. func (s *Stat) CTime() time.Time { - return s.Info.ModTime() + return s.ModTime() } diff --git a/sftp/handler.go b/sftp/handler.go index 1461876..5fb0c9d 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -268,7 +268,7 @@ func (h *Handler) Filelist(request *sftp.Request) (sftp.ListerAt, error) { h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error performing stat on file") return nil, sftp.ErrSSHFxFailure } - return ListerAt([]os.FileInfo{st.Info}), nil + return ListerAt([]os.FileInfo{st.FileInfo}), nil default: return nil, sftp.ErrSSHFxOpUnsupported } From 464f26a2c9d33b8bc391cb312178dbf7695244ce Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 12:07:31 -0800 Subject: [PATCH 22/30] Include the request ID in the request logs --- router/middleware/middleware.go | 5 +++-- router/router.go | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/router/middleware/middleware.go b/router/middleware/middleware.go index b63d49f..ff61300 100644 --- a/router/middleware/middleware.go +++ b/router/middleware/middleware.go @@ -152,8 +152,9 @@ func (re *RequestError) asFilesystemError() (int, string) { func AttachRequestID() gin.HandlerFunc { return func(c *gin.Context) { id := uuid.New().String() - c.Header("X-Request-Id", id) + c.Set("request_id", id) c.Set("logger", log.WithField("request_id", id)) + c.Header("X-Request-Id", id) c.Next() } } @@ -311,4 +312,4 @@ func ExtractServer(c *gin.Context) *server.Server { panic("middleware/middleware: cannot extract server: not present in request context") } return v.(*server.Server) -} \ No newline at end of file +} diff --git a/router/router.go b/router/router.go index cce9f71..947a504 100644 --- a/router/router.go +++ b/router/router.go @@ -18,11 +18,13 @@ func Configure() *gin.Engine { // lifecycle and quickly seeing what was called leading to the logs. However, it isn't feasible to mix // this output in production and still get meaningful logs from it since they'll likely just be a huge // spamfest. + router.Use() router.Use(gin.LoggerWithFormatter(func(params gin.LogFormatterParams) string { log.WithFields(log.Fields{ - "client_ip": params.ClientIP, - "status": params.StatusCode, - "latency": params.Latency, + "client_ip": params.ClientIP, + "status": params.StatusCode, + "latency": params.Latency, + "request_id": params.Keys["request_id"], }).Debugf("%s %s", params.MethodColor()+params.Method+params.ResetColor(), params.Path) return "" From 6a286fb444ba54badb795acc4cc33e86001f0ac6 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 16 Jan 2021 17:06:50 -0800 Subject: [PATCH 23/30] unnecessary link --- router/router.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/router/router.go b/router/router.go index 947a504..00eea02 100644 --- a/router/router.go +++ b/router/router.go @@ -30,10 +30,6 @@ func Configure() *gin.Engine { return "" })) - router.OPTIONS("/api/system", func(c *gin.Context) { - c.Status(200) - }) - // These routes use signed URLs to validate access to the resource being requested. router.GET("/download/backup", getDownloadBackup) router.GET("/download/file", getDownloadFile) From f7788e10a0875f15c07d1501283da258f73d32ae Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 21 Jan 2021 20:45:56 -0800 Subject: [PATCH 24/30] Fix broken error handling when creating network --- environment/docker.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment/docker.go b/environment/docker.go index 68ef38d..f47e370 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -42,8 +42,9 @@ func ConfigureDocker(ctx context.Context) error { if err := createDockerNetwork(ctx, cli); err != nil { return err } + } else { + return err } - return err } config.Update(func(c *config.Configuration) { From 8920f919b1766fb3bf7ae9115832ff628ff0b365 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 21 Jan 2021 20:08:54 -0800 Subject: [PATCH 25/30] Add makefile entry for remote debugging support with dev environment --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 594cc64..e419df5 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,12 @@ debug: go build -race sudo ./wings --debug --ignore-certificate-errors --config config.yml +# Runs a remotly debuggable session for Wings allowing an IDE to connect and target +# different breakpoints. +rmdebug: + go build -gcflags "all=-N -l" -race + sudo dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./wings -- --debug --ignore-certificate-errors --config config.yml + compress: upx --brute build/wings_* From 56af6fc1f82b26246e16c56d4d781435be23b0e4 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 21 Jan 2021 20:58:40 -0800 Subject: [PATCH 26/30] Correctly abort on file missing without panic --- router/router_server_files.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/router/router_server_files.go b/router/router_server_files.go index 9679e17..44e315b 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -30,6 +30,7 @@ func getServerFileContents(c *gin.Context) { f, st, err := s.Filesystem().File(p) if err != nil { middleware.CaptureAndAbort(c, err) + return } defer f.Close() @@ -48,6 +49,7 @@ func getServerFileContents(c *gin.Context) { // take since a panic will at least be recovered and this should be incredibly // rare? middleware.CaptureAndAbort(c, err) + return } } From 5d070cbdc5f5460cbca700c30e2b22b191388651 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 21 Jan 2021 20:58:52 -0800 Subject: [PATCH 27/30] Handle edge case where a user triggers an install when the server has no data directory --- server/install.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/install.go b/server/install.go index e72555e..fe0fcea 100644 --- a/server/install.go +++ b/server/install.go @@ -447,6 +447,14 @@ func (ip *InstallationProcess) Execute() (string, error) { NetworkMode: container.NetworkMode(config.Get().Docker.Network.Mode), } + // Ensure the root directory for the server exists properly before attempting + // to trigger the reinstall of the server. It is possible the directory would + // not exist when this runs if Wings boots with a missing directory and a user + // triggers a reinstall before trying to start the server. + if err := ip.Server.EnsureDataDirectoryExists(); err != nil { + return "", err + } + ip.Server.Log().WithField("install_script", ip.tempDir()+"/install.sh").Info("creating install container for server process") // Remove the temporary directory when the installation process finishes for this server container. defer func() { From 93506994a5bb122a49c10612749a2c1c6d4d6806 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 23 Jan 2021 10:45:29 -0800 Subject: [PATCH 28/30] Ensure the root directory for a server is always create when booting wings --- cmd/root.go | 6 ++++++ server/filesystem.go | 30 ------------------------------ server/server.go | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 30 deletions(-) delete mode 100644 server/filesystem.go diff --git a/cmd/root.go b/cmd/root.go index efc5b6a..a90eab2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -164,6 +164,12 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { for _, serv := range server.GetServers().All() { s := serv + // For each server we encounter make sure the root data directory exists. + if err := s.EnsureDataDirectoryExists(); err != nil { + s.Log().Error("could not create root data directory for server: not loading server...") + continue + } + pool.Submit(func() { s.Log().Info("configuring server environment and restoring to previous state") diff --git a/server/filesystem.go b/server/filesystem.go deleted file mode 100644 index d86b526..0000000 --- a/server/filesystem.go +++ /dev/null @@ -1,30 +0,0 @@ -package server - -import ( - "os" - - "github.com/pterodactyl/wings/server/filesystem" -) - -func (s *Server) Filesystem() *filesystem.Filesystem { - return s.fs -} - -// Ensures that the data directory for the server instance exists. -func (s *Server) EnsureDataDirectoryExists() error { - if _, err := os.Stat(s.fs.Path()); err != nil && !os.IsNotExist(err) { - return err - } else if err != nil { - // Create the server data directory because it does not currently exist - // on the system. - if err := os.MkdirAll(s.fs.Path(), 0700); err != nil { - return err - } - - if err := s.fs.Chown("/"); err != nil { - s.Log().WithField("error", err).Warn("failed to chown server data directory") - } - } - - return nil -} diff --git a/server/server.go b/server/server.go index f7acc9a..0e1fb05 100644 --- a/server/server.go +++ b/server/server.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "os" "strings" "sync" @@ -221,3 +222,27 @@ func (s *Server) ProcessConfiguration() *api.ProcessConfiguration { return s.procConfig } + +// Filesystem returns an instance of the filesystem for this server. +func (s *Server) Filesystem() *filesystem.Filesystem { + return s.fs +} + +// EnsureDataDirectoryExists ensures that the data directory for the server +// instance exists. +func (s *Server) EnsureDataDirectoryExists() error { + if _, err := os.Lstat(s.fs.Path()); err != nil { + if os.IsNotExist(err) { + s.Log().Debug("server: creating root directory and setting permissions") + if err := os.MkdirAll(s.fs.Path(), 0700); err != nil { + return errors.WithStack(err) + } + if err := s.fs.Chown("/"); err != nil { + s.Log().WithField("error", err).Warn("server: failed to chown server data directory") + } + } else { + return errors.WrapIf(err, "server: failed to stat server root directory") + } + } + return nil +} \ No newline at end of file From 60416360767487fe60d2b67b7c4cb373931cd0ea Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 23 Jan 2021 11:47:53 -0800 Subject: [PATCH 29/30] Fix SSL issues --- Dockerfile | 47 +++++++++++++++----------------------- docker-compose.example.yml | 1 + 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index dcb75d8..574bfda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,22 @@ # Stage 1 (Build) -FROM golang:1.15-alpine3.12 AS builder +FROM golang:1.15-alpine3.12 AS builder -ARG VERSION - -RUN apk add --update --no-cache git=2.26.2-r0 make=4.3-r0 upx=3.96-r0 - -WORKDIR /app/ - -COPY go.mod go.sum /app/ -RUN go mod download - -COPY . /app/ - -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ - -ldflags="-s -w -X github.com/pterodactyl/wings/system.Version=$VERSION" \ - -v \ - -trimpath \ - -o wings \ - wings.go - -RUN upx wings - -# --------------------------------------- # +ARG VERSION +RUN apk add --update --no-cache git=2.26.2-r0 make=4.3-r0 upx=3.96-r0 +WORKDIR /app/ +COPY go.mod go.sum /app/ +RUN go mod download +COPY . /app/ +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -ldflags="-s -w -X github.com/pterodactyl/wings/system.Version=$VERSION" \ + -v \ + -trimpath \ + -o wings \ + wings.go +RUN upx wings # Stage 2 (Final) -FROM busybox:1.33.0 - -RUN echo "ID=\"busybox\"" > /etc/os-release - -COPY --from=builder /app/wings /usr/bin/ - -CMD [ "wings", "--config", "/etc/pterodactyl/config.yml" ] +FROM busybox:1.33.0 +RUN echo "ID=\"busybox\"" > /etc/os-release +COPY --from=builder /app/wings /usr/bin/ +CMD [ "wings", "--config", "/etc/pterodactyl/config.yml" ] diff --git a/docker-compose.example.yml b/docker-compose.example.yml index d2aeb6c..583a0d1 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -22,6 +22,7 @@ services: - "/var/lib/pterodactyl/:/var/lib/pterodactyl/" - "/var/log/pterodactyl/:/var/log/pterodactyl/" - "/tmp/pterodactyl/:/tmp/pterodactyl/" + - "/etc/ssl/certs:/etc/ssl/certs" # you may need /srv/daemon-data if you are upgrading from an old daemon #- "/srv/daemon-data/:/srv/daemon-data/" # Required for ssl if you use let's encrypt. uncomment to use. From fecacc1339fc41698bd90b2aedb925f9c5e625a9 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 23 Jan 2021 13:46:57 -0800 Subject: [PATCH 30/30] USe readonly flag on compose --- docker-compose.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 583a0d1..8a6f57c 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -22,7 +22,7 @@ services: - "/var/lib/pterodactyl/:/var/lib/pterodactyl/" - "/var/log/pterodactyl/:/var/log/pterodactyl/" - "/tmp/pterodactyl/:/tmp/pterodactyl/" - - "/etc/ssl/certs:/etc/ssl/certs" + - "/etc/ssl/certs:/etc/ssl/certs:ro" # you may need /srv/daemon-data if you are upgrading from an old daemon #- "/srv/daemon-data/:/srv/daemon-data/" # Required for ssl if you use let's encrypt. uncomment to use.