From 74097cc4addc7648f739ed130225a7273f1c0e05 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Fri, 22 May 2020 11:01:27 -0600 Subject: [PATCH 01/30] Fix sending status event to all subscribers, send disk usage when server is offline --- router/websocket/websocket.go | 34 ++++++++++++++++++++++++++++++---- server/server.go | 4 ++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index f0471c6..d187c2c 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -1,6 +1,7 @@ package websocket import ( + "encoding/json" "fmt" "github.com/gbrlsnchs/jwt/v3" "github.com/google/uuid" @@ -223,15 +224,40 @@ func (h *Handler) HandleInbound(m Message) error { h.setJwt(token) } - // On every authentication event, send the current server status back - // to the client. :) - h.server.Events().Publish(server.StatusEvent, h.server.GetState()) - h.unsafeSendJson(Message{ Event: AuthenticationSuccessEvent, Args: []string{}, }) + // On every authentication event, send the current server status back + // to the client. :) + state := h.server.GetState() + h.SendJson(&Message{ + Event: server.StatusEvent, + Args: []string{state}, + }) + + // Only send the current disk usage if the server is offline, if docker container is running, + // Environment#EnableResourcePolling() will send this data to all clients. + if state == server.ProcessOfflineState { + _ = h.server.Filesystem.HasSpaceAvailable() + + resources := server.ResourceUsage{ + Memory: 0, + MemoryLimit: 0, + CpuAbsolute: 0.0, + Disk: h.server.Resources.Disk, + } + resources.Network.RxBytes = 0 + resources.Network.TxBytes = 0 + + b, _ := json.Marshal(resources) + h.SendJson(&Message{ + Event: server.StatsEvent, + Args: []string{string(b)}, + }) + } + return nil } case SetStateEvent: diff --git a/server/server.go b/server/server.go index 82deffe..db831c4 100644 --- a/server/server.go +++ b/server/server.go @@ -247,6 +247,10 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) { } s.Resources = ResourceUsage{} + // Force the disk usage to become cached to return in a resources response + // or when connecting to the websocket of an offline server. + go s.Filesystem.HasSpaceAvailable() + // Forces the configuration to be synced with the panel. if err := s.SyncWithConfiguration(data); err != nil { return nil, err From 6d7ab865d71b7d8ebdd6c47d64c664ba29a4db92 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 25 May 2020 15:51:36 -0700 Subject: [PATCH 02/30] Fix SFTP default configuration; closes pterodactyl/panel#2045 --- cmd/root.go | 6 ++---- config/config.go | 3 --- config/config_system.go | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 1f2edd1..144cc87 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -196,10 +196,8 @@ func rootCmdRun(*cobra.Command, []string) { // Wait until all of the servers are ready to go before we fire up the HTTP server. wg.Wait() - // If the SFTP subsystem should be started, do so now. - if c.System.Sftp.UseInternalSystem { - sftp.Initialize(c) - } + // Initalize SFTP. + sftp.Initialize(c) // Ensure the archive directory exists. if err := os.MkdirAll(c.System.ArchiveDirectory, 0755); err != nil { diff --git a/config/config.go b/config/config.go index f79b0f0..2490a27 100644 --- a/config/config.go +++ b/config/config.go @@ -84,9 +84,6 @@ type Configuration struct { // Defines the configuration of the internal SFTP server. type SftpConfiguration struct { - // If set to false, the internal SFTP server will not be booted and you will need - // to run the SFTP server independent of this program. - UseInternalSystem bool `default:"true" json:"use_internal" yaml:"use_internal"` // If set to true disk checking will not be performed. This will prevent the SFTP // server from checking the total size of a directory when uploading files. DisableDiskChecking bool `default:"false" yaml:"disable_disk_checking"` diff --git a/config/config_system.go b/config/config_system.go index b5f6095..968988b 100644 --- a/config/config_system.go +++ b/config/config_system.go @@ -51,7 +51,7 @@ type SystemConfiguration struct { // the user did not press the stop button, but the process stopped cleanly. DetectCleanExitAsCrash bool `default:"true" yaml:"detect_clean_exit_as_crash"` - Sftp *SftpConfiguration `yaml:"sftp"` + Sftp SftpConfiguration `yaml:"sftp"` } // Ensures that all of the system directories exist on the system. These directories are From 54510057bbab7c5d3623c57b2ddd7439f7a5059c Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 28 May 2020 19:26:41 -0700 Subject: [PATCH 03/30] Don't block server boot if there is an error pulling an image Obviously this requires the image to exist locally, but should avoid widespread issues when Quay inevitably goes down again. closes pterodactyl/panel#2076 --- server/environment_docker.go | 37 ++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/server/environment_docker.go b/server/environment_docker.go index aa0fa02..f7c0330 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -547,12 +547,45 @@ func (d *DockerEnvironment) DisableResourcePolling() error { return errors.WithStack(err) } -// Pulls the image from Docker. +// Pulls the image from Docker. If there is an error while pulling the image from the source +// but the image already exists locally, we will report that error to the logger but continue +// with the process. +// +// The reasoning behind this is that Quay has had some serious outages as of late, and we don't +// need to block all of the servers from booting just because of that. I'd imagine in a lot of +// cases an outage shouldn't affect users too badly. It'll at least keep existing servers working +// correctly if anything. // // @todo handle authorization & local images func (d *DockerEnvironment) ensureImageExists(c *client.Client) error { - out, err := c.ImagePull(context.Background(), d.Server.Container.Image, types.ImagePullOptions{All: false}) + ctx, _ := context.WithTimeout(context.Background(), time.Second*10) + + out, err := c.ImagePull(ctx, d.Server.Container.Image, types.ImagePullOptions{All: false}) if err != nil { + images, ierr := c.ImageList(ctx, types.ImageListOptions{}) + if ierr != nil { + // Well damn, something has gone really wrong here, just go ahead and abort there + // isn't much anything we can do to try and self-recover from this. + return ierr + } + + for _, img := range images { + for _, t := range img.RepoTags { + if t == d.Server.Container.Image { + zap.S().Warnw( + "unable to pull requested image from remote source, however the image exists locally", + zap.String("server", d.Server.Uuid), + zap.String("image", d.Server.Container.Image), + zap.Error(err), + ) + + // Okay, we found a matching container image, in that case just go ahead and return + // from this function, since there is nothing else we need to do here. + return nil + } + } + } + return err } defer out.Close() From 82ffb9804d40e16a636ca18cd867d7fae10df436 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 28 May 2020 19:52:47 -0700 Subject: [PATCH 04/30] Don't lock up websocket when sending error json; ref pterodactyl/panel#2076 --- router/router_server_ws.go | 2 +- router/websocket/websocket.go | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/router/router_server_ws.go b/router/router_server_ws.go index 2917749..7fe476a 100644 --- a/router/router_server_ws.go +++ b/router/router_server_ws.go @@ -53,7 +53,7 @@ func getServerWebsocket(c *gin.Context) { } if err := handler.HandleInbound(j); err != nil { - handler.SendErrorJson(err) + handler.SendErrorJson(j, err) } } } diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index f0471c6..269eb97 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -137,10 +137,7 @@ func (h *Handler) TokenValid() error { // Sends an error back to the connected websocket instance by checking the permissions // of the token. If the user has the "receive-errors" grant we will send back the actual // error message, otherwise we just send back a standard error message. -func (h *Handler) SendErrorJson(err error) error { - h.Lock() - defer h.Unlock() - +func (h *Handler) SendErrorJson(msg Message, err error) error { j := h.GetJwt() message := "an unexpected error was encountered while handling this request" @@ -156,13 +153,14 @@ func (h *Handler) SendErrorJson(err error) error { if !server.IsSuspendedError(err) { zap.S().Errorw( "an error was encountered in the websocket process", + zap.String("event", msg.Event), zap.String("server", h.server.Uuid), zap.String("error_identifier", u.String()), zap.Error(err), ) } - return h.Connection.WriteJSON(wsm) + return h.unsafeSendJson(wsm) } // Converts an error message into a more readable representation and returns a UUID From 4b366ae19eaca751a5b32be8c4f70a72e0cced68 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 28 May 2020 19:53:12 -0700 Subject: [PATCH 05/30] Don't cause a crash loop when part of the pre-server-boot process fails --- server/environment_docker.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/environment_docker.go b/server/environment_docker.go index f7c0330..adf38bd 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -182,6 +182,10 @@ func (d *DockerEnvironment) Start() error { // that point. defer func() { if sawError { + // If we don't set it to stopping first, you'll trigger crash detection which + // we don't want to do at this point since it'll just immediately try to do the + // exact same action that lead to it crashing in the first place... + d.Server.SetState(ProcessStoppingState) d.Server.SetState(ProcessOfflineState) } }() @@ -248,8 +252,8 @@ func (d *DockerEnvironment) Start() error { return errors.WithStack(err) } - opts := types.ContainerStartOptions{} - if err := d.Client.ContainerStart(context.Background(), d.Server.Uuid, opts); err != nil { + ctx, _ := context.WithTimeout(context.Background(), time.Second * 10) + if err := d.Client.ContainerStart(ctx, d.Server.Uuid, types.ContainerStartOptions{}); err != nil { return errors.WithStack(err) } From f8bffd8391a19bfbae78c97d3a78158df02d64ee Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 28 May 2020 22:07:53 -0700 Subject: [PATCH 06/30] Start the process of migrating from zap to a cleaner CLI output for logs --- api/api.go | 14 +++--- cmd/root.go | 79 ++++++++++++++++++------------ config/config.go | 4 +- config/config_system.go | 12 ++--- environment/docker.go | 6 +-- go.mod | 3 ++ go.sum | 41 ++++++++++++++++ loggers/cli/cli.go | 93 ++++++++++++++++++++++++++++++++++++ server/environment_docker.go | 22 ++++----- server/server.go | 10 ++-- server/state.go | 2 +- sftp/server.go | 9 ++-- 12 files changed, 226 insertions(+), 69 deletions(-) create mode 100644 loggers/cli/cli.go diff --git a/api/api.go b/api/api.go index d4f136f..a3bb98f 100644 --- a/api/api.go +++ b/api/api.go @@ -4,9 +4,9 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/apex/log" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "io/ioutil" "net/http" "strings" @@ -58,13 +58,11 @@ func (r *PanelRequest) logDebug(req *http.Request) { headers[k] = []string{v[0][0:15] + "(redacted)"} } - - zap.S().Debugw( - "making request to external HTTP endpoint", - zap.String("method", req.Method), - zap.String("endpoint", req.URL.String()), - zap.Any("headers", headers), - ) + log.WithFields(log.Fields{ + "method": req.Method, + "endpoint": req.URL.String(), + "headers": headers, + }).Debug("making request to external HTTP endpoint") } func (r *PanelRequest) Get(url string) (*http.Response, error) { diff --git a/cmd/root.go b/cmd/root.go index 144cc87..71c7656 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,7 +3,9 @@ package cmd import ( "crypto/tls" "fmt" + "github.com/apex/log" "github.com/mitchellh/colorstring" + "github.com/pterodactyl/wings/loggers/cli" "net/http" "os" "path" @@ -93,10 +95,10 @@ func rootCmdRun(*cobra.Command, []string) { panic(err) } - zap.S().Infof("using configuration from path: %s", c.GetPath()) + log.WithField("path", c.GetPath()).Info("loading configuration from path") if c.Debug { - zap.S().Debugw("running in debug mode") - zap.S().Infow("certificate checking is disabled") + log.Debug("running in debug mode") + log.Info("certificate checking is disabled") http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, @@ -107,42 +109,47 @@ func rootCmdRun(*cobra.Command, []string) { config.SetDebugViaFlag(debug) if err := c.System.ConfigureDirectories(); err != nil { - zap.S().Panicw("failed to configure system directories for pterodactyl", zap.Error(err)) - return + log.Fatal("failed to configure system directories for pterodactyl") + panic(err) } - zap.S().Infof("checking for pterodactyl system user \"%s\"", c.System.Username) + log.WithField("username", c.System.Username).Info("checking for pterodactyl system user") if su, err := c.EnsurePterodactylUser(); err != nil { - zap.S().Panicw("failed to create pterodactyl system user", zap.Error(err)) + log.Error("failed to create pterodactyl system user") + panic(err) return } else { - zap.S().Infow("configured system user", zap.String("username", su.Username), zap.String("uid", su.Uid), zap.String("gid", su.Gid)) + log.WithFields(log.Fields{ + "username": su.Username, + "uid": su.Uid, + "gid": su.Gid, + }).Info("configured system user successfully") } - zap.S().Infow("beginning file permission setting on server data directories") + log.Info("beginning file permission setting on server data directories") if err := c.EnsureFilePermissions(); err != nil { - zap.S().Errorw("failed to properly chown data directories", zap.Error(err)) + log.WithField("error", err).Error("failed to properly chown data directories") } else { - zap.S().Infow("finished ensuring file permissions") + log.Info("finished ensuring file permissions") } if err := server.LoadDirectory(); err != nil { - zap.S().Fatalw("failed to load server configurations", zap.Error(errors.WithStack(err))) + log.WithField("error", err).Fatal("failed to load server configurations") return } if err := environment.ConfigureDocker(&c.Docker); err != nil { - zap.S().Fatalw("failed to configure docker environment", zap.Error(errors.WithStack(err))) + log.WithField("error", err).Fatal("failed to configure docker environment") os.Exit(1) } if err := c.WriteToDisk(); err != nil { - zap.S().Errorw("failed to save configuration to disk", zap.Error(errors.WithStack(err))) + log.WithField("error", err).Error("failed to save configuration to disk") } // Just for some nice log output. for _, s := range server.GetServers().All() { - zap.S().Infow("loaded configuration for server", zap.String("server", s.Uuid)) + log.WithField("server", s.Uuid).Info("loaded configuration for server") } // Create a new WaitGroup that limits us to 4 servers being bootstrapped at a time @@ -154,18 +161,23 @@ func rootCmdRun(*cobra.Command, []string) { wg.Add() go func(s *server.Server) { - defer wg.Done() + // Required for tracing purposes. + var err error + + defer func() { + s.Log().Trace("ensuring server environment exists").Stop(&err) + wg.Done() + }() // Create a server environment if none exists currently. This allows us to recover from Docker // being reinstalled on the host system for example. - zap.S().Infow("ensuring environment exists", zap.String("server", s.Uuid)) - if err := s.Environment.Create(); err != nil { - zap.S().Errorw("failed to create an environment for server", zap.String("server", s.Uuid), zap.Error(err)) + if err = s.Environment.Create(); err != nil { + s.Log().WithField("error", err).Error("failed to process environment") } r, err := s.Environment.IsRunning() if err != nil { - zap.S().Errorw("error checking server environment status", zap.String("server", s.Uuid), zap.Error(err)) + s.Log().WithField("error", err).Error("error checking server environment status") } // If the server is currently running on Docker, mark the process as being in that state. @@ -175,13 +187,9 @@ func rootCmdRun(*cobra.Command, []string) { // This will also validate that a server process is running if the last tracked state we have // is that it was running, but we see that the container process is not currently running. if r || (!r && s.IsRunning()) { - zap.S().Infow("detected server is running, re-attaching to process", zap.String("server", s.Uuid)) + s.Log().Info("detected server is running, re-attaching to process...") if err := s.Environment.Start(); err != nil { - zap.S().Warnw( - "failed to properly start server detected as already running", - zap.String("server", s.Uuid), - zap.Error(errors.WithStack(err)), - ) + s.Log().WithField("error", errors.WithStack(err)).Warn("failed to properly start server detected as already running") } return @@ -201,26 +209,32 @@ func rootCmdRun(*cobra.Command, []string) { // Ensure the archive directory exists. if err := os.MkdirAll(c.System.ArchiveDirectory, 0755); err != nil { - zap.S().Errorw("failed to create archive directory", zap.Error(err)) + log.WithField("error", err).Error("failed to create archive directory") } // Ensure the backup directory exists. if err := os.MkdirAll(c.System.BackupDirectory, 0755); err != nil { - zap.S().Errorw("failed to create backup directory", zap.Error(err)) + log.WithField("error", err).Error("failed to create backup directory") } - zap.S().Infow("configuring webserver", zap.Bool("ssl", c.Api.Ssl.Enabled), zap.String("host", c.Api.Host), zap.Int("port", c.Api.Port)) + log.WithFields(log.Fields{ + "ssl": c.Api.Ssl.Enabled, + "host": c.Api.Host, + "port": c.Api.Port, + }).Info("configuring webserver...") r := router.Configure() addr := fmt.Sprintf("%s:%d", c.Api.Host, c.Api.Port) if c.Api.Ssl.Enabled { if err := r.RunTLS(addr, c.Api.Ssl.CertificateFile, c.Api.Ssl.KeyFile); err != nil { - zap.S().Fatalw("failed to configure HTTPS server", zap.Error(err)) + log.WithField("error", err).Fatal("failed to configure HTTPS server") + os.Exit(1) } } else { if err := r.Run(addr); err != nil { - zap.S().Fatalw("failed to configure HTTP server", zap.Error(err)) + log.WithField("error", err).Fatal("failed to configure HTTP server") + os.Exit(1) } } } @@ -250,6 +264,9 @@ func configureLogging(debug bool) error { zap.ReplaceGlobals(logger) + log.SetHandler(cli.Default) + log.SetLevel(log.DebugLevel) + return nil } diff --git a/config/config.go b/config/config.go index 2490a27..755de18 100644 --- a/config/config.go +++ b/config/config.go @@ -3,10 +3,10 @@ package config import ( "errors" "fmt" + "github.com/apex/log" "github.com/cobaugh/osrelease" "github.com/creasty/defaults" "github.com/gbrlsnchs/jwt/v3" - "go.uber.org/zap" "gopkg.in/yaml.v2" "io/ioutil" "os" @@ -296,7 +296,7 @@ func (c *Configuration) EnsureFilePermissions() error { gid, _ := strconv.Atoi(su.Gid) if err := os.Chown(path.Join(c.System.Data, f.Name()), uid, gid); err != nil { - zap.S().Warnw("failed to chown server directory", zap.String("directory", f.Name()), zap.Error(err)) + log.WithField("error", err).WithField("directory", f.Name()).Warn("failed to chown server directory") } }(file) } diff --git a/config/config_system.go b/config/config_system.go index 968988b..ef252f7 100644 --- a/config/config_system.go +++ b/config/config_system.go @@ -1,7 +1,7 @@ package config import ( - "go.uber.org/zap" + "github.com/apex/log" "os" "path" ) @@ -57,27 +57,27 @@ type SystemConfiguration struct { // Ensures that all of the system directories exist on the system. These directories are // created so that only the owner can read the data, and no other users. func (sc *SystemConfiguration) ConfigureDirectories() error { - zap.S().Debugw("ensuring root data directory exists", zap.String("path", sc.RootDirectory)) + log.WithField("path", sc.RootDirectory).Debug("ensuring root data directory exists") if err := os.MkdirAll(sc.RootDirectory, 0700); err != nil { return err } - zap.S().Debugw("ensuring log directory exists", zap.String("path", sc.LogDirectory)) + log.WithField("path", sc.LogDirectory).Debug("ensuring log directory exists") if err := os.MkdirAll(path.Join(sc.LogDirectory, "/install"), 0700); err != nil { return err } - zap.S().Debugw("ensuring server data directory exists", zap.String("path", sc.Data)) + log.WithField("path", sc.Data).Debug("ensuring server data directory exists") if err := os.MkdirAll(sc.Data, 0700); err != nil { return err } - zap.S().Debugw("ensuring archive data directory exists", zap.String("path", sc.ArchiveDirectory)) + log.WithField("path", sc.ArchiveDirectory).Debug("ensuring archive data directory exists") if err := os.MkdirAll(sc.ArchiveDirectory, 0700); err != nil { return err } - zap.S().Debugw("ensuring backup data directory exists", zap.String("path", sc.BackupDirectory)) + log.WithField("path", sc.BackupDirectory).Debug("ensuring backup data directory exists") if err := os.MkdirAll(sc.BackupDirectory, 0700); err != nil { return err } diff --git a/environment/docker.go b/environment/docker.go index 8083a06..c2a4e3e 100644 --- a/environment/docker.go +++ b/environment/docker.go @@ -2,12 +2,12 @@ package environment import ( "context" + "github.com/apex/log" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" ) // Configures the required network for the docker environment. @@ -20,10 +20,10 @@ func ConfigureDocker(c *config.DockerConfiguration) error { resource, err := cli.NetworkInspect(context.Background(), c.Network.Name, types.NetworkInspectOptions{}) if err != nil && client.IsErrNotFound(err) { - zap.S().Infow("creating missing pterodactyl0 interface, this could take a few seconds...") + log.Info("creating missing pterodactyl0 interface, this could take a few seconds...") return createDockerNetwork(cli, c) } else if err != nil { - zap.S().Fatalw("failed to create required docker network for containers", zap.Error(err)) + log.WithField("error", err).Fatal("failed to create required docker network for containers") } switch resource.Driver { diff --git a/go.mod b/go.mod index 591753a..2f5b4ca 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/Jeffail/gabs/v2 v2.2.0 github.com/Microsoft/go-winio v0.4.7 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/apex/log v1.3.0 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a github.com/aws/aws-sdk-go v1.30.14 // indirect github.com/beevik/etree v1.1.0 @@ -27,6 +28,7 @@ require ( github.com/docker/docker v0.0.0-20180422163414-57142e89befe github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.3.3 // indirect + github.com/fatih/color v1.9.0 github.com/gabriel-vasile/mimetype v0.1.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0 github.com/ghodss/yaml v1.0.0 @@ -40,6 +42,7 @@ require ( github.com/imdario/mergo v0.3.8 github.com/klauspost/pgzip v1.2.3 github.com/magiconair/properties v1.8.1 + github.com/mattn/go-colorable v0.1.4 github.com/mattn/go-shellwords v1.0.10 // indirect github.com/mholt/archiver/v3 v3.3.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db diff --git a/go.sum b/go.sum index 89d3e3e..c63597c 100644 --- a/go.sum +++ b/go.sum @@ -18,11 +18,19 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 h1:bZ28Hqta7TFAK3Q08CMvv8y3/8ATaEqv2nGoc6yff6c= github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U= +github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA= +github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs= +github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.14 h1:vZfX2b/fknc9wKcytbLWykM7in5k6dbQ8iHTJDUP1Ng= github.com/aws/aws-sdk-go v1.30.14/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -59,6 +67,10 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v0.1.4 h1:5mcsq3+DXypREUkW+1juhjeKmE/XnWgs+paHMJn7lf8= @@ -73,6 +85,7 @@ github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM= github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -121,6 +134,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8= github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/icza/dyno v0.0.0-20200205103839-49cb13720835 h1:f1irK5f03uGGj+FjgQfZ5VhdKNVQVJ4skHsedzVohQ4= @@ -129,9 +143,11 @@ github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -157,6 +173,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -169,9 +186,14 @@ github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= @@ -197,6 +219,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -238,17 +262,22 @@ github.com/pterodactyl/sftp-server v1.1.2/go.mod h1:KjSONrenRr1oCh94QIVAU6yEzMe+ github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0= github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -269,6 +298,11 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber-go/zap v1.9.1/go.mod h1:GY+83l3yxBcBw2kmHu/sAWwItnTn+ynxHCRo+WiIQOY= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -312,6 +346,7 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -336,6 +371,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -360,12 +396,14 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03i golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY= @@ -413,12 +451,15 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/loggers/cli/cli.go b/loggers/cli/cli.go new file mode 100644 index 0000000..8e82227 --- /dev/null +++ b/loggers/cli/cli.go @@ -0,0 +1,93 @@ +package cli + +import ( + "fmt" + "github.com/apex/log" + "github.com/apex/log/handlers/cli" + color2 "github.com/fatih/color" + "github.com/mattn/go-colorable" + "github.com/pkg/errors" + "io" + "os" + "sync" + "time" +) + +var Default = New(os.Stderr) + +var bold = color2.New(color2.Bold) + +var Strings = [...]string{ + log.DebugLevel: "DEBUG", + log.InfoLevel: " INFO", + log.WarnLevel: " WARN", + log.ErrorLevel: "ERROR", + log.FatalLevel: "FATAL", +} + +type Handler struct { + mu sync.Mutex + Writer io.Writer + Padding int +} + +func New(w io.Writer) *Handler { + if f, ok := w.(*os.File); ok { + return &Handler{Writer: colorable.NewColorable(f), Padding: 2} + } + + return &Handler{Writer: w, Padding: 2} +} + +type tracer interface { + StackTrace() errors.StackTrace +} + +// HandleLog implements log.Handler. +func (h *Handler) HandleLog(e *log.Entry) error { + color := cli.Colors[e.Level] + level := Strings[e.Level] + names := e.Fields.Names() + + h.mu.Lock() + defer h.mu.Unlock() + + color.Fprintf(h.Writer, "%s: [%s] %-25s", bold.Sprintf("%*s", h.Padding+1, level), time.Now().Format(time.StampMilli), e.Message) + + for _, name := range names { + if name == "source" { + continue + } + + fmt.Fprintf(h.Writer, " %s=%v", color.Sprint(name), e.Fields.Get(name)) + } + + fmt.Fprintln(h.Writer) + + for _, name := range names { + if name != "error" { + continue + } + + if err, ok := e.Fields.Get("error").(error); ok { + if e, ok := errors.Cause(err).(tracer); ok { + st := e.StackTrace() + + l := len(st) + if l > 5 { + l = 5 + } + + br := color2.New(color2.Bold, color2.FgRed) + + fmt.Fprintf(h.Writer, "\n%s%+v\n\n", br.Sprintf("Stacktrace:"), st[0:l]) + } else { + fmt.Printf("\n\nINVALID TRACER\n\n") + } + } else { + fmt.Printf("\n\nINVALID ERROR\n\n") + } + } + + return nil +} diff --git a/server/environment_docker.go b/server/environment_docker.go index adf38bd..afccf29 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/apex/log" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" @@ -15,7 +16,6 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "io" "os" "strconv" @@ -141,7 +141,7 @@ func (d *DockerEnvironment) InSituUpdate() error { // state. This ensures that unexpected container deletion while Wings is running does // not result in the server becoming unbootable. func (d *DockerEnvironment) OnBeforeStart() error { - zap.S().Infow("syncing server configuration with Panel", zap.String("server", d.Server.Uuid)) + d.Server.Log().Info("syncing server configuration with panel") if err := d.Server.Sync(); err != nil { return err } @@ -422,7 +422,7 @@ func (d *DockerEnvironment) Attach() error { d.attached = true go func() { if err := d.EnableResourcePolling(); err != nil { - zap.S().Warnw("failed to enabled resource polling on server", zap.String("server", d.Server.Uuid), zap.Error(errors.WithStack(err))) + d.Server.Log().WithField("error", errors.WithStack(err)).Warn("failed to enable resource polling on server") } }() @@ -470,7 +470,7 @@ func (d *DockerEnvironment) FollowConsoleOutput() error { } if err := s.Err(); err != nil { - zap.S().Warnw("error processing scanner line in console output", zap.String("server", d.Server.Uuid), zap.Error(err)) + d.Server.Log().WithField("error", err).Warn("error processing scanner line in console output") } }(reader) @@ -500,7 +500,7 @@ func (d *DockerEnvironment) EnableResourcePolling() error { if err := dec.Decode(&v); err != nil { if err != io.EOF { - zap.S().Warnw("encountered error processing server stats; stopping collection", zap.Error(err)) + d.Server.Log().WithField("error", err).Warn("encountered error processing server stats, stopping collection") } d.DisableResourcePolling() @@ -576,12 +576,10 @@ func (d *DockerEnvironment) ensureImageExists(c *client.Client) error { for _, img := range images { for _, t := range img.RepoTags { if t == d.Server.Container.Image { - zap.S().Warnw( - "unable to pull requested image from remote source, however the image exists locally", - zap.String("server", d.Server.Uuid), - zap.String("image", d.Server.Container.Image), - zap.Error(err), - ) + d.Server.Log().WithFields(log.Fields{ + "image": d.Server.Container.Image, + "error": errors.New(err.Error()), + }).Warn("unable to pull requested image from remote source, however the image exists locally") // Okay, we found a matching container image, in that case just go ahead and return // from this function, since there is nothing else we need to do here. @@ -594,7 +592,7 @@ func (d *DockerEnvironment) ensureImageExists(c *client.Client) error { } defer out.Close() - zap.S().Debugw("pulling docker image... this could take a bit of time", zap.String("image", d.Server.Container.Image)) + log.WithField("image", d.Server.Container.Image).Debug("pulling docker image... this could take a bit of time") // I'm not sure what the best approach here is, but this will block execution until the image // is done being pulled, which is what we need. diff --git a/server/server.go b/server/server.go index 82deffe..b510185 100644 --- a/server/server.go +++ b/server/server.go @@ -2,13 +2,13 @@ package server import ( "fmt" + "github.com/apex/log" "github.com/creasty/defaults" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" "github.com/remeh/sizedwaitgroup" - "go.uber.org/zap" "math" "os" "strings" @@ -194,13 +194,13 @@ func LoadDirectory() error { s, err := FromConfiguration(data) if err != nil { - zap.S().Errorw("failed to load server, skipping...", zap.String("server", uuid), zap.Error(err)) + log.WithField("server", uuid).WithField("error", err).Error("failed to load server, skipping...") return } if state, exists := states[s.Uuid]; exists { s.SetState(state) - zap.S().Debugw("loaded server state from cache", zap.String("server", s.Uuid), zap.String("state", s.GetState())) + s.Log().WithField("state", s.GetState()).Debug("loaded server state from cache file") } servers.Add(s) @@ -282,6 +282,10 @@ eloop: return out } +func (s *Server) Log() *log.Entry { + return log.WithField("server", s.Uuid) +} + // Syncs the state of the server on the Panel with Wings. This ensures that we're always // using the state of the server from the Panel and allows us to not require successful // API calls to Wings to do things. diff --git a/server/state.go b/server/state.go index a4d8a36..39648f7 100644 --- a/server/state.go +++ b/server/state.go @@ -82,7 +82,7 @@ func (s *Server) SetState(state string) error { s.State = state // Emit the event to any listeners that are currently registered. - zap.S().Debugw("saw server status change event", zap.String("server", s.Uuid), zap.String("status", s.State)) + s.Log().WithField("status", s.State).Debug("saw server status change event") s.Events().Publish(StatusEvent, s.State) // Release the lock as it is no longer needed for the following actions. diff --git a/sftp/server.go b/sftp/server.go index 54d8012..c554ce9 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -1,6 +1,7 @@ package sftp import ( + "github.com/apex/log" "github.com/pkg/errors" "github.com/pterodactyl/sftp-server" "github.com/pterodactyl/wings/api" @@ -38,7 +39,7 @@ func Initialize(config *config.Configuration) error { // a long running operation. go func(instance *sftp_server.Server) { if err := c.Initalize(); err != nil { - zap.S().Named("sftp").Errorw("failed to initialize SFTP subsystem", zap.Error(errors.WithStack(err))) + log.WithField("subsystem", "sftp").WithField("error", errors.WithStack(err)).Error("failed to initialize SFTP subsystem") } }(c) @@ -73,7 +74,8 @@ func validateDiskSpace(fs sftp_server.FileSystem) bool { // the server's UUID if the credentials were valid. func validateCredentials(c sftp_server.AuthenticationRequest) (*sftp_server.AuthenticationResponse, error) { resp, err := api.NewRequester().ValidateSftpCredentials(c) - zap.S().Named("sftp").Debugw("validating credentials for SFTP connection", zap.String("username", c.User)) + + log.WithFields(log.Fields{"subsystem": "sftp", "username": c.User}).Debug("validating credentials for SFTP connection") if err != nil { return resp, err } @@ -86,6 +88,7 @@ func validateCredentials(c sftp_server.AuthenticationRequest) (*sftp_server.Auth return resp, errors.New("no matching server with UUID found") } - zap.S().Named("sftp").Debugw("matched user to server instance, credentials successfully validated", zap.String("username", c.User), zap.String("server", s.Uuid)) + s.Log().WithFields(log.Fields{"subsystem": "sftp", "username": c.User}).Debug("matched user to server instance, credentials successfully validated") + return resp, err } From 359564bd9174c312d2adbd04e9d16901295638b3 Mon Sep 17 00:00:00 2001 From: Carlo Field Date: Fri, 29 May 2020 17:44:49 +0200 Subject: [PATCH 07/30] #2078 - fix BindJSON calls --- router/router_server.go | 14 +++++++++++--- router/router_server_backup.go | 6 ++++-- router/router_server_files.go | 20 ++++++++++++++++---- router/router_system.go | 7 +++++-- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/router/router_server.go b/router/router_server.go index 0412d7d..c866495 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -46,7 +46,10 @@ func postServerPower(c *gin.Context) { s := GetServer(c.Param("server")) var data server.PowerAction - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } if !data.IsValid() { c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{ @@ -98,8 +101,13 @@ func postServerCommands(c *gin.Context) { return } - var data struct{ Commands []string `json:"commands"` } - c.BindJSON(&data) + var data struct { + Commands []string `json:"commands"` + } + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } for _, command := range data.Commands { if err := s.Environment.SendCommand(command); err != nil { diff --git a/router/router_server_backup.go b/router/router_server_backup.go index 2928b3c..a154542 100644 --- a/router/router_server_backup.go +++ b/router/router_server_backup.go @@ -15,7 +15,10 @@ func postServerBackup(c *gin.Context) { s := GetServer(c.Param("server")) data := &backup.Request{} - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } var adapter backup.BackupInterface var err error @@ -41,7 +44,6 @@ func postServerBackup(c *gin.Context) { } }(adapter, s) - c.Status(http.StatusAccepted) } diff --git a/router/router_server_files.go b/router/router_server_files.go index f0b28e6..09f1d3d 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -89,7 +89,10 @@ func putServerRenameFile(c *gin.Context) { RenameFrom string `json:"rename_from"` RenameTo string `json:"rename_to"` } - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } if data.RenameFrom == "" || data.RenameTo == "" { c.AbortWithStatusJSON(http.StatusUnprocessableEntity, gin.H{ @@ -113,7 +116,10 @@ func postServerCopyFile(c *gin.Context) { var data struct { Location string `json:"location"` } - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } if err := s.Filesystem.Copy(data.Location); err != nil { TrackedServerError(err, s).AbortWithServerError(c) @@ -130,7 +136,10 @@ func postServerDeleteFile(c *gin.Context) { var data struct { Location string `json:"location"` } - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } if err := s.Filesystem.Delete(data.Location); err != nil { TrackedServerError(err, s).AbortWithServerError(c) @@ -167,7 +176,10 @@ func postServerCreateDirectory(c *gin.Context) { Name string `json:"name"` Path string `json:"path"` } - c.BindJSON(&data) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&data); err != nil { + return + } if err := s.Filesystem.CreateDirectory(data.Name, data.Path); err != nil { TrackedServerError(err, s).AbortWithServerError(c) diff --git a/router/router_system.go b/router/router_system.go index 085f383..07553f6 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -77,7 +77,10 @@ func postUpdateConfiguration(c *gin.Context) { // A copy of the configuration we're using to bind the data recevied into. cfg := *config.Get() - c.BindJSON(&cfg) + // BindJSON sends 400 if the request fails, all we need to do is return + if err := c.BindJSON(&cfg); err != nil { + return + } config.Set(&cfg) if err := config.Get().WriteToDisk(); err != nil { @@ -90,4 +93,4 @@ func postUpdateConfiguration(c *gin.Context) { } c.Status(http.StatusNoContent) -} \ No newline at end of file +} From 0ae286d617896152541dc14311298b4cb323257a Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 31 May 2020 12:42:10 -0700 Subject: [PATCH 08/30] Correctly handle empty values from the API requests; mergo by default thinks these "empty" values should be skipped --- server/environment_docker.go | 4 +++- server/update.go | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/environment_docker.go b/server/environment_docker.go index afccf29..00324ac 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -122,11 +122,13 @@ func (d *DockerEnvironment) InSituUpdate() error { return errors.WithStack(err) } + ctx, _ := context.WithTimeout(context.Background(), time.Second * 10) u := container.UpdateConfig{ Resources: d.getResourcesForServer(), } - if _, err := d.Client.ContainerUpdate(context.Background(), d.Server.Uuid, u); err != nil { + d.Server.Log().WithField("limits", fmt.Sprintf("%+v", u.Resources)).Debug("updating server container on-the-fly with passed limits") + if _, err := d.Client.ContainerUpdate(ctx, d.Server.Uuid, u); err != nil { return errors.WithStack(err) } diff --git a/server/update.go b/server/update.go index 741bcf8..f62c89f 100644 --- a/server/update.go +++ b/server/update.go @@ -34,6 +34,16 @@ func (s *Server) UpdateDataStructure(data []byte, background bool) error { return errors.WithStack(err) } + // Don't explode if we're setting CPU limits to 0. Mergo sees that as an empty value + // so it won't override the value we've passed through in the API call. However, we can + // safely assume that we're passing through valid data structures here. I foresee this + // backfiring at some point, but until then... + // + // We'll go ahead and do this with swap as well. + s.Build.CpuLimit = src.Build.CpuLimit + s.Build.Swap = src.Build.Swap + s.Build.DiskSpace = src.Build.DiskSpace + // Mergo can't quite handle this boolean value correctly, so for now we'll just // handle this edge case manually since none of the other data passed through in this // request is going to be boolean. Allegedly. @@ -81,12 +91,9 @@ func (s *Server) runBackgroundActions() { // Update the environment in place, allowing memory and CPU usage to be adjusted // on the fly without the user needing to reboot (theoretically). go func(server *Server) { + server.Log().Info("performing server limit modification on-the-fly") if err := server.Environment.InSituUpdate(); err != nil { - zap.S().Warnw( - "failed to perform in-situ update of server environment", - zap.String("server", server.Uuid), - zap.Error(err), - ) + server.Log().WithField("error", err).Warn("failed to perform on-the-fly update of the server environment") } }(s) From 5c2686fc6d2ae93ed5eaed8c9d80443de5e46b62 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Wed, 10 Jun 2020 15:00:59 -0600 Subject: [PATCH 09/30] Fix multiple server status messages --- router/websocket/websocket.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index 4a329b6..2f85f65 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -218,15 +218,28 @@ func (h *Handler) HandleInbound(m Message) error { return err } - if token.HasPermission(PermissionConnect) { - h.setJwt(token) - } + // Check if the user has previously authenticated successfully. + newConnection := h.GetJwt() == nil + // Previously there was a HasPermission(PermissionConnect) check around this, + // however NewTokenPayload will return an error if it doesn't have the connect + // permission meaning that it was a redundant function call. + h.setJwt(token) + + // Tell the client they authenticated successfully. h.unsafeSendJson(Message{ Event: AuthenticationSuccessEvent, Args: []string{}, }) + // Check if the client was refreshing their authentication token + // instead of authenticating for the first time. + if !newConnection { + // This prevents duplicate status messages as outlined in + // https://github.com/pterodactyl/panel/issues/2077 + return nil + } + // On every authentication event, send the current server status back // to the client. :) state := h.server.GetState() From 198a22f446f749ccd502280c95178fac2b26c15a Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 13 Jun 2020 10:26:35 -0700 Subject: [PATCH 10/30] Replace even more zap occurences --- installer/installer.go | 21 +++++++----- parser/helpers.go | 16 +++------ parser/parser.go | 14 ++++---- router/error.go | 22 ++++++------- router/router_server.go | 21 ++++-------- router/router_server_ws.go | 4 +-- router/router_system.go | 8 ++--- router/websocket/websocket.go | 14 +++----- server/config_parser.go | 7 ++-- server/crash.go | 7 ++-- server/install.go | 62 ++++++++++++----------------------- server/listeners.go | 9 ++--- 12 files changed, 80 insertions(+), 125 deletions(-) diff --git a/installer/installer.go b/installer/installer.go index 9f7f868..b2fdf09 100644 --- a/installer/installer.go +++ b/installer/installer.go @@ -2,13 +2,13 @@ package installer import ( "encoding/json" + "github.com/apex/log" "github.com/asaskevich/govalidator" "github.com/buger/jsonparser" "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/server" - "go.uber.org/zap" "os" "path" ) @@ -108,24 +108,27 @@ func (i *Installer) Server() *server.Server { // associated installation process based on the parameters passed through for // the server instance. func (i *Installer) Execute() { - zap.S().Debugw("creating required server data directory", zap.String("server", i.Uuid())) - if err := os.MkdirAll(path.Join(config.Get().System.Data, i.Uuid()), 0755); err != nil { - zap.S().Errorw("failed to create server data directory", zap.String("server", i.Uuid()), zap.Error(errors.WithStack(err))) + p := path.Join(config.Get().System.Data, i.Uuid()) + l := log.WithFields(log.Fields{"server": i.Uuid(), "process": "installer"}) + + l.WithField("path", p).Debug("creating required server data directory") + if err := os.MkdirAll(p, 0755); err != nil { + l.WithFields(log.Fields{"path": p, "error": errors.WithStack(err)}).Error("failed to create server data directory") return } - if err := os.Chown(path.Join(config.Get().System.Data, i.Uuid()), config.Get().System.User.Uid, config.Get().System.User.Gid); err != nil { - zap.S().Errorw("failed to chown server data directory", zap.String("server", i.Uuid()), zap.Error(errors.WithStack(err))) + if err := os.Chown(p, config.Get().System.User.Uid, config.Get().System.User.Gid); err != nil { + l.WithField("error", errors.WithStack(err)).Error("failed to chown server data directory") return } - zap.S().Debugw("creating required environment for server instance", zap.String("server", i.Uuid())) + l.Debug("creating required environment for server instance") if err := i.server.Environment.Create(); err != nil { - zap.S().Errorw("failed to create environment for server", zap.String("server", i.Uuid()), zap.Error(err)) + l.WithField("error", err).Error("failed to create environment for server") return } - zap.S().Debugw("created environment for server during install process", zap.String("server", i.Uuid())) + l.Info("successfully created environment for server during install process") } // Returns a string value from the JSON data provided. diff --git a/parser/helpers.go b/parser/helpers.go index a7c4fc8..d66d2b0 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -3,10 +3,10 @@ package parser import ( "bytes" "github.com/Jeffail/gabs/v2" + "github.com/apex/log" "github.com/buger/jsonparser" "github.com/iancoleman/strcase" "github.com/pkg/errors" - "go.uber.org/zap" "io/ioutil" "os" "regexp" @@ -120,11 +120,9 @@ func (cfr *ConfigurationFileReplacement) SetAtPathway(c *gabs.Container, path st // We're doing some regex here. r, err := regexp.Compile(strings.TrimPrefix(cfr.IfValue, "regex:")) if err != nil { - zap.S().Warnw( - "configuration if_value using invalid regexp, cannot do replacement", - zap.String("if_value", strings.TrimPrefix(cfr.IfValue, "regex:")), - zap.Error(err), - ) + log.WithFields(log.Fields{"if_value": strings.TrimPrefix(cfr.IfValue, "regex:"), "error": err}). + Warn("configuration if_value using invalid regexp, cannot perform replacement") + return nil } @@ -179,11 +177,7 @@ func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplac return match, errors.WithStack(err) } - zap.S().Debugw( - "attempted to load a configuration value that does not exist", - zap.Strings("path", path), - zap.String("filename", f.FileName), - ) + log.WithFields(log.Fields{"path": path, "filename": f.FileName}).Debug("attempted to load a configuration value that does not exist") // If there is no key, keep the original value intact, that way it is obvious there // is a replace issue at play. diff --git a/parser/parser.go b/parser/parser.go index 71177b7..af2aef4 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -3,13 +3,13 @@ package parser import ( "bufio" "encoding/json" + "github.com/apex/log" "github.com/beevik/etree" "github.com/buger/jsonparser" "github.com/icza/dyno" "github.com/magiconair/properties" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "gopkg.in/ini.v1" "gopkg.in/yaml.v2" "io/ioutil" @@ -30,6 +30,10 @@ const ( type ConfigurationParser string +func (cp ConfigurationParser) String() string { + return string(cp) +} + // Defines a configuration file for the server startup. These will be looped over // and modified before the server finishes booting. type ConfigurationFile struct { @@ -63,11 +67,7 @@ func (f *ConfigurationFile) UnmarshalJSON(data []byte) error { } if err := json.Unmarshal(*m["replace"], &f.Replace); err != nil { - zap.S().Warnw( - "failed to unmarshal configuration file replacement", - zap.String("file", f.FileName), - zap.Error(err), - ) + log.WithField("file", f.FileName).WithField("error", err).Warn("failed to unmarshal configuration file replacement") f.Replace = []ConfigurationFileReplacement{} } @@ -131,7 +131,7 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error { // Parses a given configuration file and updates all of the values within as defined // in the API response from the Panel. func (f *ConfigurationFile) Parse(path string, internal bool) error { - zap.S().Debugw("parsing configuration file", zap.String("path", path), zap.String("parser", string(f.Parser))) + log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file") if mb, err := json.Marshal(config.Get()); err != nil { return err diff --git a/router/error.go b/router/error.go index 9097b6e..aaecb04 100644 --- a/router/error.go +++ b/router/error.go @@ -2,11 +2,11 @@ package router import ( "fmt" + "github.com/apex/log" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/pkg/errors" "github.com/pterodactyl/wings/server" - "go.uber.org/zap" "net/http" "os" ) @@ -40,6 +40,14 @@ func TrackedServerError(err error, s *server.Server) *RequestError { } } +func (e *RequestError) logger() *log.Entry { + if e.server != nil { + return e.server.Log().WithField("error_id", e.Uuid) + } + + return log.WithField("error_id", e.Uuid) +} + // Sets the output message to display to the user in the error. func (e *RequestError) SetMessage(msg string) *RequestError { e.Message = msg @@ -61,19 +69,11 @@ func (e *RequestError) AbortWithStatus(status int, c *gin.Context) { // Otherwise, log the error to zap, and then report the error back to the user. if status >= 500 { - if e.server != nil { - zap.S().Errorw("encountered error while handling HTTP request", zap.String("server", e.server.Uuid), zap.String("error_id", e.Uuid), zap.Error(e.Err)) - } else { - zap.S().Errorw("encountered error while handling HTTP request", zap.String("error_id", e.Uuid), zap.Error(e.Err)) - } + e.logger().WithField("error", e.Err).Error("encountered HTTP/500 error while handling request") c.Error(errors.WithStack(e)) } else { - if e.server != nil { - zap.S().Debugw("encountered error while handling HTTP request", zap.String("server", e.server.Uuid), zap.String("error_id", e.Uuid), zap.Error(e.Err)) - } else { - zap.S().Debugw("encountered error while handling HTTP request", zap.String("error_id", e.Uuid), zap.Error(e.Err)) - } + e.logger().WithField("error", e.Err).Debug("encountered non-HTTP/500 error while handling request") } msg := "An unexpected error was encountered while processing this request." diff --git a/router/router_server.go b/router/router_server.go index c866495..8018add 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -2,6 +2,7 @@ package router import ( "bytes" + "github.com/apex/log" "github.com/gin-gonic/gin" "github.com/pkg/errors" "github.com/pterodactyl/wings/server" @@ -74,15 +75,12 @@ func postServerPower(c *gin.Context) { // Pass the actual heavy processing off to a seperate thread to handle so that // we can immediately return a response from the server. Some of these actions // can take quite some time, especially stopping or restarting. - go func() { - if err := s.HandlePowerAction(data); err != nil { - zap.S().Errorw( - "encountered an error processing a server power action", - zap.String("server", s.Uuid), - zap.Error(err), - ) + go func(server *server.Server) { + if err := server.HandlePowerAction(data); err != nil { + server.Log().WithFields(log.Fields{"action": data, "error": err}). + Error("encountered error processing a server power action in the background") } - }() + }(s) c.Status(http.StatusAccepted) } @@ -111,12 +109,7 @@ func postServerCommands(c *gin.Context) { for _, command := range data.Commands { if err := s.Environment.SendCommand(command); err != nil { - zap.S().Warnw( - "failed to send command to server", - zap.String("server", s.Uuid), - zap.String("command", command), - zap.Error(err), - ) + s.Log().WithFields(log.Fields{"command": command, "error": err}).Warn("failed to send command to server instance") } } diff --git a/router/router_server_ws.go b/router/router_server_ws.go index 7fe476a..4f60f93 100644 --- a/router/router_server_ws.go +++ b/router/router_server_ws.go @@ -6,7 +6,6 @@ import ( "github.com/gin-gonic/gin" ws "github.com/gorilla/websocket" "github.com/pterodactyl/wings/router/websocket" - "go.uber.org/zap" ) // Upgrades a connection to a websocket and passes events along between. @@ -40,7 +39,7 @@ func getServerWebsocket(c *gin.Context) { ws.CloseServiceRestart, ws.CloseAbnormalClosure, ) { - zap.S().Warnw("error handling websocket message", zap.Error(err)) + s.Log().WithField("error", err).Warn("error handling websocket message for server") } break } @@ -57,4 +56,3 @@ func getServerWebsocket(c *gin.Context) { } } } - diff --git a/router/router_system.go b/router/router_system.go index 07553f6..f5dfed3 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -2,12 +2,12 @@ package router import ( "bytes" + "github.com/apex/log" "github.com/gin-gonic/gin" "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/installer" "github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/system" - "go.uber.org/zap" "net/http" ) @@ -59,11 +59,7 @@ func postCreateServer(c *gin.Context) { i.Execute() if err := i.Server().Install(); err != nil { - zap.S().Errorw( - "failed to run install process for server", - zap.String("server", i.Uuid()), - zap.Error(err), - ) + log.WithFields(log.Fields{"server": i.Uuid(), "error": err}).Error("failed to run install process for server") } }(install) diff --git a/router/websocket/websocket.go b/router/websocket/websocket.go index 4a329b6..674b188 100644 --- a/router/websocket/websocket.go +++ b/router/websocket/websocket.go @@ -3,6 +3,7 @@ package websocket import ( "encoding/json" "fmt" + "github.com/apex/log" "github.com/gbrlsnchs/jwt/v3" "github.com/google/uuid" "github.com/gorilla/websocket" @@ -10,7 +11,6 @@ import ( "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/router/tokens" "github.com/pterodactyl/wings/server" - "go.uber.org/zap" "net/http" "os" "strings" @@ -85,7 +85,6 @@ func (h *Handler) SendJson(v *Message) error { // If we're sending installation output but the user does not have the required // permissions to see the output, don't send it down the line. if v.Event == server.InstallOutputEvent { - zap.S().Debugf("%+v", v.Args) if !j.HasPermission(PermissionReceiveInstall) { return nil } @@ -152,13 +151,8 @@ func (h *Handler) SendErrorJson(msg Message, err error) error { wsm.Args = []string{m} if !server.IsSuspendedError(err) { - zap.S().Errorw( - "an error was encountered in the websocket process", - zap.String("event", msg.Event), - zap.String("server", h.server.Uuid), - zap.String("error_identifier", u.String()), - zap.Error(err), - ) + h.server.Log().WithFields(log.Fields{"event": msg.Event, "error_identifier": u.String(), "error": err}). + Error("failed to handle websocket process; an error was encountered processing an event") } return h.unsafeSendJson(wsm) @@ -192,7 +186,7 @@ func (h *Handler) GetJwt() *tokens.WebsocketPayload { func (h *Handler) HandleInbound(m Message) error { if m.Event != AuthenticationEvent { if err := h.TokenValid(); err != nil { - zap.S().Debugw("jwt token is no longer valid", zap.String("message", err.Error())) + log.WithField("message", err.Error()).Debug("jwt for server websocket is no longer valid") h.unsafeSendJson(Message{ Event: ErrorEvent, diff --git a/server/config_parser.go b/server/config_parser.go index 76decf2..a0f1f11 100644 --- a/server/config_parser.go +++ b/server/config_parser.go @@ -2,7 +2,6 @@ package server import ( "github.com/pterodactyl/wings/parser" - "go.uber.org/zap" "sync" ) @@ -17,15 +16,15 @@ func (s *Server) UpdateConfigurationFiles() { go func(f parser.ConfigurationFile, server *Server) { defer wg.Done() - p, err := s.Filesystem.SafePath(f.FileName) + p, err := server.Filesystem.SafePath(f.FileName) if err != nil { - zap.S().Errorw("failed to generate safe path for configuration file", zap.String("server", server.Uuid), zap.Error(err)) + server.Log().WithField("error", err).Error("failed to generate safe path for configuration file") return } if err := f.Parse(p, false); err != nil { - zap.S().Errorw("failed to parse and update server configuration file", zap.String("server", server.Uuid), zap.Error(err)) + server.Log().WithField("error", err).Error("failed to parse and update server configuration file") } }(v, s) } diff --git a/server/crash.go b/server/crash.go index 0f08079..649580f 100644 --- a/server/crash.go +++ b/server/crash.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "time" ) @@ -27,15 +26,13 @@ type CrashDetection struct { // // If the server is determined to have crashed, the process will be restarted and the // counter for the server will be incremented. -// -// @todo output event to server console func (s *Server) handleServerCrash() error { // No point in doing anything here if the server isn't currently offline, there // is no reason to do a crash detection event. If the server crash detection is // disabled we want to skip anything after this as well. if s.GetState() != ProcessOfflineState || !s.CrashDetection.Enabled { if !s.CrashDetection.Enabled { - zap.S().Debugw("server triggered crash detection but handler is disabled for server process", zap.String("server", s.Uuid)) + s.Log().Debug("server triggered crash detection but handler is disabled for server process") s.PublishConsoleOutputFromDaemon("Server detected as crashed; crash detection is disabled for this instance.") } @@ -51,7 +48,7 @@ func (s *Server) handleServerCrash() error { // If the system is not configured to detect a clean exit code as a crash, and the // crash is not the result of the program running out of memory, do nothing. if exitCode == 0 && !oomKilled && !config.Get().System.DetectCleanExitAsCrash { - zap.S().Debugw("server exited with successful code; system configured to not detect as crash", zap.String("server", s.Uuid)) + s.Log().Debug("server exited with successful exit code; system is configured to not detect this as a crash") return nil } diff --git a/server/install.go b/server/install.go index 324ced3..efa6c92 100644 --- a/server/install.go +++ b/server/install.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "github.com/apex/log" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" @@ -11,7 +12,6 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "io" "io/ioutil" "os" @@ -25,14 +25,12 @@ import ( func (s *Server) Install() error { err := s.internalInstall() - zap.S().Debugw("notifying panel of server install state", zap.String("server", s.Uuid)) + s.Log().Debug("notifying panel of server install state") if serr := s.SyncInstallState(err == nil); serr != nil { - zap.S().Warnw( - "failed to notify panel of server install state", - zap.String("server", s.Uuid), - zap.Bool("was_successful", err == nil), - zap.Error(serr), - ) + s.Log().WithFields(log.Fields{ + "was_successful": err == nil, + "error": serr, + }).Warn("failed to notify panel of server install state") } return err @@ -42,7 +40,7 @@ func (s *Server) Install() error { // does not touch any existing files for the server, other than what the script modifies. func (s *Server) Reinstall() error { if s.GetState() != ProcessOfflineState { - zap.S().Debugw("waiting for server instance to enter a stopped state", zap.String("server", s.Uuid)) + s.Log().Debug("waiting for server instance to enter a stopped state") if err := s.Environment.WaitForStop(10, true); err != nil { return err } @@ -67,14 +65,12 @@ func (s *Server) internalInstall() error { return errors.WithStack(err) } - zap.S().Infow("beginning installation process for server", zap.String("server", s.Uuid)) - + s.Log().Info("beginning installation process for server") if err := p.Run(); err != nil { return err } - zap.S().Infow("completed installation process for server", zap.String("server", s.Uuid)) - + s.Log().Info("completed installation process for server") return nil } @@ -106,13 +102,13 @@ func NewInstallationProcess(s *Server, script *api.InstallationScript) (*Install // Removes the installer container for the server. func (ip *InstallationProcess) RemoveContainer() { - err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid + "_installer", types.ContainerRemoveOptions{ + err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, }) if err != nil && !client.IsErrNotFound(err) { - zap.S().Warnw("failed to delete server installer container", zap.String("server", ip.Server.Uuid), zap.Error(errors.WithStack(err))) + ip.Server.Log().WithField("error", errors.WithStack(err)).Warn("failed to delete server install container") } } @@ -137,7 +133,7 @@ func (ip *InstallationProcess) Run() error { // If this step fails, log a warning but don't exit out of the process. This is completely // internal to the daemon's functionality, and does not affect the status of the server itself. if err := ip.AfterExecute(cid); err != nil { - zap.S().Warnw("failed to complete after-execute step of installation process", zap.String("server", ip.Server.Uuid), zap.Error(err)) + ip.Server.Log().WithField("error", err).Warn("failed to complete after-execute step of installation process") } return nil @@ -189,7 +185,7 @@ func (ip *InstallationProcess) pullInstallationImage() error { // Block continuation until the image has been pulled successfully. scanner := bufio.NewScanner(r) for scanner.Scan() { - zap.S().Debugw(scanner.Text()) + log.Debug(scanner.Text()) } if err := scanner.Err(); err != nil { @@ -265,7 +261,7 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { ctx := context.Background() defer ip.RemoveContainer() - zap.S().Debugw("pulling installation logs for server", zap.String("server", ip.Server.Uuid), zap.String("container_id", containerId)) + ip.Server.Log().WithField("container_id", containerId).Debug("pulling installation logs for server") reader, err := ip.client.ContainerLogs(ctx, containerId, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, @@ -295,12 +291,6 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { func (ip *InstallationProcess) Execute(installPath string) (string, error) { ctx := context.Background() - zap.S().Debugw( - "creating server installer container", - zap.String("server", ip.Server.Uuid), - zap.String("script_path", installPath+"/install.sh"), - ) - conf := &container.Config{ Hostname: "installer", AttachStdout: true, @@ -348,17 +338,13 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { NetworkMode: container.NetworkMode(config.Get().Docker.Network.Mode), } - zap.S().Infow("creating installer container for server process", zap.String("server", ip.Server.Uuid)) + ip.Server.Log().WithField("install_script", installPath+"/install.sh").Info("creating install container for server process") r, err := ip.client.ContainerCreate(ctx, conf, hostConf, nil, ip.Server.Uuid+"_installer") if err != nil { return "", errors.WithStack(err) } - zap.S().Infow( - "running installation script for server in container", - zap.String("server", ip.Server.Uuid), - zap.String("container_id", r.ID), - ) + ip.Server.Log().WithField("container_id", r.ID).Info("running installation script for server in container") if err := ip.client.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil { return "", err } @@ -366,11 +352,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { go func(id string) { ip.Server.Events().Publish(DaemonMessageEvent, "Starting installation process, this could take a few minutes...") if err := ip.StreamOutput(id); err != nil { - zap.S().Errorw( - "error handling streaming output for server install process", - zap.String("container_id", id), - zap.Error(err), - ) + ip.Server.Log().WithField("error", err).Error("error while handling output stream for server install process") } ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.") }(r.ID) @@ -409,12 +391,10 @@ func (ip *InstallationProcess) StreamOutput(id string) error { } if err := s.Err(); err != nil { - zap.S().Warnw( - "error processing scanner line in installation output for server", - zap.String("server", ip.Server.Uuid), - zap.String("container_id", id), - zap.Error(errors.WithStack(err)), - ) + ip.Server.Log().WithFields(log.Fields{ + "container_id": id, + "error": errors.WithStack(err), + }).Warn("error processing scanner line in installation output for server") } return nil diff --git a/server/listeners.go b/server/listeners.go index c164c99..73df884 100644 --- a/server/listeners.go +++ b/server/listeners.go @@ -1,8 +1,8 @@ package server import ( + "github.com/apex/log" "github.com/pterodactyl/wings/api" - "go.uber.org/zap" "strings" ) @@ -28,9 +28,10 @@ func (s *Server) onConsoleOutput(data string) { // set the server to that state. Only do this if the server is not currently stopped // or stopping. if s.GetState() == ProcessStartingState && strings.Contains(data, s.processConfiguration.Startup.Done) { - zap.S().Debugw( - "detected server in running state based on line output", zap.String("match", s.processConfiguration.Startup.Done), zap.String("against", data), - ) + s.Log().WithFields(log.Fields{ + "match": s.processConfiguration.Startup.Done, + "against": data, + }).Debug("detected server in running state based on console line output") s.SetState(ProcessRunningState) } From 7d4a8d7f7ef62f7704606bea891bfe9a3e7c8fe7 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 13 Jun 2020 10:40:26 -0700 Subject: [PATCH 11/30] Update everything expect transfers & sftp to not use zap --- router/router_server.go | 20 +++++++------------- router/router_server_backup.go | 3 +-- server/backup.go | 22 ++++++++++++---------- server/backup/archiver.go | 4 ++-- server/backup/backup.go | 7 +++++-- server/backup/backup_s3.go | 9 ++++++--- server/filesystem.go | 3 +-- server/state.go | 9 ++++----- server/update.go | 9 ++------- 9 files changed, 40 insertions(+), 46 deletions(-) diff --git a/router/router_server.go b/router/router_server.go index 8018add..4946223 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -6,7 +6,6 @@ import ( "github.com/gin-gonic/gin" "github.com/pkg/errors" "github.com/pterodactyl/wings/server" - "go.uber.org/zap" "net/http" "os" "strconv" @@ -137,11 +136,7 @@ func postServerInstall(c *gin.Context) { go func(serv *server.Server) { if err := serv.Install(); err != nil { - zap.S().Errorw( - "failed to execute server installation process", - zap.String("server", serv.Uuid), - zap.Error(err), - ) + serv.Log().WithField("error", err).Error("failed to execute server installation process") } }(s) @@ -154,11 +149,7 @@ func postServerReinstall(c *gin.Context) { go func(serv *server.Server) { if err := serv.Reinstall(); err != nil { - zap.S().Errorw( - "failed to complete server reinstall process", - zap.String("server", serv.Uuid), - zap.Error(err), - ) + serv.Log().WithField("error", err).Error("failed to complete server re-install process") } }(s) @@ -176,7 +167,7 @@ func deleteServer(c *gin.Context) { // Delete the server's archive if it exists. We intentionally don't return // here, if the archive fails to delete, the server can still be removed. if err := s.Archiver.DeleteIfExists(); err != nil { - zap.S().Warnw("failed to delete server archive during deletion process", zap.String("server", s.Uuid), zap.Error(err)) + s.Log().WithField("error", err).Warn("failed to delete server archive during deletion process") } // Unsubscribe all of the event listeners. @@ -197,7 +188,10 @@ func deleteServer(c *gin.Context) { // so we don't want to block the HTTP call while waiting on this. go func(p string) { if err := os.RemoveAll(p); err != nil { - zap.S().Warnw("failed to remove server files during deletion process", zap.String("path", p), zap.Error(errors.WithStack(err))) + log.WithFields(log.Fields{ + "path": p, + "error": errors.WithStack(err), + }).Warn("failed to remove server files during deletion process") } }(s.Filesystem.Path()) diff --git a/router/router_server_backup.go b/router/router_server_backup.go index a154542..162553c 100644 --- a/router/router_server_backup.go +++ b/router/router_server_backup.go @@ -6,7 +6,6 @@ import ( "github.com/gin-gonic/gin" "github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/server/backup" - "go.uber.org/zap" "net/http" ) @@ -40,7 +39,7 @@ func postServerBackup(c *gin.Context) { go func(b backup.BackupInterface, serv *server.Server) { if err := serv.Backup(b); err != nil { - zap.S().Errorw("failed to generate backup for server", zap.Error(err)) + serv.Log().WithField("error", err).Error("failed to generate backup for server") } }(adapter, s) diff --git a/server/backup.go b/server/backup.go index 67b84f4..a12b65f 100644 --- a/server/backup.go +++ b/server/backup.go @@ -2,10 +2,10 @@ package server import ( "bufio" + "github.com/apex/log" "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/server/backup" - "go.uber.org/zap" "os" "path" ) @@ -17,16 +17,15 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc rerr, err := r.SendBackupStatus(uuid, ad.ToRequest(successful)) if rerr != nil || err != nil { if err != nil { - zap.S().Errorw( - "failed to notify panel of backup status due to internal code error", - zap.String("backup", s.Uuid), - zap.Error(err), - ) + s.Log().WithFields(log.Fields{ + "backup": uuid, + "error": err, + }).Error("failed to notify panel of backup status due to internal code error") return err } - zap.S().Warnw(rerr.String(), zap.String("backup", uuid)) + s.Log().WithField("backup", uuid).Warn(rerr.String()) return errors.New(rerr.String()) } @@ -66,7 +65,7 @@ func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles // of the server files directory, and use that to generate the backup. if len(ignored) == 0 { if i, err := s.getServerwideIgnoredFiles(); err != nil { - zap.S().Warnw("failed to retrieve server ignored files", zap.String("server", s.Uuid), zap.Error(err)) + s.Log().WithField("error", err).Warn("failed to retrieve ignored files listing for server") } else { ignored = i } @@ -89,7 +88,10 @@ func (s *Server) Backup(b backup.BackupInterface) error { ad, err := b.Generate(inc, s.Filesystem.Path()) if err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { - zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Identifier()), zap.Error(err)) + s.Log().WithFields(log.Fields{ + "backup": b.Identifier(), + "error": err, + }).Warn("failed to notify panel of failed backup state") } return errors.WithStack(err) @@ -112,4 +114,4 @@ func (s *Server) Backup(b backup.BackupInterface) error { }) return nil -} \ No newline at end of file +} diff --git a/server/backup/archiver.go b/server/backup/archiver.go index 2244f2c..13a2dcb 100644 --- a/server/backup/archiver.go +++ b/server/backup/archiver.go @@ -3,9 +3,9 @@ package backup import ( "archive/tar" "context" + "github.com/apex/log" gzip "github.com/klauspost/pgzip" "github.com/remeh/sizedwaitgroup" - "go.uber.org/zap" "golang.org/x/sync/errgroup" "io" "os" @@ -67,7 +67,7 @@ func (a *Archive) Create(dest string, ctx context.Context) error { // Attempt to remove the archive if there is an error, report that error to // the logger if it fails. if rerr := os.Remove(dest); rerr != nil && !os.IsNotExist(rerr) { - zap.S().Warnw("failed to delete corrupted backup archive", zap.String("location", dest)) + log.WithField("location", dest).Warn("failed to delete corrupted backup archive") } return err diff --git a/server/backup/backup.go b/server/backup/backup.go index 876ced0..2456c34 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -3,10 +3,10 @@ package backup import ( "crypto/sha256" "encoding/hex" + "github.com/apex/log" "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "io" "os" "path" @@ -121,7 +121,10 @@ func (b *Backup) Details() *ArchiveDetails { resp, err := b.Checksum() if err != nil { - zap.S().Errorw("failed to calculate checksum for backup", zap.String("backup", b.Uuid), zap.Error(err)) + log.WithFields(log.Fields{ + "backup": b.Identifier(), + "error": err, + }).Error("failed to calculate checksum for backup") } checksum = hex.EncodeToString(resp) diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 0d1616d..e1bc0f7 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -3,7 +3,7 @@ package backup import ( "context" "fmt" - "go.uber.org/zap" + "github.com/apex/log" "io" "net/http" "os" @@ -76,8 +76,11 @@ func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) (*http.Response, erro } r.Body = rc - - zap.S().Debugw("uploading backup to remote S3 endpoint", zap.String("endpoint", s.PresignedUrl), zap.Any("headers", r.Header)) + + log.WithFields(log.Fields{ + "endpoint": s.PresignedUrl, + "headers": r.Header, + }).Debug("uploading backup to remote S3 endpoint") return http.DefaultClient.Do(r) } diff --git a/server/filesystem.go b/server/filesystem.go index a587a12..bcb8166 100644 --- a/server/filesystem.go +++ b/server/filesystem.go @@ -11,7 +11,6 @@ import ( "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/server/backup" ignore "github.com/sabhiram/go-gitignore" - "go.uber.org/zap" "io" "io/ioutil" "os" @@ -134,7 +133,7 @@ func (fs *Filesystem) HasSpaceAvailable() bool { // the cache once we've gotten it. size, err := fs.DirectorySize("/") if err != nil { - zap.S().Warnw("failed to determine directory size", zap.String("server", fs.Server.Uuid), zap.Error(err)) + fs.Server.Log().WithField("error", err).Warn("failed to determine root server directory size") } // Always cache the size, even if there is an error. We want to always return that value diff --git a/server/state.go b/server/state.go index 39648f7..033e267 100644 --- a/server/state.go +++ b/server/state.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" - "go.uber.org/zap" "io" "io/ioutil" "os" @@ -98,7 +97,7 @@ func (s *Server) SetState(state string) error { // to the disk should we forget to do it elsewhere. go func() { if err := saveServerStates(); err != nil { - zap.S().Warnw("failed to write server states to disk", zap.Error(err)) + s.Log().WithField("error", err).Warn("failed to write server states to disk") } }() @@ -111,14 +110,14 @@ func (s *Server) SetState(state string) error { // separate thread as to not block any actions currently taking place in the flow // that called this function. if (prevState == ProcessStartingState || prevState == ProcessRunningState) && s.GetState() == ProcessOfflineState { - zap.S().Infow("detected server as entering a potentially crashed state; running handler", zap.String("server", s.Uuid)) + s.Log().Info("detected server as entering a crashed state; running crash handler") go func(server *Server) { if err := server.handleServerCrash(); err != nil { if IsTooFrequentCrashError(err) { - zap.S().Infow("did not restart server after crash; occurred too soon after last", zap.String("server", server.Uuid)) + server.Log().Info("did not restart server after crash; occurred too soon after the last") } else { - zap.S().Errorw("failed to handle server crash state", zap.String("server", server.Uuid), zap.Error(err)) + server.Log().WithField("error", err).Error("failed to handle server crash") } } }(s) diff --git a/server/update.go b/server/update.go index f62c89f..a134f49 100644 --- a/server/update.go +++ b/server/update.go @@ -5,7 +5,6 @@ import ( "github.com/buger/jsonparser" "github.com/imdario/mergo" "github.com/pkg/errors" - "go.uber.org/zap" ) // Merges data passed through in JSON form into the existing server object. @@ -101,14 +100,10 @@ func (s *Server) runBackgroundActions() { // yet, do it immediately. go func(server *Server) { if server.Suspended && server.GetState() != ProcessOfflineState { - zap.S().Infow("server suspended with running process state, terminating now", zap.String("server", server.Uuid)) + server.Log().Info("server suspended with running process state, terminating now") if err := server.Environment.WaitForStop(10, true); err != nil { - zap.S().Warnw( - "failed to stop server environment after seeing suspension", - zap.String("server", server.Uuid), - zap.Error(err), - ) + server.Log().WithField("error", err).Warn("failed to terminate server environment after suspension") } } }(s) From b03aa20c8d15dc7ae414400b284f3e2bbf754e34 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 13 Jun 2020 10:45:52 -0700 Subject: [PATCH 12/30] Always use release mode --- cmd/root.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 71c7656..918baae 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "github.com/apex/log" + "github.com/gin-gonic/gin" "github.com/mitchellh/colorstring" "github.com/pterodactyl/wings/loggers/cli" "net/http" @@ -223,6 +224,8 @@ func rootCmdRun(*cobra.Command, []string) { "port": c.Api.Port, }).Info("configuring webserver...") + gin.SetMode("release") + r := router.Configure() addr := fmt.Sprintf("%s:%d", c.Api.Host, c.Api.Port) From 495ad4defd9b1b1f4094031a9d1956e6a2cd253b Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 13 Jun 2020 10:54:38 -0700 Subject: [PATCH 13/30] Cleaner gin logging in debug --- cmd/root.go | 3 --- router/router.go | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 918baae..71c7656 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "fmt" "github.com/apex/log" - "github.com/gin-gonic/gin" "github.com/mitchellh/colorstring" "github.com/pterodactyl/wings/loggers/cli" "net/http" @@ -224,8 +223,6 @@ func rootCmdRun(*cobra.Command, []string) { "port": c.Api.Port, }).Info("configuring webserver...") - gin.SetMode("release") - r := router.Configure() addr := fmt.Sprintf("%s:%d", c.Api.Host, c.Api.Port) diff --git a/router/router.go b/router/router.go index 99d9d1f..162dc57 100644 --- a/router/router.go +++ b/router/router.go @@ -1,13 +1,32 @@ package router import ( + "github.com/apex/log" "github.com/gin-gonic/gin" ) // Configures the routing infrastructure for this daemon instance. func Configure() *gin.Engine { - router := gin.Default() + gin.SetMode("release") + + router := gin.New() + + router.Use(gin.Recovery()) router.Use(SetAccessControlHeaders) + // @todo log this into a different file so you can setup IP blocking for abusive requests and such. + // This should still dump requests in debug mode since it does help with understanding the request + // lifecycle and quickly seeing what was called leading to the logs. However, it isn't feasible to mix + // this output in production and still get meaningful logs from it since they'll likely just be a huge + // spamfest. + router.Use(gin.LoggerWithFormatter(func(params gin.LogFormatterParams) string { + log.WithFields(log.Fields{ + "client_ip": params.ClientIP, + "status": params.StatusCode, + "latency": params.Latency, + }).Debugf("%s %s", params.MethodColor()+params.Method+params.ResetColor(), params.Path) + + return "" + })) router.OPTIONS("/api/system", func(c *gin.Context) { c.Status(200) From d91de3d912978f3bbafe49abd5a2c3a7f699738c Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 18 Jun 2020 21:05:00 -0700 Subject: [PATCH 14/30] Fix makefile --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 44eab6b..8db5e0c 100644 --- a/Makefile +++ b/Makefile @@ -7,4 +7,6 @@ compress: cross-build: clean build compress clean: - rm -rf build/wings_* \ No newline at end of file + rm -rf build/wings_* + +.PHONY: all build compress clean \ No newline at end of file From db31722cfcdbbc01c78b83c45166bc95848c8501 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 22 Jun 2020 20:41:45 -0700 Subject: [PATCH 15/30] Don't cause a double stacktrace on certain errors --- api/api.go | 12 +++++++++++- server/install.go | 14 ++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api/api.go b/api/api.go index a3bb98f..8f9783e 100644 --- a/api/api.go +++ b/api/api.go @@ -130,6 +130,12 @@ func (r *PanelRequest) HttpResponseCode() int { return r.Response.StatusCode } +func IsRequestError(err error) bool { + _, ok := err.(*RequestError) + + return ok +} + type RequestError struct { Code string `json:"code"` Status string `json:"status"` @@ -137,10 +143,14 @@ type RequestError struct { } // Returns the error response in a string form that can be more easily consumed. -func (re *RequestError) String() string { +func (re *RequestError) Error() string { return fmt.Sprintf("%s: %s (HTTP/%s)", re.Code, re.Detail, re.Status) } +func (re *RequestError) String() string { + return re.Error() +} + type RequestErrorBag struct { Errors []RequestError `json:"errors"` } diff --git a/server/install.go b/server/install.go index efa6c92..b89ffae 100644 --- a/server/install.go +++ b/server/install.go @@ -27,10 +27,16 @@ func (s *Server) Install() error { s.Log().Debug("notifying panel of server install state") if serr := s.SyncInstallState(err == nil); serr != nil { - s.Log().WithFields(log.Fields{ - "was_successful": err == nil, - "error": serr, - }).Warn("failed to notify panel of server install state") + l := s.Log().WithField("was_successful", err == nil) + + // If the request was successful but there was an error with this request, attach the + // error to this log entry. Otherwise ignore it in this log since whatever is calling + // this function should handle the error and will end up logging the same one. + if err == nil { + l.WithField("error", serr) + } + + l.Warn("failed to notify panel of server install state") } return err From f318962371cd8cf6de444718dc1f3bd1d9ffc1a6 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 22 Jun 2020 20:48:38 -0700 Subject: [PATCH 16/30] Ensure that more error stacks get recorded --- loggers/cli/cli.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/loggers/cli/cli.go b/loggers/cli/cli.go index 8e82227..d980250 100644 --- a/loggers/cli/cli.go +++ b/loggers/cli/cli.go @@ -70,6 +70,8 @@ func (h *Handler) HandleLog(e *log.Entry) error { } if err, ok := e.Fields.Get("error").(error); ok { + var br = color2.New(color2.Bold, color2.FgRed) + if e, ok := errors.Cause(err).(tracer); ok { st := e.StackTrace() @@ -78,11 +80,9 @@ func (h *Handler) HandleLog(e *log.Entry) error { l = 5 } - br := color2.New(color2.Bold, color2.FgRed) - fmt.Fprintf(h.Writer, "\n%s%+v\n\n", br.Sprintf("Stacktrace:"), st[0:l]) } else { - fmt.Printf("\n\nINVALID TRACER\n\n") + fmt.Fprintf(h.Writer, "\n%s\n%+v\n\n", br.Sprintf("Stacktrace:"), err) } } else { fmt.Printf("\n\nINVALID ERROR\n\n") From a3d83d23bdde2907a93ce70b48abd22728c84d7b Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 22 Jun 2020 20:52:23 -0700 Subject: [PATCH 17/30] Don't try to send space available when loading from a configuration Server is not always installed when this function is called, this will cause errors in those cases. --- server/server.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/server.go b/server/server.go index 8c8eabb..b510185 100644 --- a/server/server.go +++ b/server/server.go @@ -247,10 +247,6 @@ func FromConfiguration(data *api.ServerConfigurationResponse) (*Server, error) { } s.Resources = ResourceUsage{} - // Force the disk usage to become cached to return in a resources response - // or when connecting to the websocket of an offline server. - go s.Filesystem.HasSpaceAvailable() - // Forces the configuration to be synced with the panel. if err := s.SyncWithConfiguration(data); err != nil { return nil, err From 073247e4e1e7b10d94c028fa274c4abdf6439d77 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 22 Jun 2020 20:56:55 -0700 Subject: [PATCH 18/30] Use 15 minute context timeout for pulling, not 10 seconds... closes #2130 --- server/environment_docker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/environment_docker.go b/server/environment_docker.go index 00324ac..afe0b70 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -564,7 +564,10 @@ func (d *DockerEnvironment) DisableResourcePolling() error { // // @todo handle authorization & local images func (d *DockerEnvironment) ensureImageExists(c *client.Client) error { - ctx, _ := context.WithTimeout(context.Background(), time.Second*10) + // Give it up to 15 minutes to pull the image. I think this should cover 99.8% of cases where an + // image pull might fail. I can't imagine it will ever take more than 15 minutes to fully pull + // an image. Let me know when I am inevitably wrong here... + ctx, _ := context.WithTimeout(context.Background(), time.Minute*15) out, err := c.ImagePull(ctx, d.Server.Container.Image, types.ImagePullOptions{All: false}) if err != nil { From 1f6789cba3cb888fe61fd5cb6e3f4adc1e189459 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 22 Jun 2020 21:38:16 -0700 Subject: [PATCH 19/30] Acquire exclusive lock when installing a server Also allows aborting a server install mid-process when the server is deleted before the process finishes. --- router/router_server.go | 5 +++ server/install.go | 92 ++++++++++++++++++++++++++++++++++------- server/server.go | 18 ++++++++ 3 files changed, 100 insertions(+), 15 deletions(-) diff --git a/router/router_server.go b/router/router_server.go index 4946223..a516005 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -164,6 +164,11 @@ func deleteServer(c *gin.Context) { // to start it while this process is running. s.Suspended = true + // If the server is currently installing, abort it. + if s.IsInstalling() { + s.AbortInstallation() + } + // Delete the server's archive if it exists. We intentionally don't return // here, if the archive fails to delete, the server can still be removed. if err := s.Archiver.DeleteIfExists(); err != nil { diff --git a/server/install.go b/server/install.go index b89ffae..e5e8ad9 100644 --- a/server/install.go +++ b/server/install.go @@ -12,12 +12,14 @@ import ( "github.com/pkg/errors" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" + "golang.org/x/sync/semaphore" "io" "io/ioutil" "os" "path" "path/filepath" "sync" + "time" ) // Executes the installation stack for a server process. Bubbles any errors up to the calling @@ -84,8 +86,8 @@ type InstallationProcess struct { Server *Server Script *api.InstallationScript - client *client.Client - mutex *sync.Mutex + client *client.Client + context context.Context } // Generates a new installation process struct that will be used to create containers, @@ -94,21 +96,70 @@ func NewInstallationProcess(s *Server, script *api.InstallationScript) (*Install proc := &InstallationProcess{ Script: script, Server: s, - mutex: &sync.Mutex{}, } + ctx, cancel := context.WithCancel(context.Background()) + s.installer.cancel = &cancel + if c, err := client.NewClientWithOpts(client.FromEnv); err != nil { return nil, errors.WithStack(err) } else { proc.client = c + proc.context = ctx } return proc, nil } +// Try to obtain an exclusive lock on the installation process for the server. Waits up to 10 +// seconds before aborting with a context timeout. +func (s *Server) acquireInstallationLock() error { + if s.installer.sem == nil { + s.installer.sem = semaphore.NewWeighted(1) + } + + ctx, _ := context.WithTimeout(context.Background(), time.Second*10) + + return s.installer.sem.Acquire(ctx, 1) +} + +// Determines if the server is actively running the installation process by checking the status +// of the semaphore lock. +func (s *Server) IsInstalling() bool { + if s.installer.sem == nil { + return false + } + + if s.installer.sem.TryAcquire(1) { + // If we made it into this block it means we were able to obtain an exclusive lock + // on the semaphore. In that case, go ahead and release that lock immediately, and + // return false. + s.installer.sem.Release(1) + + return false + } + + return true +} + +// Aborts the server installation process by calling the cancel function on the installer +// context. +func (s *Server) AbortInstallation() { + if !s.IsInstalling() { + return + } + + if s.installer.cancel != nil { + cancel := *s.installer.cancel + + s.Log().Warn("aborting running installation process") + cancel() + } +} + // Removes the installer container for the server. func (ip *InstallationProcess) RemoveContainer() { - err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{ + err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", types.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, }) @@ -124,6 +175,20 @@ func (ip *InstallationProcess) RemoveContainer() { // Once the container finishes installing the results will be stored in an installation // log in the server's configuration directory. func (ip *InstallationProcess) Run() error { + ip.Server.Log().Debug("acquiring installation process lock") + if err := ip.Server.acquireInstallationLock(); err != nil { + return err + } + + // We now have an exclusive lock on this installation process. Ensure that whenever this + // process is finished that the semaphore is released so that other processes and be executed + // without encounting a wait timeout. + defer func() { + ip.Server.Log().Debug("releasing installation process lock") + ip.Server.installer.sem.Release(1) + ip.Server.installer.cancel = nil + }() + installPath, err := ip.BeforeExecute() if err != nil { return err @@ -183,7 +248,7 @@ func (ip *InstallationProcess) writeScriptToDisk() (string, error) { // Pulls the docker image to be used for the installation container. func (ip *InstallationProcess) pullInstallationImage() error { - r, err := ip.client.ImagePull(context.Background(), ip.Script.ContainerImage, types.ImagePullOptions{}) + r, err := ip.client.ImagePull(ip.context, ip.Script.ContainerImage, types.ImagePullOptions{}) if err != nil { return errors.WithStack(err) } @@ -237,7 +302,7 @@ func (ip *InstallationProcess) BeforeExecute() (string, error) { Force: true, } - if err := ip.client.ContainerRemove(context.Background(), ip.Server.Uuid+"_installer", opts); err != nil { + if err := ip.client.ContainerRemove(ip.context, ip.Server.Uuid+"_installer", opts); err != nil { if !client.IsErrNotFound(err) { e = append(e, err) } @@ -264,11 +329,10 @@ func (ip *InstallationProcess) GetLogPath() string { // process to store in the server configuration directory, and then destroys the associated // installation container. func (ip *InstallationProcess) AfterExecute(containerId string) error { - ctx := context.Background() defer ip.RemoveContainer() ip.Server.Log().WithField("container_id", containerId).Debug("pulling installation logs for server") - reader, err := ip.client.ContainerLogs(ctx, containerId, types.ContainerLogsOptions{ + reader, err := ip.client.ContainerLogs(ip.context, containerId, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, Follow: false, @@ -295,8 +359,6 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { // Executes the installation process inside a specially created docker container. func (ip *InstallationProcess) Execute(installPath string) (string, error) { - ctx := context.Background() - conf := &container.Config{ Hostname: "installer", AttachStdout: true, @@ -345,13 +407,13 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { } ip.Server.Log().WithField("install_script", installPath+"/install.sh").Info("creating install container for server process") - r, err := ip.client.ContainerCreate(ctx, conf, hostConf, nil, ip.Server.Uuid+"_installer") + r, err := ip.client.ContainerCreate(ip.context, conf, hostConf, nil, ip.Server.Uuid+"_installer") if err != nil { return "", errors.WithStack(err) } ip.Server.Log().WithField("container_id", r.ID).Info("running installation script for server in container") - if err := ip.client.ContainerStart(ctx, r.ID, types.ContainerStartOptions{}); err != nil { + if err := ip.client.ContainerStart(ip.context, r.ID, types.ContainerStartOptions{}); err != nil { return "", err } @@ -363,7 +425,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { ip.Server.Events().Publish(DaemonMessageEvent, "Installation process completed.") }(r.ID) - sChann, eChann := ip.client.ContainerWait(ctx, r.ID, container.WaitConditionNotRunning) + sChann, eChann := ip.client.ContainerWait(ip.context, r.ID, container.WaitConditionNotRunning) select { case err := <-eChann: if err != nil { @@ -379,7 +441,7 @@ func (ip *InstallationProcess) Execute(installPath string) (string, error) { // directory, as well as to a websocket listener so that the process can be viewed in // the panel by administrators. func (ip *InstallationProcess) StreamOutput(id string) error { - reader, err := ip.client.ContainerLogs(context.Background(), id, types.ContainerLogsOptions{ + reader, err := ip.client.ContainerLogs(ip.context, id, types.ContainerLogsOptions{ ShowStdout: true, ShowStderr: true, Follow: true, @@ -399,7 +461,7 @@ func (ip *InstallationProcess) StreamOutput(id string) error { if err := s.Err(); err != nil { ip.Server.Log().WithFields(log.Fields{ "container_id": id, - "error": errors.WithStack(err), + "error": errors.WithStack(err), }).Warn("error processing scanner line in installation output for server") } diff --git a/server/server.go b/server/server.go index b510185..5389841 100644 --- a/server/server.go +++ b/server/server.go @@ -1,6 +1,7 @@ package server import ( + "context" "fmt" "github.com/apex/log" "github.com/creasty/defaults" @@ -9,6 +10,7 @@ import ( "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" "github.com/remeh/sizedwaitgroup" + "golang.org/x/sync/semaphore" "math" "os" "strings" @@ -71,11 +73,27 @@ type Server struct { // started, and then cached here. processConfiguration *api.ProcessConfiguration + // Tracks the installation process for this server and prevents a server from running + // two installer processes at the same time. This also allows us to cancel a running + // installation process, for example when a server is deleted from the panel while the + // installer process is still running. + installer InstallerDetails + // Internal mutex used to block actions that need to occur sequentially, such as // writing the configuration to the disk. sync.RWMutex } +type InstallerDetails struct { + // The cancel function for the installer. This will be a non-nil value while there + // is an installer running for the server. + cancel *context.CancelFunc + + // Installer lock. You should obtain an exclusive lock on this context while running + // the installation process and release it when finished. + sem *semaphore.Weighted +} + // The build settings for a given server that impact docker container creation and // resource limits for a server instance. type BuildSettings struct { From b33f14ddd9544f286479f1183fb8b954d62ce979 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 29 Jun 2020 20:08:36 -0700 Subject: [PATCH 20/30] Correctly handle replacements with escaped values; closes #2041 --- parser/helpers.go | 16 +++++++--------- parser/parser.go | 14 +++++++------- parser/value.go | 4 +++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/parser/helpers.go b/parser/helpers.go index d66d2b0..5b58ee9 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -96,12 +96,12 @@ func (f *ConfigurationFile) IterateOverJson(data []byte) (*gabs.Container, error // If the child is a null value, nothing will happen. Seems reasonable as of the // time this code is being written. for _, child := range parsed.Path(strings.Trim(parts[0], ".")).Children() { - if err := v.SetAtPathway(child, strings.Trim(parts[1], "."), value); err != nil { + if err := v.SetAtPathway(child, strings.Trim(parts[1], "."), []byte(value)); err != nil { return nil, err } } } else { - if err = v.SetAtPathway(parsed, v.Match, value); err != nil { + if err = v.SetAtPathway(parsed, v.Match, []byte(value)); err != nil { return nil, err } } @@ -149,12 +149,12 @@ func (cfr *ConfigurationFileReplacement) SetAtPathway(c *gabs.Container, path st } // Looks up a configuration value on the Daemon given a dot-notated syntax. -func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplacement) ([]byte, error) { +func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplacement) (string, error) { // If this is not something that we can do a regex lookup on then just continue // on our merry way. If the value isn't a string, we're not going to be doing anything // with it anyways. if cfr.ReplaceWith.Type() != jsonparser.String || !configMatchRegex.Match(cfr.ReplaceWith.Value()) { - return cfr.ReplaceWith.Value(), nil + return cfr.ReplaceWith.String(), nil } // If there is a match, lookup the value in the configuration for the Daemon. If no key @@ -174,17 +174,15 @@ func (f *ConfigurationFile) LookupConfigurationValue(cfr ConfigurationFileReplac match, _, _, err := jsonparser.Get(f.configuration, path...) if err != nil { if err != jsonparser.KeyPathNotFoundError { - return match, errors.WithStack(err) + return string(match), errors.WithStack(err) } log.WithFields(log.Fields{"path": path, "filename": f.FileName}).Debug("attempted to load a configuration value that does not exist") // If there is no key, keep the original value intact, that way it is obvious there // is a replace issue at play. - return match, nil + return string(match), nil } else { - replaced := []byte(configMatchRegex.ReplaceAllString(cfr.ReplaceWith.String(), string(match))) - - return replaced, nil + return configMatchRegex.ReplaceAllString(cfr.ReplaceWith.String(), string(match)), nil } } diff --git a/parser/parser.go b/parser/parser.go index af2aef4..69e1da1 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -236,13 +236,13 @@ func (f *ConfigurationFile) parseXmlFile(path string) error { // Iterate over the elements we found and update their values. for _, element := range doc.FindElements(path) { - if xmlValueMatchRegex.Match(value) { - k := xmlValueMatchRegex.ReplaceAllString(string(value), "$1") - v := xmlValueMatchRegex.ReplaceAllString(string(value), "$2") + if xmlValueMatchRegex.MatchString(value) { + k := xmlValueMatchRegex.ReplaceAllString(value, "$1") + v := xmlValueMatchRegex.ReplaceAllString(value, "$2") element.CreateAttr(k, v) } else { - element.SetText(string(value)) + element.SetText(value) } } } @@ -313,9 +313,9 @@ func (f *ConfigurationFile) parseIniFile(path string) error { // If the key exists in the file go ahead and set the value, otherwise try to // create it in the section. if s.HasKey(k) { - s.Key(k).SetValue(string(value)) + s.Key(k).SetValue(value) } else { - if _, err := s.NewKey(k, string(value)); err != nil { + if _, err := s.NewKey(k, value); err != nil { return err } } @@ -452,7 +452,7 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error { continue } - if _, _, err := p.Set(replace.Match, string(data)); err != nil { + if _, _, err := p.Set(replace.Match, data); err != nil { return err } } diff --git a/parser/value.go b/parser/value.go index 5fd307e..b9528fc 100644 --- a/parser/value.go +++ b/parser/value.go @@ -14,7 +14,9 @@ func (cv *ReplaceValue) Value() []byte { } func (cv *ReplaceValue) String() string { - return string(cv.value) + str, _ := jsonparser.ParseString(cv.value) + + return str } func (cv *ReplaceValue) Type() jsonparser.ValueType { From 2dad3102e08ef9be01f970c3f595a470e7a3435b Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 29 Jun 2020 20:21:41 -0700 Subject: [PATCH 21/30] Fix saving of ini configuration files to the disk --- parser/parser.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 69e1da1..a86e194 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -273,12 +273,13 @@ func (f *ConfigurationFile) parseXmlFile(path string) error { // Parses an ini file. func (f *ConfigurationFile) parseIniFile(path string) error { // Ini package can't handle a non-existent file, so handle that automatically here - // by creating it if not exists. + // by creating it if not exists. Then, immediately close the file since we will use + // other methods to write the new contents. file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return err } - defer file.Close() + file.Close() cfg, err := ini.Load(path) if err != nil { @@ -321,16 +322,7 @@ func (f *ConfigurationFile) parseIniFile(path string) error { } } - // Truncate the file before attempting to write the changes. - if err := os.Truncate(path, 0); err != nil { - return err - } - - if _, err := cfg.WriteTo(file); err != nil { - return err - } - - return nil + return cfg.SaveTo(path) } // Parses a json file updating any matching key/value pairs. If a match is not found, the From 05a47304893fdbb1d31101a1c1538d026b62d618 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 29 Jun 2020 20:33:54 -0700 Subject: [PATCH 22/30] Fix configuration file saving to disk using the config command closes pterodactyl/panel#2135 --- cmd/configure.go | 5 ++--- config/config.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/configure.go b/cmd/configure.go index 588639a..8538721 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2/terminal" - "github.com/creasty/defaults" "github.com/pterodactyl/wings/config" "github.com/spf13/cobra" "io/ioutil" @@ -147,8 +146,8 @@ func configureCmdRun(cmd *cobra.Command, args []string) { b, err := ioutil.ReadAll(res.Body) - cfg := new(config.Configuration) - if err := defaults.Set(cfg); err != nil { + cfg, err := config.NewFromPath(configPath) + if err != nil { panic(err) } diff --git a/config/config.go b/config/config.go index 755de18..14c8c63 100644 --- a/config/config.go +++ b/config/config.go @@ -186,6 +186,27 @@ func GetJwtAlgorithm() *jwt.HMACSHA { return _jwtAlgo } +// Create a new struct and set the path where it should be stored. +func NewFromPath(path string) (*Configuration, error) { + c := new(Configuration) + if err := defaults.Set(c); err != nil { + return c, err + } + + c.unsafeSetPath(path) + + return c, nil +} + +// Sets the path where the configuration file is located on the server. This function should +// not be called except by processes that are generating the configuration such as the configration +// command shipped with this software. +func (c *Configuration) unsafeSetPath(path string) { + c.Lock() + c.path = path + c.Unlock() +} + // Returns the path for this configuration file. func (c *Configuration) GetPath() string { return c.path From d284c4aec92975867b66528df7a34ee302fc4984 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 29 Jun 2020 20:42:26 -0700 Subject: [PATCH 23/30] Fix lock obtainment to avoid freeze --- config/config.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 14c8c63..82ef67d 100644 --- a/config/config.go +++ b/config/config.go @@ -132,7 +132,7 @@ func ReadConfiguration(path string) (*Configuration, error) { } // Track the location where we created this configuration. - c.path = path + c.unsafeSetPath(path) // Replace environment variables within the configuration file with their // values from the host system. @@ -209,6 +209,9 @@ func (c *Configuration) unsafeSetPath(path string) { // Returns the path for this configuration file. func (c *Configuration) GetPath() string { + c.RLock() + defer c.RUnlock() + return c.path } @@ -266,11 +269,10 @@ func (c *Configuration) setSystemUser(u *user.User) error { gid, _ := strconv.Atoi(u.Gid) c.Lock() - defer c.Unlock() - c.System.Username = u.Username c.System.User.Uid = uid c.System.User.Gid = gid + c.Unlock() return c.WriteToDisk() } @@ -331,6 +333,10 @@ func (c *Configuration) EnsureFilePermissions() error { // lock on the file. This prevents something else from writing at the exact same time and // leading to bad data conditions. func (c *Configuration) WriteToDisk() error { + // Obtain an exclusive write against the configuration file. + c.writeLock.Lock() + defer c.writeLock.Unlock() + ccopy := *c // If debugging is set with the flag, don't save that to the configuration file, otherwise // you'll always end up in debug mode. @@ -347,10 +353,6 @@ func (c *Configuration) WriteToDisk() error { return err } - // Obtain an exclusive write against the configuration file. - c.writeLock.Lock() - defer c.writeLock.Unlock() - if err := ioutil.WriteFile(c.GetPath(), b, 0644); err != nil { return err } From d6a3d9adb13d8f365ff1f16a4ffdea9b83b3e2e9 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Mon, 29 Jun 2020 20:56:13 -0700 Subject: [PATCH 24/30] Don't obliterate custom SSL locations if defined closes pterodactyl/panel#2121 --- router/router_system.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/router/router_system.go b/router/router_system.go index f5dfed3..2e1ed13 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -9,6 +9,7 @@ import ( "github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/system" "net/http" + "strings" ) // Returns information about the system that wings is running on. @@ -78,6 +79,16 @@ func postUpdateConfiguration(c *gin.Context) { return } + // Keep the SSL certificates the same since the Panel will send through Lets Encrypt + // default locations. However, if we picked a different location manually we don't + // want to override that. + // + // If you pass through manual locations in the API call this logic will be skipped. + if strings.HasPrefix(cfg.Api.Ssl.KeyFile, "/etc/letsencrypt/live/") { + cfg.Api.Ssl.KeyFile = ccopy.Api.Ssl.KeyFile + cfg.Api.Ssl.CertificateFile = ccopy.Api.Ssl.CertificateFile + } + config.Set(&cfg) if err := config.Get().WriteToDisk(); err != nil { // If there was an error writing to the disk, revert back to the configuration we had From 79a582a5f274046350ce0280d7f429cf0e02f58c Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Tue, 30 Jun 2020 20:45:36 -0700 Subject: [PATCH 25/30] Generate a nicer install log with more detailed information --- server/install.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/server/install.go b/server/install.go index e5e8ad9..99faadd 100644 --- a/server/install.go +++ b/server/install.go @@ -13,6 +13,7 @@ import ( "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/config" "golang.org/x/sync/semaphore" + "html/template" "io" "io/ioutil" "os" @@ -349,7 +350,37 @@ func (ip *InstallationProcess) AfterExecute(containerId string) error { defer f.Close() // We write the contents of the container output to a more "permanent" file so that they - // can be referenced after this container is deleted. + // can be referenced after this container is deleted. We'll also include the environment + // variables passed into the container to make debugging things a little easier. + ip.Server.Log().WithField("path", ip.GetLogPath()).Debug("writing most recent installation logs to disk") + + tmpl, err := template.New("header").Parse(`Pterodactyl Server Installation Log + +| +| Details +| ------------------------------ + Server UUID: {{.Server.Uuid}} + Container Image: {{.Script.ContainerImage}} + Container Entrypoint: {{.Script.Entrypoint}} + +| +| Environment Variables +| ------------------------------ +{{ range $key, $value := .Server.GetEnvironmentVariables }} {{ $value }} +{{ end }} + +| +| Script Output +| ------------------------------ +`) + if err != nil { + return errors.WithStack(err) + } + + if err := tmpl.Execute(f, ip); err != nil { + return errors.WithStack(err) + } + if _, err := io.Copy(f, reader); err != nil { return errors.WithStack(err) } From ea2630946a0cd6117ac5c792d3939a169cba4521 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Tue, 30 Jun 2020 20:56:55 -0700 Subject: [PATCH 26/30] Sync server state with Panel before performing installation to ensure information is up to date --- router/router_server.go | 2 +- router/router_system.go | 2 +- server/install.go | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/router/router_server.go b/router/router_server.go index a516005..87c0b67 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -135,7 +135,7 @@ func postServerInstall(c *gin.Context) { s := GetServer(c.Param("server")) go func(serv *server.Server) { - if err := serv.Install(); err != nil { + if err := serv.Install(true); err != nil { serv.Log().WithField("error", err).Error("failed to execute server installation process") } }(s) diff --git a/router/router_system.go b/router/router_system.go index 2e1ed13..27e8e10 100644 --- a/router/router_system.go +++ b/router/router_system.go @@ -59,7 +59,7 @@ func postCreateServer(c *gin.Context) { go func(i *installer.Installer) { i.Execute() - if err := i.Server().Install(); err != nil { + if err := i.Server().Install(false); err != nil { log.WithFields(log.Fields{"server": i.Uuid(), "error": err}).Error("failed to run install process for server") } }(install) diff --git a/server/install.go b/server/install.go index 99faadd..6a7805e 100644 --- a/server/install.go +++ b/server/install.go @@ -25,7 +25,17 @@ import ( // Executes the installation stack for a server process. Bubbles any errors up to the calling // function which should handle contacting the panel to notify it of the server state. -func (s *Server) Install() error { +// +// Pass true as the first arugment in order to execute a server sync before the process to +// ensure the latest information is used. +func (s *Server) Install(sync bool) error { + if sync { + s.Log().Info("syncing server state with remote source before executing installation process") + if err := s.Sync(); err != nil { + return err + } + } + err := s.internalInstall() s.Log().Debug("notifying panel of server install state") @@ -55,7 +65,7 @@ func (s *Server) Reinstall() error { } } - return s.Install() + return s.Install(true) } // Internal installation function used to simplify reporting back to the Panel. From e5b844d2c4baf000d9a3b0a79af55a00d88c7003 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Tue, 30 Jun 2020 21:34:47 -0700 Subject: [PATCH 27/30] Support automatically generating SSL certificates --- cmd/root.go | 44 ++++++++++++++++++++++++++++++++++++++------ go.mod | 5 +++-- go.sum | 4 ++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 71c7656..2f9bdf8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,6 +6,7 @@ import ( "github.com/apex/log" "github.com/mitchellh/colorstring" "github.com/pterodactyl/wings/loggers/cli" + "golang.org/x/crypto/acme/autocert" "net/http" "os" "path" @@ -27,11 +28,19 @@ import ( var configPath = config.DefaultLocation var debug = false var shouldRunProfiler = false +var useAutomaticTls = false +var tlsHostname = "" var root = &cobra.Command{ Use: "wings", Short: "The wings of the pterodactyl game management panel", Long: ``, + PreRun: func(cmd *cobra.Command, args []string) { + if useAutomaticTls && len(tlsHostname) == 0 { + fmt.Println("A TLS hostname must be provided when running wings with automatic TLS, e.g.:\n\n ./wings --auto-tls --tls-hostname my.example.com") + os.Exit(1) + } + }, Run: rootCmdRun, } @@ -39,6 +48,8 @@ func init() { root.PersistentFlags().StringVar(&configPath, "config", config.DefaultLocation, "set the location for the configuration file") root.PersistentFlags().BoolVar(&debug, "debug", false, "pass in order to run wings in debug mode") root.PersistentFlags().BoolVar(&shouldRunProfiler, "profile", false, "pass in order to profile wings") + root.PersistentFlags().BoolVar(&useAutomaticTls, "auto-tls", false, "pass in order to have wings generate and manage it's own SSL certificates using Let's Encrypt") + root.PersistentFlags().StringVar(&tlsHostname, "tls-hostname", "", "required with --auto-tls, the FQDN for the generated SSL certificate") root.AddCommand(configureCmd) } @@ -218,17 +229,38 @@ func rootCmdRun(*cobra.Command, []string) { } log.WithFields(log.Fields{ - "ssl": c.Api.Ssl.Enabled, - "host": c.Api.Host, - "port": c.Api.Port, - }).Info("configuring webserver...") + "use_ssl": c.Api.Ssl.Enabled, + "use_auto_tls": useAutomaticTls && len(tlsHostname) > 0, + "host_address": c.Api.Host, + "host_port": c.Api.Port, + }).Info("configuring internal webserver") r := router.Configure() addr := fmt.Sprintf("%s:%d", c.Api.Host, c.Api.Port) - if c.Api.Ssl.Enabled { + if useAutomaticTls && len(tlsHostname) > 0 { + m := autocert.Manager{ + Prompt: autocert.AcceptTOS, + Cache: autocert.DirCache(path.Join(c.System.RootDirectory, "/.tls-cache")), + HostPolicy: autocert.HostWhitelist(tlsHostname), + } + + log.WithField("hostname", tlsHostname). + Info("webserver is now listening with auto-TLS enabled; certifcates will be automatically generated by Let's Encrypt") + + // We don't use the autotls runner here since we need to specify a port other than 443 + // to be using for SSL connections for Wings. + s := &http.Server{Addr: addr, TLSConfig: m.TLSConfig(), Handler: r} + + go http.ListenAndServe(":http", m.HTTPHandler(nil)) + if err := s.ListenAndServeTLS("", ""); err != nil { + log.WithFields(log.Fields{"auto_tls": true, "tls_hostname": tlsHostname, "error": err}). + Fatal("failed to configure HTTP server using auto-tls") + os.Exit(1) + } + } else if c.Api.Ssl.Enabled { if err := r.RunTLS(addr, c.Api.Ssl.CertificateFile, c.Api.Ssl.KeyFile); err != nil { - log.WithField("error", err).Fatal("failed to configure HTTPS server") + log.WithFields(log.Fields{"auto_tls": false, "error": err}).Fatal("failed to configure HTTPS server") os.Exit(1) } } else { diff --git a/go.mod b/go.mod index 2f5b4ca..885a0a8 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,8 @@ require ( github.com/gabriel-vasile/mimetype v0.1.4 github.com/gbrlsnchs/jwt/v3 v3.0.0-rc.0 github.com/ghodss/yaml v1.0.0 - github.com/gin-gonic/gin v1.6.2 + github.com/gin-gonic/autotls v0.0.0-20200518075542-45033372a9ad + github.com/gin-gonic/gin v1.6.3 github.com/golang/protobuf v1.3.5 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.0 @@ -63,7 +64,7 @@ require ( github.com/stretchr/objx v0.2.0 // indirect github.com/yuin/goldmark v1.1.30 // indirect go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 // indirect + golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 // indirect golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a diff --git a/go.sum b/go.sum index c63597c..f9ec8af 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,12 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/autotls v0.0.0-20200518075542-45033372a9ad h1:qXUH5CUVcICSzL8DedgF39LCsrVoMFbswxByQVtd5h8= +github.com/gin-gonic/autotls v0.0.0-20200518075542-45033372a9ad/go.mod h1:yLpIL/Gol/Y5/A36WyQNCRIAVuhiuHEtITA6UQ7EghY= github.com/gin-gonic/gin v1.6.2 h1:88crIK23zO6TqlQBt+f9FrPJNKm9ZEr7qjp9vl/d5TM= github.com/gin-gonic/gin v1.6.2/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= From 65809b5731e8f2ef39117bd9514b0333fb72fc3c Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 2 Jul 2020 20:32:17 -0700 Subject: [PATCH 28/30] Don't crash when passing in an environment variable that isn't a string value --- installer/installer.go | 4 ++-- server/environment_docker.go | 32 +------------------------------ server/server.go | 37 +++++++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/installer/installer.go b/installer/installer.go index b2fdf09..37ce0cb 100644 --- a/installer/installer.go +++ b/installer/installer.go @@ -34,7 +34,7 @@ func New(data []byte) (*Installer, error) { Suspended: false, State: server.ProcessOfflineState, Invocation: getString(data, "invocation"), - EnvVars: make(map[string]string), + EnvVars: make(server.EnvironmentVariables), Build: server.BuildSettings{ MemoryLimit: getInt(data, "build", "memory"), Swap: getInt(data, "build", "swap"), @@ -55,7 +55,7 @@ func New(data []byte) (*Installer, error) { if b, _, _, err := jsonparser.Get(data, "environment"); err != nil { return nil, errors.WithStack(err) } else { - s.EnvVars = make(map[string]string) + s.EnvVars = make(server.EnvironmentVariables) if err := json.Unmarshal(b, &s.EnvVars); err != nil { return nil, errors.WithStack(err) } diff --git a/server/environment_docker.go b/server/environment_docker.go index afe0b70..14597e6 100644 --- a/server/environment_docker.go +++ b/server/environment_docker.go @@ -653,7 +653,7 @@ func (d *DockerEnvironment) Create() error { ExposedPorts: d.exposedPorts(), Image: d.Server.Container.Image, - Env: d.environmentVariables(), + Env: d.Server.GetEnvironmentVariables(), Labels: map[string]string{ "Service": "Pterodactyl", @@ -804,36 +804,6 @@ func (d *DockerEnvironment) parseLogToStrings(b []byte) ([]string, error) { return out, nil } -// Returns the environment variables for a server in KEY="VALUE" form. -func (d *DockerEnvironment) environmentVariables() []string { - zone, _ := time.Now().In(time.Local).Zone() - - var out = []string{ - fmt.Sprintf("TZ=%s", zone), - fmt.Sprintf("STARTUP=%s", d.Server.Invocation), - fmt.Sprintf("SERVER_MEMORY=%d", d.Server.Build.MemoryLimit), - fmt.Sprintf("SERVER_IP=%s", d.Server.Allocations.DefaultMapping.Ip), - fmt.Sprintf("SERVER_PORT=%d", d.Server.Allocations.DefaultMapping.Port), - } - -eloop: - for k, v := range d.Server.EnvVars { - for _, e := range out { - if strings.HasPrefix(e, strings.ToUpper(k)) { - continue eloop - } - } - - out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k), v)) - } - - return out -} - -func (d *DockerEnvironment) volumes() map[string]struct{} { - return nil -} - // Converts the server allocation mappings into a format that can be understood // by Docker. func (d *DockerEnvironment) portBindings() nat.PortMap { diff --git a/server/server.go b/server/server.go index 5389841..7b7792c 100644 --- a/server/server.go +++ b/server/server.go @@ -13,6 +13,7 @@ import ( "golang.org/x/sync/semaphore" "math" "os" + "strconv" "strings" "sync" "time" @@ -24,6 +25,36 @@ func GetServers() *Collection { return servers } +type EnvironmentVariables map[string]interface{} + +// Ugly hacky function to handle environment variables that get passed through as not-a-string +// from the Panel. Ideally we'd just say only pass strings, but that is a fragile idea and if a +// string wasn't passed through you'd cause a crash or the server to become unavailable. For now +// try to handle the most likely values from the JSON and hope for the best. +func (ev EnvironmentVariables) Get(key string) string { + val, ok := ev[key] + if !ok { + return "" + } + + switch val.(type) { + case int: + return strconv.Itoa(val.(int)) + case int32: + return strconv.FormatInt(val.(int64), 10) + case int64: + return strconv.FormatInt(val.(int64), 10) + case float32: + return fmt.Sprintf("%f", val.(float32)) + case float64: + return fmt.Sprintf("%f", val.(float64)) + case bool: + return strconv.FormatBool(val.(bool)) + } + + return val.(string) +} + // High level definition for a server instance being controlled by Wings. type Server struct { // The unique identifier for the server that should be used when referencing @@ -43,7 +74,7 @@ type Server struct { // An array of environment variables that should be passed along to the running // server process. - EnvVars map[string]string `json:"environment" yaml:"environment"` + EnvVars EnvironmentVariables `json:"environment" yaml:"environment"` Archiver Archiver `json:"-" yaml:"-"` CrashDetection CrashDetection `json:"crash_detection" yaml:"crash_detection"` @@ -287,14 +318,14 @@ func (s *Server) GetEnvironmentVariables() []string { } eloop: - for k, v := range s.EnvVars { + for k := range s.EnvVars { for _, e := range out { if strings.HasPrefix(e, strings.ToUpper(k)) { continue eloop } } - out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k), v)) + out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k), s.EnvVars.Get(k))) } return out From 82912595b78dfa2964d9602d5749de0f38a53b27 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Thu, 2 Jul 2020 21:03:11 -0700 Subject: [PATCH 29/30] Update SFTP logic for authentication to avoid brute forces; replicates logic from #9 Co-Authored-By: Stepan Fedotov --- api/sftp_endpoints.go | 4 ---- go.mod | 2 +- go.sum | 2 ++ sftp/server.go | 30 +++++++++++++++++++++++++++--- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/api/sftp_endpoints.go b/api/sftp_endpoints.go index f0f7df8..a626c07 100644 --- a/api/sftp_endpoints.go +++ b/api/sftp_endpoints.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/pkg/errors" "github.com/pterodactyl/sftp-server" - "go.uber.org/zap" ) func (r *PanelRequest) ValidateSftpCredentials(request sftp_server.AuthenticationRequest) (*sftp_server.AuthenticationResponse, error) { @@ -23,13 +22,10 @@ func (r *PanelRequest) ValidateSftpCredentials(request sftp_server.Authenticatio if r.HasError() { if r.HttpResponseCode() >= 400 && r.HttpResponseCode() < 500 { - zap.S().Debugw("failed to validate server credentials for SFTP", zap.String("error", r.Error().String())) - return nil, new(sftp_server.InvalidCredentialsError) } rerr := errors.New(r.Error().String()) - zap.S().Warnw("error validating SFTP credentials", zap.Error(rerr)) return nil, rerr } diff --git a/go.mod b/go.mod index 885a0a8..2a472a3 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.4.0 github.com/pkg/sftp v1.11.0 // indirect - github.com/pterodactyl/sftp-server v1.1.2 + github.com/pterodactyl/sftp-server v1.1.4 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 github.com/smartystreets/goconvey v1.6.4 // indirect diff --git a/go.sum b/go.sum index f9ec8af..dcbbc06 100644 --- a/go.sum +++ b/go.sum @@ -263,6 +263,8 @@ github.com/pterodactyl/sftp-server v1.1.1 h1:IjuOy21BNZxfejKnXG1RgLxXAYylDqBVpbK github.com/pterodactyl/sftp-server v1.1.1/go.mod h1:b1VVWYv0RF9rxSZQqaD/rYXriiRMNPsbV//CKMXR4ag= github.com/pterodactyl/sftp-server v1.1.2 h1:5bI9upe0kBRn9ALDabn9S2GVU5gkYvSErYgs32dAKjk= github.com/pterodactyl/sftp-server v1.1.2/go.mod h1:KjSONrenRr1oCh94QIVAU6yEzMe+Hd7r/JHrh5/oQHs= +github.com/pterodactyl/sftp-server v1.1.4 h1:JESuEuZ+d2tajMjuQblPOlGISM9Uc2xOzk7irVF9PQ0= +github.com/pterodactyl/sftp-server v1.1.4/go.mod h1:KjSONrenRr1oCh94QIVAU6yEzMe+Hd7r/JHrh5/oQHs= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY= github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/sftp/server.go b/sftp/server.go index c554ce9..8a9f243 100644 --- a/sftp/server.go +++ b/sftp/server.go @@ -8,6 +8,7 @@ import ( "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/server" "go.uber.org/zap" + "regexp" ) func Initialize(config *config.Configuration) error { @@ -70,13 +71,36 @@ func validateDiskSpace(fs sftp_server.FileSystem) bool { return s.Filesystem.HasSpaceAvailable() } +var validUsernameRegexp = regexp.MustCompile(`^(?i)(.+)\.([a-z0-9]{8})$`) + // Validates a set of credentials for a SFTP login aganist Pterodactyl Panel and returns // the server's UUID if the credentials were valid. func validateCredentials(c sftp_server.AuthenticationRequest) (*sftp_server.AuthenticationResponse, error) { - resp, err := api.NewRequester().ValidateSftpCredentials(c) - log.WithFields(log.Fields{"subsystem": "sftp", "username": c.User}).Debug("validating credentials for SFTP connection") + + f := log.Fields{ + "subsystem": "sftp", + "username": c.User, + "ip": c.IP, + } + + // If the username doesn't meet the expected format that the Panel would even recognize just go ahead + // and bail out of the process here to avoid accidentially brute forcing the panel if a bot decides + // to connect to spam username attempts. + if !validUsernameRegexp.MatchString(c.User) { + log.WithFields(f).Warn("failed to validate user credentials (invalid format)") + + return nil, new(sftp_server.InvalidCredentialsError) + } + + resp, err := api.NewRequester().ValidateSftpCredentials(c) if err != nil { + if sftp_server.IsInvalidCredentialsError(err) { + log.WithFields(f).Warn("failed to validate user credentials (invalid username or password)") + } else { + log.WithFields(f).Error("encountered an error while trying to validate user credentials") + } + return resp, err } @@ -88,7 +112,7 @@ func validateCredentials(c sftp_server.AuthenticationRequest) (*sftp_server.Auth return resp, errors.New("no matching server with UUID found") } - s.Log().WithFields(log.Fields{"subsystem": "sftp", "username": c.User}).Debug("matched user to server instance, credentials successfully validated") + s.Log().WithFields(f).Debug("credentials successfully validated and matched user to server instance") return resp, err } From 860e300c22c2064fd0bb4049ec3b102bf5715c40 Mon Sep 17 00:00:00 2001 From: Jakob Date: Sat, 4 Jul 2020 21:30:38 +0200 Subject: [PATCH 30/30] enable codeql security scanning --- .github/workflows/codeql-analysis.yml | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..d52ec6a --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,35 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 21 * * 6' + +jobs: + CodeQL-Build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + with: + languages: go + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1