Potential fix for console becoming unresponsive (#55)

* Potentially fix console logs not being pulled after a server has been running for a while

* Add container_id to resource polling debug logs
This commit is contained in:
Matthew Penner 2020-09-07 14:04:56 -06:00 committed by GitHub
parent 7ba32aca84
commit a81146d730
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 31 additions and 44 deletions

View File

@ -15,6 +15,9 @@ import (
// Attach to the instance and then automatically emit an event whenever the resource usage for the
// server process changes.
func (e *Environment) pollResources(ctx context.Context) error {
log.WithField("container_id", e.Id).Debug("starting resource polling..")
defer log.WithField("container_id", e.Id).Debug("resource polling stopped")
if e.State() == environment.ProcessOfflineState {
return errors.New("attempting to enable resource polling on a stopped server instance")
}
@ -23,6 +26,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
if err != nil {
return errors.WithStack(err)
}
defer stats.Body.Close()
dec := json.NewDecoder(stats.Body)
@ -30,6 +34,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
var v *types.StatsJSON
@ -45,7 +50,6 @@ func (e *Environment) pollResources(ctx context.Context) error {
// Disable collection if the server is in an offline state and this process is still running.
if e.State() == environment.ProcessOfflineState {
log.WithField("container_id", e.Id).Debug("process in offline state while resource polling is still active; stopping poll")
return nil
}

View File

@ -7,8 +7,7 @@ import (
"encoding/json"
"github.com/docker/docker/api/types"
"github.com/pkg/errors"
"io"
"os"
"strconv"
)
type dockerLogLine struct {
@ -38,44 +37,25 @@ func (e *Environment) SendCommand(c string) error {
// Reads the log file for the server. This does not care if the server is running or not, it will
// simply try to read the last X bytes of the file and return them.
func (e *Environment) Readlog(len int64) ([]string, error) {
j, err := e.client.ContainerInspect(context.Background(), e.Id)
func (e *Environment) Readlog(lines int) ([]string, error) {
r, err := e.client.ContainerLogs(context.Background(), e.Id, types.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
Tail: strconv.Itoa(lines),
})
if err != nil {
return nil, err
return nil, errors.WithStack(err)
}
defer r.Close()
var out []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
out = append(out, scanner.Text())
}
if j.LogPath == "" {
return nil, errors.New("empty log path defined for server")
}
f, err := os.Open(j.LogPath)
if err != nil {
return nil, err
}
defer f.Close()
// Check if the length of the file is smaller than the amount of data that was requested
// for reading. If so, adjust the length to be the total length of the file. If this is not
// done an error is thrown since we're reading backwards, and not forwards.
if stat, err := os.Stat(j.LogPath); err != nil {
return nil, err
} else if stat.Size() < len {
len = stat.Size()
}
// Seed to the end of the file and then move backwards until the length is met to avoid
// reading the entirety of the file into memory.
if _, err := f.Seek(-len, io.SeekEnd); err != nil {
return nil, err
}
b := make([]byte, len)
if _, err := f.Read(b); err != nil && err != io.EOF {
return nil, err
}
return e.parseLogToStrings(b)
return out, nil
}
// Docker stores the logs for server output in a JSON format. This function will iterate over the JSON
@ -87,6 +67,7 @@ func (e *Environment) parseLogToStrings(b []byte) ([]string, error) {
scanner := bufio.NewScanner(bytes.NewReader(b))
for scanner.Scan() {
var l dockerLogLine
// Unmarshal the contents and allow up to a single error before bailing out of the process. We
// do this because if you're arbitrarily reading a length of the file you'll likely end up
// with the first line in the output being improperly formatted JSON. In those cases we want to

View File

@ -89,6 +89,6 @@ type ProcessEnvironment interface {
SendCommand(string) error
// Reads the log file for the process from the end backwards until the provided
// number of bytes is met.
Readlog(int64) ([]string, error)
// number of lines is met.
Readlog(int) ([]string, error)
}

View File

@ -31,9 +31,11 @@ func getServer(c *gin.Context) {
func getServerLogs(c *gin.Context) {
s := GetServer(c.Param("server"))
l, _ := strconv.ParseInt(c.DefaultQuery("size", "8192"), 10, 64)
l, _ := strconv.Atoi(c.DefaultQuery("size", "100"))
if l <= 0 {
l = 2048
l = 100
} else if l > 100 {
l = 100
}
out, err := s.ReadLogfile(l)

View File

@ -144,7 +144,7 @@ func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) err
}
// Reads the log file for a server up to a specified number of bytes.
func (s *Server) ReadLogfile(len int64) ([]string, error) {
func (s *Server) ReadLogfile(len int) ([]string, error) {
return s.Environment.Readlog(len)
}
@ -182,4 +182,4 @@ func (s *Server) ProcessConfiguration() *api.ProcessConfiguration {
defer s.RUnlock()
return s.procConfig
}
}