Potential fix for console becoming unresponsive (#55)
* Potentially fix console logs not being pulled after a server has been running for a while * Add container_id to resource polling debug logs
This commit is contained in:
parent
7ba32aca84
commit
a81146d730
|
@ -15,6 +15,9 @@ import (
|
||||||
// Attach to the instance and then automatically emit an event whenever the resource usage for the
|
// Attach to the instance and then automatically emit an event whenever the resource usage for the
|
||||||
// server process changes.
|
// server process changes.
|
||||||
func (e *Environment) pollResources(ctx context.Context) error {
|
func (e *Environment) pollResources(ctx context.Context) error {
|
||||||
|
log.WithField("container_id", e.Id).Debug("starting resource polling..")
|
||||||
|
defer log.WithField("container_id", e.Id).Debug("resource polling stopped")
|
||||||
|
|
||||||
if e.State() == environment.ProcessOfflineState {
|
if e.State() == environment.ProcessOfflineState {
|
||||||
return errors.New("attempting to enable resource polling on a stopped server instance")
|
return errors.New("attempting to enable resource polling on a stopped server instance")
|
||||||
}
|
}
|
||||||
|
@ -23,6 +26,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.WithStack(err)
|
return errors.WithStack(err)
|
||||||
}
|
}
|
||||||
|
defer stats.Body.Close()
|
||||||
|
|
||||||
dec := json.NewDecoder(stats.Body)
|
dec := json.NewDecoder(stats.Body)
|
||||||
|
|
||||||
|
@ -30,6 +34,7 @@ func (e *Environment) pollResources(ctx context.Context) error {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
|
|
||||||
default:
|
default:
|
||||||
var v *types.StatsJSON
|
var v *types.StatsJSON
|
||||||
|
|
||||||
|
@ -45,7 +50,6 @@ func (e *Environment) pollResources(ctx context.Context) error {
|
||||||
// Disable collection if the server is in an offline state and this process is still running.
|
// Disable collection if the server is in an offline state and this process is still running.
|
||||||
if e.State() == environment.ProcessOfflineState {
|
if e.State() == environment.ProcessOfflineState {
|
||||||
log.WithField("container_id", e.Id).Debug("process in offline state while resource polling is still active; stopping poll")
|
log.WithField("container_id", e.Id).Debug("process in offline state while resource polling is still active; stopping poll")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,8 +7,7 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"io"
|
"strconv"
|
||||||
"os"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type dockerLogLine struct {
|
type dockerLogLine struct {
|
||||||
|
@ -38,44 +37,25 @@ func (e *Environment) SendCommand(c string) error {
|
||||||
|
|
||||||
// Reads the log file for the server. This does not care if the server is running or not, it will
|
// Reads the log file for the server. This does not care if the server is running or not, it will
|
||||||
// simply try to read the last X bytes of the file and return them.
|
// simply try to read the last X bytes of the file and return them.
|
||||||
func (e *Environment) Readlog(len int64) ([]string, error) {
|
func (e *Environment) Readlog(lines int) ([]string, error) {
|
||||||
j, err := e.client.ContainerInspect(context.Background(), e.Id)
|
r, err := e.client.ContainerLogs(context.Background(), e.Id, types.ContainerLogsOptions{
|
||||||
|
ShowStdout: true,
|
||||||
|
ShowStderr: true,
|
||||||
|
Tail: strconv.Itoa(lines),
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
|
||||||
|
var out []string
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(r)
|
||||||
|
for scanner.Scan() {
|
||||||
|
out = append(out, scanner.Text())
|
||||||
}
|
}
|
||||||
|
|
||||||
if j.LogPath == "" {
|
return out, nil
|
||||||
return nil, errors.New("empty log path defined for server")
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.Open(j.LogPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
// Check if the length of the file is smaller than the amount of data that was requested
|
|
||||||
// for reading. If so, adjust the length to be the total length of the file. If this is not
|
|
||||||
// done an error is thrown since we're reading backwards, and not forwards.
|
|
||||||
if stat, err := os.Stat(j.LogPath); err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else if stat.Size() < len {
|
|
||||||
len = stat.Size()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Seed to the end of the file and then move backwards until the length is met to avoid
|
|
||||||
// reading the entirety of the file into memory.
|
|
||||||
if _, err := f.Seek(-len, io.SeekEnd); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
b := make([]byte, len)
|
|
||||||
|
|
||||||
if _, err := f.Read(b); err != nil && err != io.EOF {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return e.parseLogToStrings(b)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Docker stores the logs for server output in a JSON format. This function will iterate over the JSON
|
// Docker stores the logs for server output in a JSON format. This function will iterate over the JSON
|
||||||
|
@ -87,6 +67,7 @@ func (e *Environment) parseLogToStrings(b []byte) ([]string, error) {
|
||||||
scanner := bufio.NewScanner(bytes.NewReader(b))
|
scanner := bufio.NewScanner(bytes.NewReader(b))
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
var l dockerLogLine
|
var l dockerLogLine
|
||||||
|
|
||||||
// Unmarshal the contents and allow up to a single error before bailing out of the process. We
|
// Unmarshal the contents and allow up to a single error before bailing out of the process. We
|
||||||
// do this because if you're arbitrarily reading a length of the file you'll likely end up
|
// do this because if you're arbitrarily reading a length of the file you'll likely end up
|
||||||
// with the first line in the output being improperly formatted JSON. In those cases we want to
|
// with the first line in the output being improperly formatted JSON. In those cases we want to
|
||||||
|
|
|
@ -89,6 +89,6 @@ type ProcessEnvironment interface {
|
||||||
SendCommand(string) error
|
SendCommand(string) error
|
||||||
|
|
||||||
// Reads the log file for the process from the end backwards until the provided
|
// Reads the log file for the process from the end backwards until the provided
|
||||||
// number of bytes is met.
|
// number of lines is met.
|
||||||
Readlog(int64) ([]string, error)
|
Readlog(int) ([]string, error)
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,9 +31,11 @@ func getServer(c *gin.Context) {
|
||||||
func getServerLogs(c *gin.Context) {
|
func getServerLogs(c *gin.Context) {
|
||||||
s := GetServer(c.Param("server"))
|
s := GetServer(c.Param("server"))
|
||||||
|
|
||||||
l, _ := strconv.ParseInt(c.DefaultQuery("size", "8192"), 10, 64)
|
l, _ := strconv.Atoi(c.DefaultQuery("size", "100"))
|
||||||
if l <= 0 {
|
if l <= 0 {
|
||||||
l = 2048
|
l = 100
|
||||||
|
} else if l > 100 {
|
||||||
|
l = 100
|
||||||
}
|
}
|
||||||
|
|
||||||
out, err := s.ReadLogfile(l)
|
out, err := s.ReadLogfile(l)
|
||||||
|
|
|
@ -144,7 +144,7 @@ func (s *Server) SyncWithConfiguration(cfg *api.ServerConfigurationResponse) err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reads the log file for a server up to a specified number of bytes.
|
// Reads the log file for a server up to a specified number of bytes.
|
||||||
func (s *Server) ReadLogfile(len int64) ([]string, error) {
|
func (s *Server) ReadLogfile(len int) ([]string, error) {
|
||||||
return s.Environment.Readlog(len)
|
return s.Environment.Readlog(len)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -182,4 +182,4 @@ func (s *Server) ProcessConfiguration() *api.ProcessConfiguration {
|
||||||
defer s.RUnlock()
|
defer s.RUnlock()
|
||||||
|
|
||||||
return s.procConfig
|
return s.procConfig
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user