Replace encoding/json with goccy/go-json for cpu and memory usage improvement

This new package has significant better resource usage, and we do a _lot_ of JSON parsing in this application, so any amount of improvement becomes significant
This commit is contained in:
Dane Everitt 2022-01-23 15:17:40 -05:00
parent 301788805c
commit 34c0db9dff
21 changed files with 39 additions and 43 deletions

View File

@ -2,7 +2,6 @@ package cmd
import ( import (
"crypto/tls" "crypto/tls"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -14,6 +13,7 @@ import (
"github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2"
"github.com/AlecAivazis/survey/v2/terminal" "github.com/AlecAivazis/survey/v2/terminal"
"github.com/goccy/go-json"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"

View File

@ -2,7 +2,6 @@ package cmd
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -20,6 +19,7 @@ import (
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/parsers/kernel" "github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/parsers/operatingsystem" "github.com/docker/docker/pkg/parsers/operatingsystem"
"github.com/goccy/go-json"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"

View File

@ -2,10 +2,10 @@ package config
import ( import (
"encoding/base64" "encoding/base64"
"encoding/json"
"sort" "sort"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/goccy/go-json"
) )
type dockerNetworkInterfaces struct { type dockerNetworkInterfaces struct {

View File

@ -3,7 +3,6 @@ package docker
import ( import (
"bufio" "bufio"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"strconv" "strconv"
@ -12,6 +11,7 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/apex/log" "github.com/apex/log"
"github.com/buger/jsonparser"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/mount"
@ -364,11 +364,6 @@ func (e *Environment) scanOutput(reader io.ReadCloser) {
go e.followOutput() go e.followOutput()
} }
type imagePullStatus struct {
Status string `json:"status"`
Progress string `json:"progress"`
}
// Pulls the image from Docker. If there is an error while pulling the image // Pulls the image from Docker. If there is an error while pulling the image
// from the source but the image already exists locally, we will report that // from the source but the image already exists locally, we will report that
// error to the logger but continue with the process. // error to the logger but continue with the process.
@ -454,12 +449,11 @@ func (e *Environment) ensureImageExists(image string) error {
scanner := bufio.NewScanner(out) scanner := bufio.NewScanner(out)
for scanner.Scan() { for scanner.Scan() {
s := imagePullStatus{} b := scanner.Bytes()
fmt.Println(scanner.Text()) status, _ := jsonparser.GetString(b, "status")
progress, _ := jsonparser.GetString(b, "progress")
if err := json.Unmarshal(scanner.Bytes(), &s); err == nil { e.Events().Publish(environment.DockerImagePullStatus, status+" "+progress)
e.Events().Publish(environment.DockerImagePullStatus, s.Status+" "+s.Progress)
}
} }
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {

View File

@ -2,13 +2,13 @@ package docker
import ( import (
"context" "context"
"encoding/json"
"io" "io"
"math" "math"
"time" "time"
"emperror.dev/errors" "emperror.dev/errors"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
) )

5
go.mod
View File

@ -45,6 +45,10 @@ require (
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
) )
require github.com/goccy/go-json v0.9.4
require golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
require ( require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect
@ -102,7 +106,6 @@ require (
go.uber.org/atomic v1.9.0 // indirect go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.6 // indirect golang.org/x/text v0.3.6 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect

5
go.sum
View File

@ -371,6 +371,8 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k= github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k=
github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-json v0.9.4 h1:L8MLKG2mvVXiQu07qB6hmfqeSYQdOnqPot2GhsIwIaI=
github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@ -1111,8 +1113,9 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=

View File

@ -2,7 +2,6 @@ package parser
import ( import (
"bufio" "bufio"
"encoding/json"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
@ -14,6 +13,7 @@ import (
"github.com/buger/jsonparser" "github.com/buger/jsonparser"
"github.com/icza/dyno" "github.com/icza/dyno"
"github.com/magiconair/properties" "github.com/magiconair/properties"
"github.com/goccy/go-json"
"gopkg.in/ini.v1" "gopkg.in/ini.v1"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
@ -80,8 +80,8 @@ func (cp ConfigurationParser) String() string {
return string(cp) return string(cp)
} }
// Defines a configuration file for the server startup. These will be looped over // ConfigurationFile defines a configuration file for the server startup. These
// and modified before the server finishes booting. // will be looped over and modified before the server finishes booting.
type ConfigurationFile struct { type ConfigurationFile struct {
FileName string `json:"file"` FileName string `json:"file"`
Parser ConfigurationParser `json:"parser"` Parser ConfigurationParser `json:"parser"`
@ -92,12 +92,10 @@ type ConfigurationFile struct {
configuration []byte configuration []byte
} }
// Custom unmarshaler for configuration files. If there is an error while parsing out the // UnmarshalJSON is a custom unmarshaler for configuration files. If there is an
// replacements, don't fail the entire operation, just log a global warning so someone can // error while parsing out the replacements, don't fail the entire operation,
// find the issue, and return an empty array of replacements. // just log a global warning so someone can find the issue, and return an empty
// // array of replacements.
// I imagine people will notice configuration replacement isn't working correctly and then
// the logs should help better expose that issue.
func (f *ConfigurationFile) UnmarshalJSON(data []byte) error { func (f *ConfigurationFile) UnmarshalJSON(data []byte) error {
var m map[string]*json.RawMessage var m map[string]*json.RawMessage
if err := json.Unmarshal(data, &m); err != nil { if err := json.Unmarshal(data, &m); err != nil {

View File

@ -3,7 +3,6 @@ package remote
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@ -14,6 +13,7 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/apex/log" "github.com/apex/log"
"github.com/cenkalti/backoff/v4" "github.com/cenkalti/backoff/v4"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/system" "github.com/pterodactyl/wings/system"
) )

View File

@ -1,11 +1,11 @@
package remote package remote
import ( import (
"encoding/json"
"regexp" "regexp"
"strings" "strings"
"github.com/apex/log" "github.com/apex/log"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/parser" "github.com/pterodactyl/wings/parser"
) )

View File

@ -2,7 +2,6 @@ package downloader
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net" "net"
@ -15,6 +14,7 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/server"
) )

View File

@ -2,11 +2,11 @@ package router
import ( import (
"context" "context"
"encoding/json"
"time" "time"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
ws "github.com/gorilla/websocket" ws "github.com/gorilla/websocket"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/router/middleware" "github.com/pterodactyl/wings/router/middleware"
"github.com/pterodactyl/wings/router/websocket" "github.com/pterodactyl/wings/router/websocket"

View File

@ -1,13 +1,13 @@
package tokens package tokens
import ( import (
"encoding/json"
"strings" "strings"
"sync" "sync"
"time" "time"
"github.com/apex/log" "github.com/apex/log"
"github.com/gbrlsnchs/jwt/v3" "github.com/gbrlsnchs/jwt/v3"
"github.com/goccy/go-json"
) )
// The time at which Wings was booted. No JWT's created before this time are allowed to // The time at which Wings was booted. No JWT's created before this time are allowed to

View File

@ -2,11 +2,12 @@ package websocket
import ( import (
"context" "context"
"encoding/json"
"sync" "sync"
"time" "time"
"emperror.dev/errors" "emperror.dev/errors"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/events" "github.com/pterodactyl/wings/events"
"github.com/pterodactyl/wings/server" "github.com/pterodactyl/wings/server"
) )

View File

@ -2,7 +2,6 @@ package websocket
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"strings" "strings"
@ -14,6 +13,7 @@ import (
"github.com/gbrlsnchs/jwt/v3" "github.com/gbrlsnchs/jwt/v3"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"

View File

@ -1,12 +1,12 @@
package filesystem package filesystem
import ( import (
"encoding/json"
"os" "os"
"strconv" "strconv"
"time" "time"
"github.com/gabriel-vasile/mimetype" "github.com/gabriel-vasile/mimetype"
"github.com/goccy/go-json"
) )
type Stat struct { type Stat struct {

View File

@ -125,7 +125,7 @@ func (s *Server) StartEventListeners() {
l.Trigger() l.Trigger()
} }
s.emitProcUsage() s.Events().Publish(StatsEvent, s.Proc())
}() }()
case e := <-docker: case e := <-docker:
go func() { go func() {

View File

@ -2,7 +2,6 @@ package server
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"os" "os"
@ -14,6 +13,7 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/apex/log" "github.com/apex/log"
"github.com/gammazero/workerpool" "github.com/gammazero/workerpool"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"

View File

@ -50,7 +50,3 @@ func (ru *ResourceUsage) Reset() {
ru.Network.TxBytes = 0 ru.Network.TxBytes = 0
ru.Network.RxBytes = 0 ru.Network.RxBytes = 0
} }
func (s *Server) emitProcUsage() {
s.Events().Publish(StatsEvent, s.Proc())
}

View File

@ -2,7 +2,6 @@ package server
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"os" "os"
@ -12,6 +11,8 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/apex/log" "github.com/apex/log"
"github.com/creasty/defaults" "github.com/creasty/defaults"
"github.com/goccy/go-json"
"github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/environment" "github.com/pterodactyl/wings/environment"
"github.com/pterodactyl/wings/events" "github.com/pterodactyl/wings/events"
@ -312,7 +313,7 @@ func (s *Server) OnStateChange() {
// views in the Panel correctly display 0. // views in the Panel correctly display 0.
if st == environment.ProcessOfflineState { if st == environment.ProcessOfflineState {
s.resources.Reset() s.resources.Reset()
s.emitProcUsage() s.Events().Publish(StatsEvent, s.Proc())
} }
// If server was in an online state, and is now in an offline state we should handle // If server was in an online state, and is now in an offline state we should handle

View File

@ -4,7 +4,6 @@ import (
"bufio" "bufio"
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"strconv" "strconv"
@ -12,6 +11,7 @@ import (
"time" "time"
"emperror.dev/errors" "emperror.dev/errors"
"github.com/goccy/go-json"
) )
var ( var (