MCorange/wings
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#3896bk] Fix handling of YAML/JSON when parsing configuration files

Browse Source
This commit is contained in:
Dane Everitt 2019-12-01 11:19:49 -08:00
parent 0e51db1c03
commit cae0c70b0f
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
3 changed files with 99 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -4,6 +4,7 @@ go 1.12
require (
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/Jeffail/gabs/v2 v2.2.0
github.com/Microsoft/go-winio v0.4.7 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
@ -21,6 +22,7 @@ require (
github.com/google/uuid v1.1.1
github.com/gorilla/websocket v1.4.0
github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
github.com/imdario/mergo v0.3.8
github.com/julienschmidt/httprouter v1.2.0
github.com/kr/pretty v0.1.0 // indirect

4
go.sum
View File

@ -1,5 +1,7 @@
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Jeffail/gabs/v2 v2.2.0 h1:7touC+WzbQ7LO5+mwgxT44miyTqAVCOlIWLA6PiIB5w=
github.com/Jeffail/gabs/v2 v2.2.0/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
github.com/Microsoft/go-winio v0.4.7 h1:vOvDiY/F1avSWlCWiKJjdYKz2jVjTK3pWPHndeG4OAY=
github.com/Microsoft/go-winio v0.4.7/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
@ -43,6 +45,8 @@ github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQ
github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=

145
parser/parser.go
View File

@ -4,12 +4,14 @@ import (
"bufio"
"encoding/json"
"fmt"
"github.com/Jeffail/gabs/v2"
"github.com/buger/jsonparser"
"github.com/ghodss/yaml"
"github.com/iancoleman/strcase"
"github.com/magiconair/properties"
"github.com/pkg/errors"
"github.com/pterodactyl/wings/config"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
"io/ioutil"
"os"
"regexp"
@ -43,6 +45,10 @@ type ConfigurationFile struct {
FileName string `json:"file"`
Parser ConfigurationParser `json:"parser"`
Replace []ConfigurationFileReplacement `json:"replace"`
// Tracks Wings' configuration so that we can quickly get values
// out of it when variables request it.
configuration []byte
}
// Defines a single find/replace instance for a given server configuration file.
@ -80,6 +86,9 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error {
func (f *ConfigurationFile) Parse(path string) error {
zap.S().Debugw("parsing configuration file", zap.String("path", path), zap.String("parser", string(f.Parser)))
mb, _ := json.Marshal(config.Get())
f.configuration = mb
var err error
switch f.Parser {
@ -92,71 +101,96 @@ func (f *ConfigurationFile) Parse(path string) error {
case Yaml, "yml":
err = f.parseYamlFile(path)
break
case Json:
err = f.parseJsonFile(path)
break
}
return err
}
// Parses a yaml file and updates any matching key/value pairs before persisting
// it back to the disk.
func (f *ConfigurationFile) parseYamlFile(path string) error {
// Gets the []byte representation of a configuration file to be passed through to other
// handler functions. If the file does not currently exist, it will be created.
func readFileBytes(path string) ([]byte, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return errors.WithStack(err)
return nil, err
}
defer file.Close()
b, err := ioutil.ReadAll(file)
return ioutil.ReadAll(file)
}
// Iterate over an unstructured JSON/YAML/etc. interface and set all of the required key/value pairs
// for the configuration file.
func (f *ConfigurationFile) IterateOverJson(data []byte) (*gabs.Container, error) {
parsed, err := gabs.ParseJSON(data)
if err != nil {
return nil, err
}
for _, v := range f.Replace {
value, err := f.lookupConfigurationValue(v.Value)
if err != nil {
return nil, err
}
if _, err = parsed.SetP(value, v.Match); err != nil {
return nil, err
}
}
return parsed, nil
}
// Prases a json file updating any matching key/value pairs. If a match is not found, the
// value is set regardless in the file. See the commentary in parseYamlFile for more details
// about what is happening during this process.
func (f *ConfigurationFile) parseJsonFile(path string) error {
b, err := readFileBytes(path)
if err != nil {
return errors.WithStack(err)
}
var raw interface{}
// Unmarshall the yaml data into a raw interface such that we can work with any arbitrary
// data structure.
if err := yaml.Unmarshal(b, &raw); err != nil {
data, err := f.IterateOverJson(b)
if err != nil {
return errors.WithStack(err)
}
// Create an indexable map that we can use while looping through elements.
m := raw.(map[interface{}]interface{})
output := []byte(data.StringIndent("", " "))
return ioutil.WriteFile(path, output, 0644)
}
for _, v := range f.Replace {
value, err := lookupConfigurationValue(v.Value)
if err != nil {
return errors.WithStack(err)
}
layer := m
nest := strings.Split(v.Match, ".")
// Split the key name on any periods, as we do this, initalize the struct for the yaml
// data at that key and then reset the later to point to that newly created layer. If
// we have reached the last split item, set the value of the key to the value defined
// in the replacement data.
for i, key := range nest {
if i == (len(nest) - 1) {
layer[key] = value
} else {
// Don't overwrite the key if it exists in the data already. But, if it is missing,
// go ahead and create the key otherwise we'll hit a panic when trying to access an
// index that does not exist.
if m[key] == nil {
layer[key] = make(map[interface{}]interface{})
}
layer = m[key].(map[interface{}]interface{})
}
}
}
file.Close()
if o, err := yaml.Marshal(m); err != nil {
// Parses a yaml file and updates any matching key/value pairs before persisting
// it back to the disk.
func (f *ConfigurationFile) parseYamlFile(path string) error {
b, err := readFileBytes(path)
if err != nil {
return errors.WithStack(err)
} else {
return ioutil.WriteFile(path, o, 0644)
}
// Unmarshal the yaml data into a JSON interface such that we can work with
// any arbitrary data structure. If we don't do this, I can't use gabs which
// makes working with unknown JSON signficiantly easier.
jsonBytes, err := yaml.YAMLToJSON(b)
if err != nil {
return errors.WithStack(err)
}
// Now that the data is converted, treat it just like JSON and pass it to the
// iterator function to update values as necessary.
data, err := f.IterateOverJson(jsonBytes)
if err != nil {
return errors.WithStack(err)
}
// Remarshal the JSON into YAML format before saving it back to the disk.
marshaled, err := yaml.JSONToYAML(data.Bytes())
if err != nil {
return errors.WithStack(err)
}
return ioutil.WriteFile(path, marshaled, 0644)
}
// Parses a text file using basic find and replace. This is a highly inefficient method of
@ -210,7 +244,7 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error {
}
for _, replace := range f.Replace {
v, err := lookupConfigurationValue(replace.Value)
v, err := f.lookupConfigurationValue(replace.Value)
if err != nil {
return errors.WithStack(err)
}
@ -231,10 +265,7 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error {
}
// Looks up a configuration value on the Daemon given a dot-notated syntax.
func lookupConfigurationValue(value string) (string, error) {
// @todo there is probably a much better way to handle this
mb, _ := json.Marshal(config.Get())
func (f *ConfigurationFile) lookupConfigurationValue(value string) (string, error) {
if !configMatchRegex.Match([]byte(value)) {
return value, nil
}
@ -244,7 +275,17 @@ func lookupConfigurationValue(value string) (string, error) {
// daemon configuration here.
v := configMatchRegex.ReplaceAllString(value, "$1")
match, err := jsonparser.GetString(mb, strings.Split(v, ".")...)
var path []string
// The camel casing is important here, the configuration for the Daemon does not use
// JSON, and as such all of the keys will be generated in CamelCase format, rather than
// the expected snake_case from the old Daemon.
for _, value := range strings.Split(v, ".") {
path = append(path, strcase.ToCamel(value))
}
// Look for the key in the configuration file, and if found return that value to the
// calling function.
match, err := jsonparser.GetString(f.configuration, path...)
if err != nil {
if err != jsonparser.KeyPathNotFoundError {
return "", errors.WithStack(err)