[#3896bk] Fix handling of YAML/JSON when parsing configuration files

go.mod

@@ -4,6 +4,7 @@ go 1.12
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Jeffail/gabs/v2 v2.2.0
 	github.com/Microsoft/go-winio v0.4.7 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
@@ -21,6 +22,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.0
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
+	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
 	github.com/imdario/mergo v0.3.8
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/kr/pretty v0.1.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Jeffail/gabs/v2 v2.2.0 h1:7touC+WzbQ7LO5+mwgxT44miyTqAVCOlIWLA6PiIB5w=
+github.com/Jeffail/gabs/v2 v2.2.0/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
 github.com/Microsoft/go-winio v0.4.7 h1:vOvDiY/F1avSWlCWiKJjdYKz2jVjTK3pWPHndeG4OAY=
 github.com/Microsoft/go-winio v0.4.7/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
@@ -43,6 +45,8 @@ github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQ
 github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8=
+github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
 github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=

parser/parser.go

@@ -4,12 +4,14 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/Jeffail/gabs/v2"
 	"github.com/buger/jsonparser"
+	"github.com/ghodss/yaml"
+	"github.com/iancoleman/strcase"
 	"github.com/magiconair/properties"
 	"github.com/pkg/errors"
 	"github.com/pterodactyl/wings/config"
 	"go.uber.org/zap"
-	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
 	"regexp"
@@ -43,6 +45,10 @@ type ConfigurationFile struct {
 	FileName string                         `json:"file"`
 	Parser   ConfigurationParser            `json:"parser"`
 	Replace  []ConfigurationFileReplacement `json:"replace"`
+
+	// Tracks Wings' configuration so that we can quickly get values
+	// out of it when variables request it.
+	configuration []byte
 }
 
 // Defines a single find/replace instance for a given server configuration file.
@@ -80,6 +86,9 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error {
 func (f *ConfigurationFile) Parse(path string) error {
 	zap.S().Debugw("parsing configuration file", zap.String("path", path), zap.String("parser", string(f.Parser)))
 
+	mb, _ := json.Marshal(config.Get())
+	f.configuration = mb
+
 	var err error
 
 	switch f.Parser {
@@ -92,71 +101,96 @@ func (f *ConfigurationFile) Parse(path string) error {
 	case Yaml, "yml":
 		err = f.parseYamlFile(path)
 		break
+	case Json:
+		err = f.parseJsonFile(path)
+		break
 	}
 
 	return err
 }
 
-// Parses a yaml file and updates any matching key/value pairs before persisting
+// Gets the []byte representation of a configuration file to be passed through to other
-// it back to the disk.
+// handler functions. If the file does not currently exist, it will be created.
-func (f *ConfigurationFile) parseYamlFile(path string) error {
+func readFileBytes(path string) ([]byte, error) {
 	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
 	if err != nil {
-		return errors.WithStack(err)
+		return nil, err
 	}
 	defer file.Close()
 
-	b, err := ioutil.ReadAll(file)
+	return ioutil.ReadAll(file)
+}
+
+// Iterate over an unstructured JSON/YAML/etc. interface and set all of the required key/value pairs
+// for the configuration file.
+func (f *ConfigurationFile) IterateOverJson(data []byte) (*gabs.Container, error) {
+	parsed, err := gabs.ParseJSON(data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range f.Replace {
+		value, err := f.lookupConfigurationValue(v.Value)
+		if err != nil {
+			return nil, err
+		}
+
+		if _, err = parsed.SetP(value, v.Match); err != nil {
+			return nil, err
+		}
+	}
+
+	return parsed, nil
+}
+
+// Prases a json file updating any matching key/value pairs. If a match is not found, the
+// value is set regardless in the file. See the commentary in parseYamlFile for more details
+// about what is happening during this process.
+func (f *ConfigurationFile) parseJsonFile(path string) error {
+	b, err := readFileBytes(path)
 	if err != nil {
 		return errors.WithStack(err)
 	}
 
-	var raw interface{}
+	data, err := f.IterateOverJson(b)
-	// Unmarshall the yaml data into a raw interface such that we can work with any arbitrary
+	if err != nil {
-	// data structure.
-	if err := yaml.Unmarshal(b, &raw); err != nil {
 		return errors.WithStack(err)
 	}
 
-	// Create an indexable map that we can use while looping through elements.
+	output := []byte(data.StringIndent("", "    "))
-	m := raw.(map[interface{}]interface{})
+	return ioutil.WriteFile(path, output, 0644)
+}
 
-	for _, v := range f.Replace {
+// Parses a yaml file and updates any matching key/value pairs before persisting
-		value, err := lookupConfigurationValue(v.Value)
+// it back to the disk.
-		if err != nil {
+func (f *ConfigurationFile) parseYamlFile(path string) error {
-			return errors.WithStack(err)
+	b, err := readFileBytes(path)
-		}
+	if err != nil {
-
-		layer := m
-		nest := strings.Split(v.Match, ".")
-
-		// Split the key name on any periods, as we do this, initalize the struct for the yaml
-		// data at that key and then reset the later to point to that newly created layer. If
-		// we have reached the last split item, set the value of the key to the value defined
-		// in the replacement data.
-		for i, key := range nest {
-			if i == (len(nest) - 1) {
-				layer[key] = value
-			} else {
-				// Don't overwrite the key if it exists in the data already. But, if it is missing,
-				// go ahead and create the key otherwise we'll hit a panic when trying to access an
-				// index that does not exist.
-				if m[key] == nil {
-					layer[key] = make(map[interface{}]interface{})
-				}
-
-				layer = m[key].(map[interface{}]interface{})
-			}
-		}
-	}
-
-	file.Close()
-
-	if o, err := yaml.Marshal(m); err != nil {
 		return errors.WithStack(err)
-	} else {
-		return ioutil.WriteFile(path, o, 0644)
 	}
+
+	// Unmarshal the yaml data into a JSON interface such that we can work with
+	// any arbitrary data structure. If we don't do this, I can't use gabs which
+	// makes working with unknown JSON signficiantly easier.
+	jsonBytes, err := yaml.YAMLToJSON(b)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	// Now that the data is converted, treat it just like JSON and pass it to the
+	// iterator function to update values as necessary.
+	data, err := f.IterateOverJson(jsonBytes)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	// Remarshal the JSON into YAML format before saving it back to the disk.
+	marshaled, err := yaml.JSONToYAML(data.Bytes())
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return ioutil.WriteFile(path, marshaled, 0644)
 }
 
 // Parses a text file using basic find and replace. This is a highly inefficient method of
@@ -210,7 +244,7 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error {
 	}
 
 	for _, replace := range f.Replace {
-		v, err := lookupConfigurationValue(replace.Value)
+		v, err := f.lookupConfigurationValue(replace.Value)
 		if err != nil {
 			return errors.WithStack(err)
 		}
@@ -231,10 +265,7 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error {
 }
 
 // Looks up a configuration value on the Daemon given a dot-notated syntax.
-func lookupConfigurationValue(value string) (string, error) {
+func (f *ConfigurationFile) lookupConfigurationValue(value string) (string, error) {
-	// @todo there is probably a much better way to handle this
-	mb, _ := json.Marshal(config.Get())
-
 	if !configMatchRegex.Match([]byte(value)) {
 		return value, nil
 	}
@@ -244,7 +275,17 @@ func lookupConfigurationValue(value string) (string, error) {
 	// daemon configuration here.
 	v := configMatchRegex.ReplaceAllString(value, "$1")
 
-	match, err := jsonparser.GetString(mb, strings.Split(v, ".")...)
+	var path []string
+	// The camel casing is important here, the configuration for the Daemon does not use
+	// JSON, and as such all of the keys will be generated in CamelCase format, rather than
+	// the expected snake_case from the old Daemon.
+	for _, value := range strings.Split(v, ".")	{
+		path = append(path, strcase.ToCamel(value))
+	}
+
+	// Look for the key in the configuration file, and if found return that value to the
+	// calling function.
+	match, err := jsonparser.GetString(f.configuration, path...)
 	if err != nil {
 		if err != jsonparser.KeyPathNotFoundError {
 			return "", errors.WithStack(err)