Better configuration loading process for servers

Loads up to 10 configurations at once in parallel

config.go

@@ -1,6 +1,7 @@
-package wings
+package main
 
 import (
+	"github.com/pterodactyl/wings/environment"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
@@ -15,7 +16,7 @@ type Configuration struct {
 	Data string
 
 	Api    *ApiConfiguration
-	Docker *DockerConfiguration
+	Docker *environment.DockerConfiguration
 
 	// Determines if permissions for a server should be set automatically on
 	// daemon boot. This can take a long time on systems with many servers, or on
@@ -78,39 +79,6 @@ type ApiConfiguration struct {
 	UploadLimit int `yaml:"upload_limit"`
 }
 
-// Defines the docker configuration used by the daemon when interacting with
-// containers and networks on the system.
-type DockerConfiguration struct {
-	Container struct {
-		User string
-	}
-
-	// Network configuration that should be used when creating a new network
-	// for containers run through the daemon.
-	Network struct {
-		// The interface that should be used to create the network. Must not conflict
-		// with any other interfaces in use by Docker or on the system.
-		Interface string
-
-		// The name of the network to use. If this network already exists it will not
-		// be created. If it is not found, a new network will be created using the interface
-		// defined.
-		Name string
-	}
-
-	// If true, container images will be updated when a server starts if there
-	// is an update available. If false the daemon will not attempt updates and will
-	// defer to the host system to manage image updates.
-	UpdateImages bool `yaml:"update_images"`
-
-	// The location of the Docker socket.
-	Socket string
-
-	// Defines the location of the timezone file on the host system that should
-	// be mounted into the created containers so that they all use the same time.
-	TimezonePath string `yaml:"timezone_path"`
-}
-
 // Configures the default values for many of the configuration options present
 // in the structs. If these values are set in the configuration file they will
 // be overridden. However, if they don't exist and we don't set these here, all
@@ -146,7 +114,7 @@ func (c *Configuration) SetDefaults() {
 	c.Throttles.CheckInterval = 100
 
 	// Configure the defaults for Docker connection and networks.
-	c.Docker = &DockerConfiguration{}
+	c.Docker = &environment.DockerConfiguration{}
 	c.Docker.UpdateImages = true
 	c.Docker.Socket = "/var/run/docker.sock"
 	c.Docker.Network.Name = "pterodactyl_nw"

const.go

@@ -1,4 +1,4 @@
-package wings
+package main
 
 import "os"
 

environment/docker.go

@@ -1,16 +1,46 @@
 package environment
 
 import (
-	"github.com/pterodactyl/wings"
 	"os"
 )
 
-type Docker struct {
+// Defines the docker configuration used by the daemon when interacting with
-	*Controller
+// containers and networks on the system.
+type DockerConfiguration struct {
+	Container struct {
+		User string
+	}
 
+	// Network configuration that should be used when creating a new network
+	// for containers run through the daemon.
+	Network struct {
+		// The interface that should be used to create the network. Must not conflict
+		// with any other interfaces in use by Docker or on the system.
+		Interface string
+
+		// The name of the network to use. If this network already exists it will not
+		// be created. If it is not found, a new network will be created using the interface
+		// defined.
+		Name string
+	}
+
+	// If true, container images will be updated when a server starts if there
+	// is an update available. If false the daemon will not attempt updates and will
+	// defer to the host system to manage image updates.
+	UpdateImages bool `yaml:"update_images"`
+
+	// The location of the Docker socket.
+	Socket string
+
+	// Defines the location of the timezone file on the host system that should
+	// be mounted into the created containers so that they all use the same time.
+	TimezonePath string `yaml:"timezone_path"`
+}
+
+type Docker struct {
 	// Defines the configuration for the Docker instance that will allow us to connect
 	// and create and modify containers.
-	Configuration wings.DockerConfiguration
+	Configuration DockerConfiguration
 }
 
 // Ensure that the Docker environment is always implementing all of the methods

environment/environment.go

@@ -1,7 +1,6 @@
 package environment
 
 import (
-	"github.com/pterodactyl/wings/server"
 	"os"
 )
 
@@ -24,12 +23,4 @@ type Environment interface {
 	// Terminates a running server instance using the provided signal. If the server
 	// is not running no error should be returned.
 	Terminate(signal os.Signal) error
-}
+}
-
-// Defines an environment controller for a server instance. This can either be
-// a docker environment where the server is running in a container, or a host
-// CLI environment where it is not running in a container at all (theoretically).
-type Controller struct {
-	// The server instance attached to this environment.
-	Server *server.Server
-}

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/pelletier/go-toml v1.1.0 // indirect
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce // indirect
+	github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce
 	github.com/rifflock/lfshook v0.0.0-20180227222202-bf539943797a // indirect
 	github.com/shirou/gopsutil v0.0.0-20180227225847-5776ff9c7c5d
 	github.com/sirupsen/logrus v1.0.5

go.sum

@@ -30,11 +30,8 @@ github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTM
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/hashicorp/hcl v0.0.0-20180320202055-f40e974e75af h1:g6buE1uPu/jBP1YRkIqjoqBz5CraM5TukA+MmymQQXA=
 github.com/hashicorp/hcl v0.0.0-20180320202055-f40e974e75af/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/lestrrat-go/file-rotatelogs v2.1.0+incompatible h1:UPkGVYGAK580uBFe7gLbQ0/GjZ3T1h3QQOrvkRB7Bjw=
 github.com/lestrrat-go/file-rotatelogs v2.1.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
-github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 h1:Bvq8AziQ5jFF4BHGAEDSqwPW1NJS3XshxbRCxtjFAZc=
 github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042/go.mod h1:TPpsiPUEh0zFL1Snz4crhMlBe60PYxRHr5oFF3rRYg0=
 github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54=
 github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -54,7 +51,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce h1:aP+C+YbHZfOQlutA4p4soHi7rVUqHQdWEVMSkHfDTqY=
 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
-github.com/rifflock/lfshook v0.0.0-20180227222202-bf539943797a h1:zOAPcZGA4TZZv8zEI+uqbYXgyjmXtMcRVSnR0T0J2t0=
 github.com/rifflock/lfshook v0.0.0-20180227222202-bf539943797a/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/shirou/gopsutil v0.0.0-20180227225847-5776ff9c7c5d h1:TW8oufRzBs0qROjv16ll0N780gaBcgSu1TxKjwJMebM=
 github.com/shirou/gopsutil v0.0.0-20180227225847-5776ff9c7c5d/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
@@ -64,7 +60,6 @@ github.com/spf13/afero v1.0.2 h1:5bRmqmInNmNFkI9NG9O0Xc/Lgl9wOWWUUA/O8XZqTCo=
 github.com/spf13/afero v1.0.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -94,7 +89,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

server/server.go

@@ -1,13 +1,13 @@
 package server
 
 import (
-	"github.com/pterodactyl/wings"
 	"github.com/pterodactyl/wings/environment"
 	"github.com/remeh/sizedwaitgroup"
 	"go.uber.org/zap"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"os"
+	"path"
 	"strings"
 )
 
@@ -79,38 +79,62 @@ type Allocations struct {
 
 // Iterates over a given directory and loads all of the servers listed before returning
 // them to the calling function.
-func LoadDirectory(dir string) (*[]Server, error) {
+func LoadDirectory(dir string, cfg environment.DockerConfiguration) ([]*Server, error) {
-	wg := sizedwaitgroup.New(5)
+	// We could theoretically use a standard wait group here, however doing
+	// that introduces the potential to crash the program due to too many
+	// open files. This wouldn't happen on a small setup, but once the daemon is
+	// handling many servers you run that risk.
+	//
+	// For now just process 10 files at a time, that should be plenty fast to
+	// read and parse the YAML. We should probably make this configurable down
+	// the road to help big instances scale better.
+	wg := sizedwaitgroup.New(10)
 
 	f, err := ioutil.ReadDir(dir)
 	if err != nil {
 		return nil, err
 	}
 
-	zap.S().Debug("starting loop")
+	var servers []*Server
+
 	for _, file := range f {
-		if !strings.HasSuffix(file.Name(), ".yml") {
+		if !strings.HasSuffix(file.Name(), ".yml") || file.IsDir() {
 			continue
 		}
 
 		wg.Add()
+		// For each of the YAML files we find, parse it and create a new server
+		// configuration object that can then be returned to the caller.
 		go func(file os.FileInfo) {
-			zap.S().Debugw("processing in parallel", zap.String("name", file.Name()))
+			defer wg.Done()
-			wg.Done()
+
+			b, err := ioutil.ReadFile(path.Join(dir, file.Name()))
+			if err != nil {
+				zap.S().Errorw("failed to read server configuration file, skipping...", zap.Error(err))
+				return
+			}
+
+			s, err := FromConfiguration(b, cfg)
+			if err != nil {
+				zap.S().Errorw("failed to parse server configuration, skipping...", zap.Error(err))
+				return
+			}
+
+			servers = append(servers, s)
 		}(file)
 	}
 
+	// Wait until we've processed all of the configuration files in the directory
+	// before continuing.
 	wg.Wait()
 
-	zap.S().Debug("done processing files")
+	return servers, nil
-
-	return nil, nil
 }
 
 // Initalizes a server using a data byte array. This will be marshaled into the
 // given struct using a YAML marshaler. This will also configure the given environment
 // for a server.
-func FromConfiguration(data []byte, cfg wings.DockerConfiguration) (*Server, error) {
+func FromConfiguration(data []byte, cfg environment.DockerConfiguration) (*Server, error) {
 	s := &Server{}
 
 	if err := yaml.Unmarshal(data, s); err != nil {
@@ -122,9 +146,6 @@ func FromConfiguration(data []byte, cfg wings.DockerConfiguration) (*Server, err
 	// some modifications here obviously.
 	var env environment.Environment
 	env = &environment.Docker{
-		Controller: &environment.Controller{
-			Server: s,
-		},
 		Configuration: cfg,
 	}
 

wings.go

@@ -1,4 +1,4 @@
-package wings
+package main
 
 import (
 	"flag"
@@ -36,9 +36,15 @@ func main() {
 		zap.S().Debugw("running in debug mode")
 	}
 
-	zap.S().Infow("configuration initalized", zap.Any("config", c))
+	servers, err := server.LoadDirectory("config/servers", *c.Docker)
+	if err != nil {
+		zap.S().Fatalw("failed to load server configurations", zap.Error(err))
+		return
+	}
 
-	server.LoadDirectory("config/servers")
+	for _, s := range servers {
+		zap.S().Infow("loaded configuration for server", zap.String("server", s.Uuid))
+	}
 }
 
 // Configures the global logger for Zap so that we can call it from any location