Add base logic to start implementing support for mounted filesystems

cmd/migrate_vhd.go

@@ -0,0 +1,102 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"emperror.dev/errors"
+	"github.com/apex/log"
+	"github.com/pterodactyl/wings/config"
+	"github.com/pterodactyl/wings/internal/vhd"
+	"github.com/pterodactyl/wings/loggers/cli"
+	"github.com/pterodactyl/wings/remote"
+	"github.com/pterodactyl/wings/server"
+	"github.com/spf13/cobra"
+)
+
+type MigrateVHDCommand struct {
+	manager *server.Manager
+}
+
+func newMigrateVHDCommand() *cobra.Command {
+	return &cobra.Command{
+		Use: "migrate-vhd",
+		Short: "migrates existing data from a directory tree into virtual hard-disks",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			log.SetLevel(log.DebugLevel)
+			log.SetHandler(cli.Default)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			client := remote.NewFromConfig(config.Get())
+			manager, err := server.NewManager(cmd.Context(), client)
+			if err != nil {
+				log.WithField("error", err).Fatal("failed to create new server manager")
+			}
+			c := &MigrateVHDCommand{
+				manager: manager,
+			}
+			if err := c.Run(cmd.Context()); err != nil {
+				log.WithField("error", err).Fatal("failed to execute command")
+			}
+		},
+	}
+}
+
+// Run executes the migration command.
+func (m *MigrateVHDCommand) Run(ctx context.Context) error {
+	root := filepath.Join(config.Get().System.Data, ".disks")
+	if err := os.MkdirAll(root, 0600); err != nil {
+		return errors.Wrap(err, "failed to create root directory for virtual disks")
+	}
+
+	for _, s := range m.manager.All() {
+		s.Log().Debug("starting migration of server contents to virtual disk...")
+
+		v := s.Filesystem().NewVHD()
+		if err := v.Allocate(ctx); err != nil {
+			return errors.WithStackIf(err)
+		}
+
+		if err := v.MakeFilesystem(ctx); err != nil {
+			// If the filesystem already exists no worries, just move on with our
+			// day here.
+			if !errors.Is(err, vhd.ErrFilesystemExists) {
+				return errors.WithStack(err)
+			}
+		}
+
+		bak := strings.TrimSuffix(s.Filesystem().Path(), "/") + "_bak"
+		// Create a backup directory of the server files if one does not already exist
+		// at that location. If one does exists we'll just assume it is good to go and
+		// rely on it to provide the files we'll need.
+		if _, err := os.Lstat(bak); os.IsNotExist(err) {
+			if err := os.Rename(s.Filesystem().Path(), bak); err != nil {
+				return errors.Wrap(err, "failed to rename existing data directory for backup")
+			}
+		} else if err != nil {
+			return errors.WithStack(err)
+		}
+
+		if err := os.RemoveAll(s.Filesystem().Path()); err != nil && !os.IsNotExist(err) {
+			return errors.Wrap(err, "failed to remove base server files path")
+		}
+
+		// Attempt to mount the disk at the expected path now that we've created
+		// a backup of the server files.
+		if err := v.Mount(ctx); err != nil && !errors.Is(err, vhd.ErrFilesystemMounted) {
+			return errors.WithStackIf(err)
+		}
+
+		// Copy over the files from the backup for this server.
+		cmd := exec.CommandContext(ctx, "cp", "-a", bak + "/.", s.Filesystem().Path())
+		if err := cmd.Run(); err != nil {
+			return errors.WithStack(err)
+		}
+
+		s.Log().Info("finished migration to virtual disk...")
+	}
+	return nil
+}

cmd/root.go

@@ -47,8 +47,16 @@ var (
 var rootCommand = &cobra.Command{
 	Use:   "wings",
 	Short: "Runs the API server allowing programmatic control of game servers for Pterodactyl Panel.",
-	PreRun: func(cmd *cobra.Command, args []string) {
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
 		initConfig()
+		if ok, _ := cmd.Flags().GetBool("ignore-certificate-errors"); ok {
+			log.Warn("running with --ignore-certificate-errors: TLS certificate host chains and name will not be verified")
+			http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
+				InsecureSkipVerify: true,
+			}
+		}
+	},
+	PreRun: func(cmd *cobra.Command, args []string) {
 		initLogging()
 		if tls, _ := cmd.Flags().GetBool("auto-tls"); tls {
 			if host, _ := cmd.Flags().GetString("tls-hostname"); host == "" {
@@ -77,6 +85,7 @@ func Execute() {
 func init() {
 	rootCommand.PersistentFlags().StringVar(&configPath, "config", config.DefaultLocation, "set the location for the configuration file")
 	rootCommand.PersistentFlags().BoolVar(&debug, "debug", false, "pass in order to run wings in debug mode")
+	rootCommand.PersistentFlags().Bool("ignore-certificate-errors", false, "ignore certificate verification errors when executing API calls")
 
 	// Flags specifically used when running the API.
 	rootCommand.Flags().Bool("pprof", false, "if the pprof profiler should be enabled. The profiler will bind to localhost:6060 by default")
@@ -84,11 +93,11 @@ func init() {
 	rootCommand.Flags().Int("pprof-port", 6060, "If provided with --pprof, the port it will run on")
 	rootCommand.Flags().Bool("auto-tls", false, "pass in order to have wings generate and manage its own SSL certificates using Let's Encrypt")
 	rootCommand.Flags().String("tls-hostname", "", "required with --auto-tls, the FQDN for the generated SSL certificate")
-	rootCommand.Flags().Bool("ignore-certificate-errors", false, "ignore certificate verification errors when executing API calls")
 
 	rootCommand.AddCommand(versionCommand)
 	rootCommand.AddCommand(configureCmd)
 	rootCommand.AddCommand(newDiagnosticsCommand())
+	rootCommand.AddCommand(newMigrateVHDCommand())
 }
 
 func rootCmdRun(cmd *cobra.Command, _ []string) {
@@ -96,13 +105,6 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
 	log.Debug("running in debug mode")
 	log.WithField("config_file", configPath).Info("loading configuration from file")
 
-	if ok, _ := cmd.Flags().GetBool("ignore-certificate-errors"); ok {
-		log.Warn("running with --ignore-certificate-errors: TLS certificate host chains and name will not be verified")
-		http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
-	}
-
 	if err := config.ConfigureTimezone(); err != nil {
 		log.WithField("error", err).Fatal("failed to detect system timezone or use supplied configuration value")
 	}

config/config.go

@@ -120,6 +120,14 @@ type RemoteQueryConfiguration struct {
 
 // SystemConfiguration defines basic system configuration settings.
 type SystemConfiguration struct {
+	// UseVirtualDisks sets Wings to use virtual hard-disks when storing server
+	// files. This allows for more enforced disk space limits, at a slight performance
+	// cost.
+	//
+	// Generally this only needs to be enabled on systems with a large untrusted
+	// user presence, it is not necessary for self-hosting instances.
+	UseVirtualDisks bool `json:"use_virtual_disks" yaml:"use_virtual_disks"`
+
 	// The root directory where all of the pterodactyl data is stored at.
 	RootDirectory string `default:"/var/lib/pterodactyl" yaml:"root_directory"`
 

internal/vhd/vhd.go

@@ -0,0 +1,162 @@
+package vhd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"emperror.dev/errors"
+)
+
+var (
+	ErrFilesystemMounted = errors.Sentinel("vhd: filesystem is already mounted")
+	ErrFilesystemExists  = errors.Sentinel("vhd: filesystem already exists on disk")
+)
+
+type Disk struct {
+	size int64
+
+	diskPath string
+	mountAt  string
+}
+
+// New returns a new Disk instance. The "size" parameter should be provided in
+// megabytes of space allowed for the disk.
+func New(size int64, diskPath string, mountAt string) *Disk {
+	if diskPath == "" || mountAt == "" {
+		panic("vhd: cannot specify an empty disk or mount path")
+	}
+	return &Disk{size, diskPath, mountAt}
+}
+
+// Exists reports if the disk exists on the system yet or not. This only verifies
+// the presence of the disk image, not the validity of it.
+func (d *Disk) Exists() (bool, error) {
+	_, err := os.Lstat(d.diskPath)
+	if err == nil || os.IsNotExist(err) {
+		return err == nil, nil
+	}
+	return false, errors.WithStack(err)
+}
+
+// IsMounted checks to see if the given disk is currently mounted.
+func (d *Disk) IsMounted(ctx context.Context) (bool, error) {
+	find := d.mountAt + " ext4"
+	cmd := exec.CommandContext(ctx, "grep", "-qs", find, "/proc/mounts")
+	if err := cmd.Run(); err != nil {
+		if v, ok := err.(*exec.ExitError); ok {
+			if v.ExitCode() == 1 {
+				return false, nil
+			}
+		}
+		return false, errors.Wrap(err, "vhd: failed to execute grep for mount existence")
+	}
+	return true, nil
+}
+
+// Mount attempts to mount the disk as configured. If it does not exist or the
+// mount command fails an error will be returned to the caller. This does not
+// attempt to create the disk if it is missing from the filesystem.
+//
+// Attempting to mount a disk which does not exist will result in an error being
+// returned to the caller. If the disk is already mounted an ErrFilesystemMounted
+// error is returned to the caller.
+func (d *Disk) Mount(ctx context.Context) error {
+	if _, err := os.Lstat(d.mountAt); err != nil && !os.IsNotExist(err) {
+		return errors.Wrap(err, "vhd: failed to stat mount path")
+	} else if os.IsNotExist(err) {
+		if err := os.MkdirAll(d.mountAt, 0600); err != nil {
+			return errors.Wrap(err, "vhd: failed to create mount path")
+		}
+	}
+	if isMounted, err := d.IsMounted(ctx); err != nil {
+		return errors.WithStackIf(err)
+	} else if isMounted {
+		return ErrFilesystemMounted
+	}
+	cmd := exec.CommandContext(ctx, "mount", "-t", "auto", "-o", "loop", d.diskPath, d.mountAt)
+	if _, err := cmd.Output(); err != nil {
+		msg := "vhd: failed to mount disk"
+		if v, ok := err.(*exec.ExitError); ok {
+			msg = msg + ": " + strings.Trim(string(v.Stderr), ".\n")
+		}
+		return errors.Wrap(err, msg)
+	}
+	return nil
+}
+
+// Unmount attempts to unmount the disk from the system. If the disk is not
+// currently mounted this function is a no-op and no error is returned. Any
+// other error encountered while unmounting will return an error to the caller.
+func (d *Disk) Unmount(ctx context.Context) error {
+	cmd := exec.CommandContext(ctx, "umount", d.mountAt)
+	if err := cmd.Run(); err != nil {
+		if v, ok := err.(*exec.ExitError); !ok || v.ExitCode() != 32 {
+			return errors.Wrap(err, "vhd: failed to execute unmount command for disk")
+		}
+	}
+	return nil
+}
+
+// Allocate executes the "fallocate" command on the disk. This will first unmount
+// the disk from the system before attempting to actually allocate the space. If
+// this disk already exists on the machine it will be resized accordingly.
+//
+// DANGER! This will unmount the disk from the machine while performing this
+// action, use caution when calling it during normal processes.
+func (d *Disk) Allocate(ctx context.Context) error {
+	if exists, err := d.Exists(); exists {
+		// If the disk currently exists attempt to unmount the mount point before
+		// allocating space.
+		if err := d.Unmount(ctx); err != nil {
+			return errors.WithStackIf(err)
+		}
+	} else if err != nil {
+		return errors.Wrap(err, "vhd: failed to check for existence of root disk")
+	}
+	cmd := exec.CommandContext(ctx, "fallocate", "-l", fmt.Sprintf("%dM", d.size), d.diskPath)
+	fmt.Println(cmd.String())
+	if _, err := cmd.Output(); err != nil {
+		msg := "vhd: failed to execute fallocate command"
+		if v, ok := err.(*exec.ExitError); ok {
+			msg = msg + ": " + strings.Trim(string(v.Stderr), ".\n")
+		}
+		return errors.Wrap(err, msg)
+	}
+	return nil
+}
+
+// MakeFilesystem will attempt to execute the "mkfs" command against the disk on
+// the machine. If the disk has already been created this command will return an
+// ErrFilesystemExists error to the caller. You should manually unmount the disk
+// if it shouldn't be mounted at this point.
+func (d *Disk) MakeFilesystem(ctx context.Context) error {
+	// If no error is returned when mounting DO NOT execute this command as it will
+	// completely destroy the data stored at that location.
+	err := d.Mount(ctx)
+	if err == nil || errors.Is(err, ErrFilesystemMounted) {
+		// If it wasn't already mounted try to clean up at this point and unmount
+		// the disk. If this fails just ignore it for now.
+		if err != nil {
+			_ = d.Unmount(ctx)
+		}
+		return ErrFilesystemExists
+	}
+	if !strings.Contains(err.Error(), "can't find in /etc/fstab") {
+		return errors.WrapIf(err, "vhd: unexpected error from mount command")
+	}
+	// As long as we got an error back that was because we couldn't find thedisk
+	// in the /etc/fstab file we're good. Otherwise it means the disk probably exists
+	// or something else went wrong.
+	//
+	// Because this is a destructive command and non-tty based exection of it implies
+	// "-F" (force), we need to only run it when we can guarantee it doesn't already
+	// exist. No vague "maybe that error is expected" allowed here.
+	cmd := exec.CommandContext(ctx, "mkfs", "-t", "ext4", d.diskPath)
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "vhd: failed to make filesystem for disk")
+	}
+	return nil
+}

remote/http.go

@@ -17,6 +17,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/goccy/go-json"
 
+	"github.com/pterodactyl/wings/config"
 	"github.com/pterodactyl/wings/system"
 )
 
@@ -59,6 +60,18 @@ func New(base string, opts ...ClientOption) Client {
 	return &c
 }
 
+// NewFromConfig returns a new Client using the configuration passed through
+// by the caller.
+func NewFromConfig(cfg *config.Configuration, opts ...ClientOption) Client {
+	passOpts := []ClientOption{
+		WithCredentials(cfg.AuthenticationTokenId, cfg.AuthenticationToken),
+		WithHttpClient(&http.Client{
+			Timeout: time.Second * time.Duration(cfg.RemoteQuery.Timeout),
+		}),
+	}
+	return New(cfg.PanelLocation, append(passOpts, opts...)...)
+}
+
 // WithCredentials sets the credentials to use when making request to the remote
 // API endpoint.
 func WithCredentials(id, token string) ClientOption {

server/filesystem/vhd.go

@@ -0,0 +1,17 @@
+package filesystem
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/pterodactyl/wings/config"
+	"github.com/pterodactyl/wings/internal/vhd"
+)
+
+func (fs *Filesystem) NewVHD() *vhd.Disk {
+	parts := strings.Split(fs.root, "/")
+	disk := filepath.Join(config.Get().System.Data, ".disks/", parts[len(parts)-1]+".img")
+
+	return vhd.New(250, disk, fs.root)
+	// return vhd.New(fs.diskLimit/1024/1024, disk, fs.root)
+}