diff --git a/go.mod b/go.mod index 9da1925..f7c2653 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/Microsoft/go-winio v0.4.7 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a - github.com/aws/aws-sdk-go v1.30.9 // indirect github.com/beevik/etree v1.1.0 github.com/buger/jsonparser v0.0.0-20191204142016-1a29609e0929 github.com/cobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 @@ -37,6 +36,7 @@ require ( github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/imdario/mergo v0.3.8 + github.com/klauspost/pgzip v1.2.3 github.com/magiconair/properties v1.8.1 github.com/mattn/go-shellwords v1.0.10 // indirect github.com/mholt/archiver/v3 v3.3.0 @@ -52,10 +52,10 @@ require ( github.com/pkg/sftp v1.10.1 // indirect github.com/pterodactyl/sftp-server v1.1.1 github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce + github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.7 - github.com/stretchr/objx v0.2.0 // indirect - github.com/yuin/goldmark v1.1.30 // indirect + github.com/stretchr/testify v1.5.1 // indirect go.uber.org/atomic v1.5.1 // indirect go.uber.org/multierr v1.4.0 // indirect go.uber.org/zap v1.13.0 diff --git a/go.sum b/go.sum index fb13015..685fd22 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,6 @@ github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.30.9 h1:DntpBUKkchINPDbhEzDRin1eEn1TG9TZFlzWPf0i8to= -github.com/aws/aws-sdk-go v1.30.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -82,7 +80,6 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -127,8 +124,6 @@ github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -146,6 +141,8 @@ github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.3 h1:Ce2to9wvs/cuJ2b86/CKQoTYr9VHfpanYosZ0UBJqdw= +github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= @@ -232,6 +229,8 @@ github.com/remeh/sizedwaitgroup v0.0.0-20180822144253-5e7302b12cce/go.mod h1:3j2 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0= +github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -251,7 +250,6 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -273,7 +271,6 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -301,8 +298,6 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8 h1:fpnn/HnJONpIu6hkXi1u/7rR0NzilgWr4T0JmWkEitk= -golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -322,7 +317,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -346,8 +340,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= @@ -368,8 +360,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200403190813-44a64ad78b9b h1:AFZdJUT7jJYXQEC29hYH/WZkoV7+KhwxQGmdZ19yYoY= -golang.org/x/tools v0.0.0-20200403190813-44a64ad78b9b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290 h1:NXNmtp0ToD36cui5IqWy95LC4Y6vT/4y3RnPxlQPinU= golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools/gopls v0.1.3/go.mod h1:vrCQzOKxvuiZLjCKSmbbov04oeBQQOb4VQqwYK2PWIY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= diff --git a/server/backup.go b/server/backup.go index 8c8e809..992b265 100644 --- a/server/backup.go +++ b/server/backup.go @@ -35,7 +35,12 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc // let the actual backup system handle notifying the panel of the status, but that // won't emit a websocket event. func (s *Server) BackupLocal(b *backup.LocalBackup) error { - if err := b.Backup(s.Filesystem.Path()); err != nil { + inc, err := s.Filesystem.GetIncludedFiles(s.Filesystem.Path(), b.IgnoredFiles) + if err != nil { + return errors.WithStack(err) + } + + if err := b.Backup(inc, s.Filesystem.Path()); err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Uuid), zap.Error(err)) } diff --git a/server/backup/archiver.go b/server/backup/archiver.go new file mode 100644 index 0000000..2244f2c --- /dev/null +++ b/server/backup/archiver.go @@ -0,0 +1,111 @@ +package backup + +import ( + "archive/tar" + "context" + gzip "github.com/klauspost/pgzip" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + "io" + "os" + "strings" + "sync" +) + +type Archive struct { + sync.Mutex + + TrimPrefix string + Files *IncludedFiles +} + +// Creates an archive at dest with all of the files definied in the included files struct. +func (a *Archive) Create(dest string, ctx context.Context) error { + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer f.Close() + + gzw := gzip.NewWriter(f) + defer gzw.Close() + + tw := tar.NewWriter(gzw) + defer tw.Close() + + wg := sizedwaitgroup.New(10) + g, ctx := errgroup.WithContext(ctx) + // Iterate over all of the files to be included and put them into the archive. This is + // done as a concurrent goroutine to speed things along. If an error is encountered at + // any step, the entire process is aborted. + for p, s := range a.Files.All() { + if (*s).IsDir() { + continue + } + + pa := p + st := s + + g.Go(func() error { + wg.Add() + defer wg.Done() + + select { + case <-ctx.Done(): + return ctx.Err() + default: + return a.addToArchive(pa, st, tw) + } + }) + } + + // Block until the entire routine is completed. + if err := g.Wait(); err != nil { + f.Close() + + // Attempt to remove the archive if there is an error, report that error to + // the logger if it fails. + if rerr := os.Remove(dest); rerr != nil && !os.IsNotExist(rerr) { + zap.S().Warnw("failed to delete corrupted backup archive", zap.String("location", dest)) + } + + return err + } + + return nil +} + +// Adds a single file to the existing tar archive writer. +func (a *Archive) addToArchive(p string, s *os.FileInfo, w *tar.Writer) error { + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + + st := *s + header := &tar.Header{ + // Trim the long server path from the name of the file so that the resulting + // archive is exactly how the user would see it in the panel file manager. + Name: strings.TrimPrefix(p, a.TrimPrefix), + Size: st.Size(), + Mode: int64(st.Mode()), + ModTime: st.ModTime(), + } + + // These actions must occur sequentially, even if this function is called multiple + // in parallel. You'll get some nasty panic's otherwise. + a.Lock() + defer a.Unlock() + + if err = w.WriteHeader(header); err != nil { + return err + } + + if _, err := io.Copy(w, f); err != nil { + return err + } + + return nil +} diff --git a/server/backup/backup.go b/server/backup/backup.go index 268b1a1..70eda89 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -10,7 +10,7 @@ type Backup interface { // Generates a backup in whatever the configured source for the specific // implementation is. - Backup(dir string) error + Backup(*IncludedFiles, string) error // Returns a SHA256 checksum for the generated backup. Checksum() ([]byte, error) diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index b6fab45..b81efb9 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -1,16 +1,15 @@ package backup import ( + "context" "crypto/sha256" "encoding/hex" - "github.com/mholt/archiver/v3" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" "go.uber.org/zap" "io" "os" "path" - "strings" "sync" ) @@ -79,25 +78,15 @@ func (b *LocalBackup) Remove() error { // Generates a backup of the selected files and pushes it to the defined location // for this instance. -func (b *LocalBackup) Backup(dir string) error { - if err := archiver.Archive([]string{dir}, b.Path()); err != nil { - if strings.HasPrefix(err.Error(), "file already exists") { - if rerr := os.Remove(b.Path()); rerr != nil { - return errors.WithStack(rerr) - } - - // Re-attempt this backup by calling it with the same information. - return b.Backup(dir) - } - - // If there was some error with the archive, just go ahead and ensure the backup - // is completely destroyed at this point. Ignore any errors from this function. - os.Remove(b.Path()) - - return errors.WithStack(err) +func (b *LocalBackup) Backup(included *IncludedFiles, prefix string) error { + a := &Archive{ + TrimPrefix: prefix, + Files: included, } - return nil + err := a.Create(b.Path(), context.Background()) + + return err } // Return the size of the generated backup. @@ -162,4 +151,4 @@ func (b *LocalBackup) ensureLocalBackupLocation() error { } return nil -} \ No newline at end of file +} diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 1dc45f5..4e78131 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -16,7 +16,7 @@ func (s *S3Backup) Identifier() string { return s.Uuid } -func (s *S3Backup) Backup(dir string) error { +func (s *S3Backup) Backup(included *IncludedFiles, prefix string) error { panic("implement me") } diff --git a/server/backup/included.go b/server/backup/included.go new file mode 100644 index 0000000..5f40cbe --- /dev/null +++ b/server/backup/included.go @@ -0,0 +1,31 @@ +package backup + +import ( + "os" + "sync" +) + +type IncludedFiles struct { + sync.RWMutex + files map[string]*os.FileInfo +} + +// Pushes an additional file or folder onto the struct. +func (i *IncludedFiles) Push(info *os.FileInfo, p string) { + i.Lock() + defer i.Unlock() + + if i.files == nil { + i.files = make(map[string]*os.FileInfo) + } + + i.files[p] = info +} + +// Returns all of the files that were marked as being included. +func (i *IncludedFiles) All() map[string]*os.FileInfo { + i.RLock() + defer i.RUnlock() + + return i.files +} diff --git a/server/filesystem.go b/server/filesystem.go index 28e06ba..3f2d1f7 100644 --- a/server/filesystem.go +++ b/server/filesystem.go @@ -9,6 +9,8 @@ import ( "github.com/gabriel-vasile/mimetype" "github.com/pkg/errors" "github.com/pterodactyl/wings/config" + "github.com/pterodactyl/wings/server/backup" + ignore "github.com/sabhiram/go-gitignore" "go.uber.org/zap" "io" "io/ioutil" @@ -561,3 +563,43 @@ func (fs *Filesystem) EnsureDataDirectory() error { return nil } + +// Given a directory, iterate through all of the files and folders within it and determine +// if they should be included in the output based on an array of ignored matches. This uses +// standard .gitignore formatting to make that determination. +// +// If no ignored files are passed through you'll get the entire directory listing. +func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.IncludedFiles, error) { + cleaned, err := fs.SafePath(dir) + if err != nil { + return nil, err + } + + w := fs.NewWalker() + ctx := context.Background() + + i, err := ignore.CompileIgnoreLines(ignored...) + if err != nil { + return nil, err + } + + // Walk through all of the files and directories on a server. This callback only returns + // files found, and will keep walking deeper and deeper into directories. + inc := new(backup.IncludedFiles) + if err := w.Walk(cleaned, ctx, func(f os.FileInfo, p string) bool { + // Avoid unnecessary parsing if there are no ignored files, nothing will match anyways + // so no reason to call the function. + if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(p, fs.Path() + "/")) { + inc.Push(&f, p) + } + + // We can't just abort if the path is technically ignored. It is possible there is a nested + // file or folder that should not be excluded, so in this case we need to just keep going + // until we get to a final state. + return true + }); err != nil { + return nil, err + } + + return inc, nil +} diff --git a/server/filesystem_walker.go b/server/filesystem_walker.go index bc73886..0db761a 100644 --- a/server/filesystem_walker.go +++ b/server/filesystem_walker.go @@ -38,6 +38,7 @@ func (fw *FileWalker) Walk(dir string, ctx context.Context, callback func (os.Fi for _, f := range files { if f.IsDir() { + fi := f p := filepath.Join(cleaned, f.Name()) // Recursively call this function to continue digging through the directory tree within // a seperate goroutine. If the context is canceled abort this process. @@ -49,7 +50,7 @@ func (fw *FileWalker) Walk(dir string, ctx context.Context, callback func (os.Fi // If the callback returns true, go ahead and keep walking deeper. This allows // us to programatically continue deeper into directories, or stop digging // if that pathway knows it needs nothing else. - if callback(f, p) { + if callback(fi, p) { return fw.Walk(p, ctx, callback) }