From 507d0100cf8ec54fdee6327483a42bcd65ba28c2 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 26 Apr 2020 17:20:26 -0700 Subject: [PATCH] Hilariously rough code to get a backup pushed into S3 --- go.mod | 12 ++-- go.sum | 15 +++++ server/backup/backup_local.go | 24 ++------ server/backup/backup_s3.go | 108 ++++++++++++++++++++++++++++++---- 4 files changed, 122 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index f7c2653..efd25f7 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/Microsoft/go-winio v0.4.7 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a + github.com/aws/aws-sdk-go v1.30.14 // indirect github.com/beevik/etree v1.1.0 github.com/buger/jsonparser v0.0.0-20191204142016-1a29609e0929 github.com/cobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 @@ -55,16 +56,17 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v0.0.7 - github.com/stretchr/testify v1.5.1 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/yuin/goldmark v1.1.30 // indirect go.uber.org/atomic v1.5.1 // indirect go.uber.org/multierr v1.4.0 // indirect go.uber.org/zap v1.13.0 - golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 // indirect + golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 // indirect golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect - golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect + golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 // indirect golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a - golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect - golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290 // indirect + golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect + golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.51.0 gopkg.in/yaml.v2 v2.2.8 diff --git a/go.sum b/go.sum index 685fd22..34edf73 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.30.14 h1:vZfX2b/fknc9wKcytbLWykM7in5k6dbQ8iHTJDUP1Ng= +github.com/aws/aws-sdk-go v1.30.14/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -80,6 +82,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= @@ -124,6 +127,8 @@ github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -250,6 +255,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -271,6 +277,7 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -300,6 +307,8 @@ golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -317,9 +326,12 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0 h1:Jcxah/M+oLZ/R4/z5RzfPzGbPXnVDPkEDtf2JnuxN+U= +golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -342,6 +354,8 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY= golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -362,6 +376,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290 h1:NXNmtp0ToD36cui5IqWy95LC4Y6vT/4y3RnPxlQPinU= golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools/gopls v0.1.3/go.mod h1:vrCQzOKxvuiZLjCKSmbbov04oeBQQOb4VQqwYK2PWIY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index f05a799..133993e 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -122,12 +122,11 @@ func (b *LocalBackup) Details() *ArchiveDetails { go func() { defer wg.Done() - st, err := os.Stat(b.Path()) - if err != nil { + if s, err := b.Size(); err != nil { return + } else { + sz = s } - - sz = st.Size() }() wg.Wait() @@ -140,19 +139,4 @@ func (b *LocalBackup) Details() *ArchiveDetails { func (b *LocalBackup) Ignored() []string { return b.IgnoredFiles -} - -// Ensures that the local backup destination for files exists. -func (b *LocalBackup) ensureLocalBackupLocation() error { - d := config.Get().System.BackupDirectory - - if _, err := os.Stat(d); err != nil { - if !os.IsNotExist(err) { - return errors.WithStack(err) - } - - return os.MkdirAll(d, 0700) - } - - return nil -} +} \ No newline at end of file diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index afb7af5..f9cdbb7 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -1,5 +1,18 @@ package backup +import ( + "context" + "crypto/sha256" + "fmt" + "github.com/pkg/errors" + "github.com/pterodactyl/wings/config" + "io" + "net/http" + "os" + "path" + "strconv" +) + type S3Backup struct { // The UUID of this backup object. This must line up with a backup from // the panel instance. @@ -23,29 +36,100 @@ func (s *S3Backup) Identifier() string { } func (s *S3Backup) Backup(included *IncludedFiles, prefix string) error { - panic("implement me") -} - -func (s *S3Backup) Checksum() ([]byte, error) { - return []byte(""), nil + defer s.Remove() + + a := &Archive{ + TrimPrefix: prefix, + Files: included, + } + + if err := a.Create(s.Path(), context.Background()); err != nil { + return err + } + + fmt.Println(s.PresignedUrl) + + r, err := http.NewRequest(http.MethodPut, s.PresignedUrl, nil) + if err != nil { + return err + } + + if sz, err := s.Size(); err != nil { + return err + } else { + r.ContentLength = sz + r.Header.Add("Content-Length", strconv.Itoa(int(sz))) + r.Header.Add("Content-Type", "application/x-gzip") + } + + var rc io.ReadCloser + if f, err := os.Open(s.Path()); err != nil { + return err + } else { + rc = f + } + defer rc.Close() + + r.Body = rc + resp, err := http.DefaultClient.Do(r) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + io.Copy(os.Stdout, resp.Body) + return fmt.Errorf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) + } + + return nil } +// Return the size of the generated backup. func (s *S3Backup) Size() (int64, error) { - return 0, nil + st, err := os.Stat(s.Path()) + if err != nil { + return 0, errors.WithStack(err) + } + + return st.Size(), nil } +// Returns the path for this specific backup. S3 backups are only stored on the disk +// long enough for us to get the details we need before uploading them to S3. func (s *S3Backup) Path() string { - return "" + return path.Join(config.Get().System.BackupDirectory, s.Uuid+".tmp") +} + +// Returns the SHA256 checksum of a backup. +func (s *S3Backup) Checksum() ([]byte, error) { + h := sha256.New() + + f, err := os.Open(s.Path()) + if err != nil { + return []byte{}, errors.WithStack(err) + } + defer f.Close() + + if _, err := io.Copy(h, f); err != nil { + return []byte{}, errors.WithStack(err) + } + + return h.Sum(nil), nil +} + +// Removes a backup from the system. +func (s *S3Backup) Remove() error { + return os.Remove(s.Path()) } func (s *S3Backup) Details() *ArchiveDetails { - return &ArchiveDetails{} + return &ArchiveDetails{ + Checksum: "checksum", + Size: 1024, + } } func (s *S3Backup) Ignored() []string { return s.IgnoredFiles } - -func (s *S3Backup) Remove() error { - return nil -}