From 0bd28a4480db5e765bbd3c41d6fdfbfa53f0abd4 Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sat, 9 May 2020 19:24:30 -0700 Subject: [PATCH] Cleanup S3 support; send actual backup details in response --- server/backup.go | 4 +- server/backup/backup.go | 2 +- server/backup/backup_local.go | 8 ++-- server/backup/backup_s3.go | 69 +++++++++++++++++------------------ 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/server/backup.go b/server/backup.go index f1b9a0d..67b84f4 100644 --- a/server/backup.go +++ b/server/backup.go @@ -86,7 +86,8 @@ func (s *Server) Backup(b backup.BackupInterface) error { return errors.WithStack(err) } - if err := b.Generate(inc, s.Filesystem.Path()); err != nil { + ad, err := b.Generate(inc, s.Filesystem.Path()) + if err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { zap.S().Warnw("failed to notify panel of failed backup state", zap.String("backup", b.Identifier()), zap.Error(err)) } @@ -96,7 +97,6 @@ func (s *Server) Backup(b backup.BackupInterface) error { // Try to notify the panel about the status of this backup. If for some reason this request // fails, delete the archive from the daemon and return that error up the chain to the caller. - ad := b.Details() if notifyError := s.notifyPanelOfBackup(b.Identifier(), ad, true); notifyError != nil { b.Remove() diff --git a/server/backup/backup.go b/server/backup/backup.go index e882ada..876ced0 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -49,7 +49,7 @@ type BackupInterface interface { // Generates a backup in whatever the configured source for the specific // implementation is. - Generate(*IncludedFiles, string) error + Generate(*IncludedFiles, string) (*ArchiveDetails, error) // Returns the ignored files for this backup instance. Ignored() []string diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index 6c1f926..a6afaa3 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -41,13 +41,15 @@ func (b *LocalBackup) Remove() error { // Generates a backup of the selected files and pushes it to the defined location // for this instance. -func (b *LocalBackup) Generate(included *IncludedFiles, prefix string) error { +func (b *LocalBackup) Generate(included *IncludedFiles, prefix string) (*ArchiveDetails, error) { a := &Archive{ TrimPrefix: prefix, Files: included, } - err := a.Create(b.Path(), context.Background()) + if err := a.Create(b.Path(), context.Background()); err != nil { + return nil, err + } - return err + return b.Details(), nil } diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index d856978..0d1616d 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -3,6 +3,7 @@ package backup import ( "context" "fmt" + "go.uber.org/zap" "io" "net/http" "os" @@ -21,7 +22,9 @@ type S3Backup struct { var _ BackupInterface = (*S3Backup)(nil) -func (s *S3Backup) Generate(included *IncludedFiles, prefix string) error { +// Generates a new backup on the disk, moves it into the S3 bucket via the provided +// presigned URL, and then deletes the backup from the disk. +func (s *S3Backup) Generate(included *IncludedFiles, prefix string) (*ArchiveDetails, error) { defer s.Remove() a := &Archive{ @@ -30,45 +33,26 @@ func (s *S3Backup) Generate(included *IncludedFiles, prefix string) error { } if err := a.Create(s.Path(), context.Background()); err != nil { - return err + return nil, err } - fmt.Println(s.PresignedUrl) - - r, err := http.NewRequest(http.MethodPut, s.PresignedUrl, nil) + rc, err := os.Open(s.Path()) if err != nil { - return err - } - - if sz, err := s.Size(); err != nil { - return err - } else { - r.ContentLength = sz - r.Header.Add("Content-Length", strconv.Itoa(int(sz))) - r.Header.Add("Content-Type", "application/x-gzip") - } - - var rc io.ReadCloser - if f, err := os.Open(s.Path()); err != nil { - return err - } else { - rc = f + return nil, err } defer rc.Close() - r.Body = rc - resp, err := http.DefaultClient.Do(r) - if err != nil { - return err - } - defer resp.Body.Close() + if resp, err := s.generateRemoteRequest(rc); err != nil { + return nil, err + } else { + resp.Body.Close() - if resp.StatusCode != http.StatusOK { - io.Copy(os.Stdout, resp.Body) - return fmt.Errorf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to put S3 object, %d:%s", resp.StatusCode, resp.Status) + } } - return nil + return s.Details(), err } // Removes a backup from the system. @@ -76,9 +60,24 @@ func (s *S3Backup) Remove() error { return os.Remove(s.Path()) } -func (s *S3Backup) Details() *ArchiveDetails { - return &ArchiveDetails{ - Checksum: "checksum", - Size: 1024, +// Generates the remote S3 request and begins the upload. +func (s *S3Backup) generateRemoteRequest(rc io.ReadCloser) (*http.Response, error) { + r, err := http.NewRequest(http.MethodPut, s.PresignedUrl, nil) + if err != nil { + return nil, err } + + if sz, err := s.Size(); err != nil { + return nil, err + } else { + r.ContentLength = sz + r.Header.Add("Content-Length", strconv.Itoa(int(sz))) + r.Header.Add("Content-Type", "application/x-gzip") + } + + r.Body = rc + + zap.S().Debugw("uploading backup to remote S3 endpoint", zap.String("endpoint", s.PresignedUrl), zap.Any("headers", r.Header)) + + return http.DefaultClient.Do(r) }