diff --git a/go.mod b/go.mod index d5293a4..a809d00 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/pkg/sftp v1.11.0 github.com/prometheus/common v0.11.1 // indirect github.com/remeh/sizedwaitgroup v1.0.0 - github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 + github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect github.com/ulikunitz/xz v0.5.7 // indirect diff --git a/go.sum b/go.sum index 03b36ad..86d3a43 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0= github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ= +github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f h1:8P2MkG70G76gnZBOPGwmMIgwBb/rESQuwsJ7K8ds4NE= +github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/server/backup.go b/server/backup.go index 147497f..10dbe1e 100644 --- a/server/backup.go +++ b/server/backup.go @@ -1,11 +1,11 @@ package server import ( - "bufio" "emperror.dev/errors" "github.com/apex/log" "github.com/pterodactyl/wings/api" "github.com/pterodactyl/wings/server/backup" + "io/ioutil" "os" "path" ) @@ -30,58 +30,38 @@ func (s *Server) notifyPanelOfBackup(uuid string, ad *backup.ArchiveDetails, suc } // Get all of the ignored files for a server based on its .pteroignore file in the root. -func (s *Server) getServerwideIgnoredFiles() ([]string, error) { - var ignored []string - +func (s *Server) getServerwideIgnoredFiles() (string, error) { f, err := os.Open(path.Join(s.Filesystem().Path(), ".pteroignore")) if err != nil { if !os.IsNotExist(err) { - return nil, err - } - } else { - scanner := bufio.NewScanner(f) - for scanner.Scan() { - // Only include non-empty lines, for the sake of clarity... - if t := scanner.Text(); t != "" { - ignored = append(ignored, t) - } + return "", err } - if err := scanner.Err(); err != nil { - return nil, err - } + return "", nil } - return ignored, nil -} - -// Get the backup files to include when generating it. -func (s *Server) GetIncludedBackupFiles(ignored []string) (*backup.IncludedFiles, error) { - // If no ignored files are present in the request, check for a .pteroignore file in the root - // of the server files directory, and use that to generate the backup. - if len(ignored) == 0 { - if i, err := s.getServerwideIgnoredFiles(); err != nil { - s.Log().WithField("error", err).Warn("failed to retrieve ignored files listing for server") - } else { - ignored = i - } + b, err := ioutil.ReadAll(f) + if err != nil { + return "", err } - // Get the included files based on the root path and the ignored files provided. - return s.Filesystem().GetIncludedFiles(s.Filesystem().Path(), ignored) + return string(b), nil } // Performs a server backup and then emits the event over the server websocket. We // let the actual backup system handle notifying the panel of the status, but that // won't emit a websocket event. func (s *Server) Backup(b backup.BackupInterface) error { - // Get the included files based on the root path and the ignored files provided. - inc, err := s.GetIncludedBackupFiles(b.Ignored()) - if err != nil { - return err + ignored := b.Ignored() + if b.Ignored() == "" { + if i, err := s.getServerwideIgnoredFiles(); err != nil { + log.WithField("server", s.Id()).WithField("error", err).Warn("failed to get server-wide ignored files") + } else { + ignored = i + } } - ad, err := b.Generate(inc, s.Filesystem().Path()) + ad, err := b.Generate(s.Filesystem().Path(), ignored) if err != nil { if notifyError := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); notifyError != nil { s.Log().WithFields(log.Fields{ @@ -109,7 +89,8 @@ func (s *Server) Backup(b backup.BackupInterface) error { // Try to notify the panel about the status of this backup. If for some reason this request // fails, delete the archive from the daemon and return that error up the chain to the caller. if notifyError := s.notifyPanelOfBackup(b.Identifier(), ad, true); notifyError != nil { - b.Remove() + _ = b.Remove() + s.Log().WithField("error", notifyError).Info("failed to notify panel of successful backup state") return err } else { @@ -118,7 +99,7 @@ func (s *Server) Backup(b backup.BackupInterface) error { // Emit an event over the socket so we can update the backup in realtime on // the frontend for the server. - s.Events().PublishJson(BackupCompletedEvent+":"+b.Identifier(), map[string]interface{}{ + _ = s.Events().PublishJson(BackupCompletedEvent+":"+b.Identifier(), map[string]interface{}{ "uuid": b.Identifier(), "is_successful": true, "checksum": ad.Checksum, diff --git a/server/backup/archiver.go b/server/backup/archiver.go deleted file mode 100644 index 408f2b8..0000000 --- a/server/backup/archiver.go +++ /dev/null @@ -1,150 +0,0 @@ -package backup - -import ( - "archive/tar" - "context" - "emperror.dev/errors" - "github.com/apex/log" - "github.com/juju/ratelimit" - gzip "github.com/klauspost/pgzip" - "github.com/pterodactyl/wings/config" - "github.com/remeh/sizedwaitgroup" - "golang.org/x/sync/errgroup" - "io" - "os" - "runtime" - "strings" - "sync" -) - -type Archive struct { - sync.Mutex - - TrimPrefix string - Files *IncludedFiles -} - -// Creates an archive at dst with all of the files defined in the included files struct. -func (a *Archive) Create(dst string, ctx context.Context) error { - f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) - if err != nil { - return err - } - defer f.Close() - - // Select a writer based off of the WriteLimit configuration option. - var writer io.Writer - if writeLimit := config.Get().System.Backups.WriteLimit; writeLimit < 1 { - // If there is no write limit, use the file as the writer. - writer = f - } else { - // Token bucket with a capacity of "writeLimit" MiB, adding "writeLimit" MiB/s - bucket := ratelimit.NewBucketWithRate(float64(writeLimit)*1024*1024, int64(writeLimit)*1024*1024) - - // Wrap the file writer with the token bucket limiter. - writer = ratelimit.Writer(f, bucket) - } - - maxCpu := runtime.NumCPU() / 2 - if maxCpu > 4 { - maxCpu = 4 - } - - gzw, err := gzip.NewWriterLevel(writer, gzip.BestSpeed) - if err != nil { - return errors.WithMessage(err, "failed to create gzip writer") - } - if err := gzw.SetConcurrency(1<<20, maxCpu); err != nil { - return errors.WithMessage(err, "failed to set gzip concurrency") - } - - defer gzw.Flush() - defer gzw.Close() - - tw := tar.NewWriter(gzw) - defer tw.Flush() - defer tw.Close() - - wg := sizedwaitgroup.New(10) - g, ctx := errgroup.WithContext(ctx) - // Iterate over all of the files to be included and put them into the archive. This is - // done as a concurrent goroutine to speed things along. If an error is encountered at - // any step, the entire process is aborted. - for _, p := range a.Files.All() { - p := p - g.Go(func() error { - wg.Add() - defer wg.Done() - - select { - case <-ctx.Done(): - return ctx.Err() - default: - return a.addToArchive(p, tw) - } - }) - } - - // Block until the entire routine is completed. - if err := g.Wait(); err != nil { - f.Close() - - // Attempt to remove the archive if there is an error, report that error to - // the logger if it fails. - if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) { - log.WithField("location", dst).Warn("failed to delete corrupted backup archive") - } - - return err - } - - return nil -} - -// Adds a single file to the existing tar archive writer. -func (a *Archive) addToArchive(p string, w *tar.Writer) error { - f, err := os.Open(p) - if err != nil { - // If you try to backup something that no longer exists (got deleted somewhere during the process - // but not by this process), just skip over it and don't kill the entire backup. - if os.IsNotExist(err) { - return nil - } - - return err - } - defer f.Close() - - s, err := f.Stat() - if err != nil { - // Same as above, don't kill the process just because the file no longer exists. - if os.IsNotExist(err) { - return nil - } - - return err - } - - name := strings.TrimPrefix(p, a.TrimPrefix) - header, err := tar.FileInfoHeader(s, name) - if err != nil { - return errors.WithMessage(err, "failed to get tar#FileInfoHeader for "+name) - } - header.Name = name - - // These actions must occur sequentially, even if this function is called multiple - // in parallel. You'll get some nasty panic's otherwise. - a.Lock() - defer a.Unlock() - - if err := w.WriteHeader(header); err != nil { - return err - } - - buf := make([]byte, 4*1024) - if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil { - return errors.WithMessage(err, "failed to copy "+header.Name+" to archive") - } - - return nil -} diff --git a/server/backup/backup.go b/server/backup/backup.go index 3bd336b..9743f45 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -40,7 +40,7 @@ type Backup struct { // An array of files to ignore when generating this backup. This should be // compatible with a standard .gitignore structure. - IgnoredFiles []string `json:"ignored_files"` + Ignore string `json:"ignore"` } // noinspection GoNameStartsWithPackageName @@ -50,12 +50,12 @@ type BackupInterface interface { // Generates a backup in whatever the configured source for the specific // implementation is. - Generate(*IncludedFiles, string) (*ArchiveDetails, error) + Generate(string, string) (*ArchiveDetails, error) // Returns the ignored files for this backup instance. - Ignored() []string + Ignored() string - // Returns a SHA256 checksum for the generated backup. + // Returns a SHA1 checksum for the generated backup. Checksum() ([]byte, error) // Returns the size of the generated backup. @@ -153,6 +153,6 @@ func (b *Backup) Details() *ArchiveDetails { } } -func (b *Backup) Ignored() []string { - return b.IgnoredFiles +func (b *Backup) Ignored() string { + return b.Ignore } diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index 43d5fea..2e14637 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -1,8 +1,8 @@ package backup import ( - "context" "errors" + "github.com/pterodactyl/wings/server/filesystem" "os" ) @@ -17,8 +17,8 @@ var _ BackupInterface = (*LocalBackup)(nil) func LocateLocal(uuid string) (*LocalBackup, os.FileInfo, error) { b := &LocalBackup{ Backup{ - Uuid: uuid, - IgnoredFiles: nil, + Uuid: uuid, + Ignore: "", }, } @@ -41,13 +41,13 @@ func (b *LocalBackup) Remove() error { // Generates a backup of the selected files and pushes it to the defined location // for this instance. -func (b *LocalBackup) Generate(included *IncludedFiles, prefix string) (*ArchiveDetails, error) { - a := &Archive{ - TrimPrefix: prefix, - Files: included, +func (b *LocalBackup) Generate(basePath, ignore string) (*ArchiveDetails, error) { + a := &filesystem.Archive{ + BasePath: basePath, + Ignore: ignore, } - if err := a.Create(b.Path(), context.Background()); err != nil { + if err := a.Create(b.Path()); err != nil { return nil, err } diff --git a/server/backup/backup_request.go b/server/backup/backup_request.go index 8a51186..a26641c 100644 --- a/server/backup/backup_request.go +++ b/server/backup/backup_request.go @@ -6,9 +6,9 @@ import ( ) type Request struct { - Adapter string `json:"adapter"` - Uuid string `json:"uuid"` - IgnoredFiles []string `json:"ignored_files"` + Adapter string `json:"adapter"` + Uuid string `json:"uuid"` + Ignore string `json:"ignore"` } // Generates a new local backup struct. @@ -19,8 +19,8 @@ func (r *Request) NewLocalBackup() (*LocalBackup, error) { return &LocalBackup{ Backup{ - Uuid: r.Uuid, - IgnoredFiles: r.IgnoredFiles, + Uuid: r.Uuid, + Ignore: r.Ignore, }, }, nil } @@ -33,8 +33,8 @@ func (r *Request) NewS3Backup() (*S3Backup, error) { return &S3Backup{ Backup: Backup{ - Uuid: r.Uuid, - IgnoredFiles: r.IgnoredFiles, + Uuid: r.Uuid, + Ignore: r.Ignore, }, }, nil } diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 74b9add..abcd417 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -1,10 +1,10 @@ package backup import ( - "context" "fmt" "github.com/apex/log" "github.com/pterodactyl/wings/api" + "github.com/pterodactyl/wings/server/filesystem" "io" "net/http" "os" @@ -19,15 +19,15 @@ var _ BackupInterface = (*S3Backup)(nil) // Generates a new backup on the disk, moves it into the S3 bucket via the provided // presigned URL, and then deletes the backup from the disk. -func (s *S3Backup) Generate(included *IncludedFiles, prefix string) (*ArchiveDetails, error) { +func (s *S3Backup) Generate(basePath, ignore string) (*ArchiveDetails, error) { defer s.Remove() - a := &Archive{ - TrimPrefix: prefix, - Files: included, + a := &filesystem.Archive{ + BasePath: basePath, + Ignore: ignore, } - if err := a.Create(s.Path(), context.Background()); err != nil { + if err := a.Create(s.Path()); err != nil { return nil, err } diff --git a/server/backup/included.go b/server/backup/included.go deleted file mode 100644 index 916fc5a..0000000 --- a/server/backup/included.go +++ /dev/null @@ -1,25 +0,0 @@ -package backup - -import ( - "sync" -) - -type IncludedFiles struct { - sync.RWMutex - files []string -} - -// Pushes an additional file or folder onto the struct. -func (i *IncludedFiles) Push(p string) { - i.Lock() - i.files = append(i.files, p) // ~~ - i.Unlock() -} - -// Returns all of the files that were marked as being included. -func (i *IncludedFiles) All() []string { - i.RLock() - defer i.RUnlock() - - return i.files -} diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go new file mode 100644 index 0000000..af38b6d --- /dev/null +++ b/server/filesystem/archive.go @@ -0,0 +1,207 @@ +package filesystem + +import ( + "archive/tar" + "emperror.dev/errors" + "github.com/juju/ratelimit" + "github.com/karrick/godirwalk" + "github.com/klauspost/pgzip" + "github.com/pterodactyl/wings/config" + "github.com/sabhiram/go-gitignore" + "io" + "os" + "path/filepath" + "strings" + "sync" +) + +const memory = 4 * 1024 + +var pool = sync.Pool{ + New: func() interface{} { + b := make([]byte, memory) + return b + }, +} + +type Archive struct { + // BasePath is the absolute path to create the archive from where Files and Ignore are + // relative to. + BasePath string + + // Ignore is a gitignore string (most likely read from a file) of files to ignore + // from the archive. + Ignore string + + // Files specifies the files to archive, this takes priority over the Ignore option, if + // unspecified, all files in the BasePath will be archived unless Ignore is set. + Files []string +} + +// Creates an archive at dst with all of the files defined in the included files struct. +func (a *Archive) Create(dst string) error { + f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer f.Close() + + // Select a writer based off of the WriteLimit configuration option. If there is no + // write limit, use the file as the writer. + var writer io.Writer = f + writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024) + if writeLimit > 0 { + // Token bucket with a capacity of "writeLimit" MiB, adding "writeLimit" MiB/s + // and then wrap the file writer with the token bucket limiter. + writer = ratelimit.Writer(f, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit)) + } + + // Create a new gzip writer around the file. + gw, _ := pgzip.NewWriterLevel(writer, pgzip.BestSpeed) + _ = gw.SetConcurrency(1<<20, 1) + defer gw.Close() + + // Create a new tar writer around the gzip writer. + tw := tar.NewWriter(gw) + defer tw.Close() + + // Configure godirwalk. + options := &godirwalk.Options{ + FollowSymbolicLinks: false, + Unsorted: true, + Callback: a.callback(tw), + } + + // If we're specifically looking for only certain files, or have requested + // that certain files be ignored we'll update the callback function to reflect + // that request. + if len(a.Files) == 0 && len(a.Ignore) > 0 { + i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...) + options.Callback = a.callback(tw, func(_ string, rp string) error { + if i.MatchesPath(rp) { + return godirwalk.SkipThis + } + return nil + }) + } else if len(a.Files) > 0 { + options.Callback = a.withFilesCallback(tw) + } + + // Recursively walk the path we are archiving. + return godirwalk.Walk(a.BasePath, options) +} + +// Callback function used to determine if a given file should be included in the archive +// being generated. +func (a *Archive) callback(tw *tar.Writer, opts ...func(path string, relative string) error) func(path string, de *godirwalk.Dirent) error { + return func(path string, de *godirwalk.Dirent) error { + // Skip directories because we walking them recursively. + if de.IsDir() { + return nil + } + + relative := filepath.ToSlash(strings.TrimPrefix(path, a.BasePath+string(filepath.Separator))) + // Call the additional options passed to this callback function. If any of them return + // a non-nil error we will exit immediately. + for _, opt := range opts { + if err := opt(path, relative); err != nil { + return err + } + } + + // Add the file to the archive, if it is nested in a directory, + // the directory will be automatically "created" in the archive. + return a.addToArchive(path, relative, tw) + } +} + +// Pushes only files defined in the Files key to the final archive. +func (a *Archive) withFilesCallback(tw *tar.Writer) func(path string, de *godirwalk.Dirent) error { + return a.callback(tw, func(p string, rp string) error { + for _, f := range a.Files { + // If the given doesn't match, or doesn't have the same prefix continue + // to the next item in the loop. + if p != f && !strings.HasPrefix(p, f) { + continue + } + // Once we have a match return a nil value here so that the loop stops and the + // call to this function will correctly include the file in the archive. If there + // are no matches we'll never make it to this line, and the final error returned + // will be the godirwalk.SkipThis error. + return nil + } + return godirwalk.SkipThis + }) +} + +// Adds a given file path to the final archive being created. +func (a *Archive) addToArchive(p string, rp string, w *tar.Writer) error { + // Lstat the file, this will give us the same information as Stat except + // that it will not follow a symlink to it's target automatically. + s, err := os.Lstat(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WithMessage(err, "failed to Lstat '"+rp+"'") + } + + // Resolve the symlink target if the file is a symlink. + var target string + if s.Mode()&os.ModeSymlink != 0 { + // Read the target of the symlink. + target, err = os.Readlink(s.Name()) + if err != nil { + return errors.WithMessage(err, "failed to read symlink target for '"+rp+"'") + } + } + + // Get the tar FileInfoHeader in order to add the file to the archive. + header, err := tar.FileInfoHeader(s, filepath.ToSlash(target)) + if err != nil { + return errors.WithMessagef(err, "failed to get tar#FileInfoHeader for '%s'", rp) + } + + // Fix the header name if the file is not a symlink. + if s.Mode()&os.ModeSymlink == 0 { + header.Name = rp + } + + // Write the tar FileInfoHeader to the archive. + if err := w.WriteHeader(header); err != nil { + return errors.WithMessagef(err, "failed to write tar#FileInfoHeader for '%s'", rp) + } + + // If the size of the file is less than 1 (most likely for symlinks), skip writing the file. + if header.Size < 1 { + return nil + } + + // If the buffer size is larger than the file size, create a smaller buffer to hold the file. + var buf []byte + if header.Size < memory { + buf = make([]byte, header.Size) + } else { + // Get a fixed-size buffer from the pool to save on allocations. + buf = pool.Get().([]byte) + defer func() { + buf = make([]byte, memory) + pool.Put(buf) + }() + } + + // Open the file. + f, err := os.Open(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WithMessagef(err, "failed to open '%s' for copying", header.Name) + } + defer f.Close() + // Copy the file's contents to the archive using our buffer. + if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil { + return errors.WithMessagef(err, "failed to copy '%s' to archive", header.Name) + } + return nil +} diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index d5f301c..595c393 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -1,11 +1,7 @@ package filesystem import ( - "context" "fmt" - "github.com/karrick/godirwalk" - "github.com/pterodactyl/wings/server/backup" - ignore "github.com/sabhiram/go-gitignore" "os" "path" "path/filepath" @@ -13,60 +9,6 @@ import ( "time" ) -// Given a directory, iterate through all of the files and folders within it and determine -// if they should be included in the output based on an array of ignored matches. This uses -// standard .gitignore formatting to make that determination. -// -// If no ignored files are passed through you'll get the entire directory listing. -func (fs *Filesystem) GetIncludedFiles(dir string, ignored []string) (*backup.IncludedFiles, error) { - cleaned, err := fs.SafePath(dir) - if err != nil { - return nil, err - } - - i, err := ignore.CompileIgnoreLines(ignored...) - if err != nil { - return nil, err - } - - // Walk through all of the files and directories on a server. This callback only returns - // files found, and will keep walking deeper and deeper into directories. - inc := new(backup.IncludedFiles) - - err = godirwalk.Walk(cleaned, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - sp := p - if e.IsSymlink() { - sp, err = fs.SafePath(p) - if err != nil { - if IsErrorCode(err, ErrCodePathResolution) { - return godirwalk.SkipThis - } - - return err - } - } - - // Only push files into the result array since archives can't create an empty directory within them. - if !e.IsDir() { - // Avoid unnecessary parsing if there are no ignored files, nothing will match anyways - // so no reason to call the function. - if len(ignored) == 0 || !i.MatchesPath(strings.TrimPrefix(sp, fs.Path()+"/")) { - inc.Push(sp) - } - } - - // We can't just abort if the path is technically ignored. It is possible there is a nested - // file or folder that should not be excluded, so in this case we need to just keep going - // until we get to a final state. - return nil - }, - }) - - return inc, err -} - // Compresses all of the files matching the given paths in the specified directory. This function // also supports passing nested paths to only compress certain files and folders when working in // a larger directory. This effectively creates a local backup, but rather than ignoring specific @@ -90,69 +32,24 @@ func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, er return nil, err } - inc := new(backup.IncludedFiles) - // Iterate over all of the cleaned paths and merge them into a large object of final file - // paths to pass into the archiver. As directories are encountered this will drop into them - // and look for all of the files. - for _, p := range cleaned { - f, err := os.Stat(p) - if err != nil { - fs.error(err).WithField("path", p).Debug("failed to stat file or directory for compression") - continue - } + a := &Archive{BasePath: cleanedRootDir, Files: cleaned} + d := path.Join( + cleanedRootDir, + fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")), + ) - if !f.IsDir() { - inc.Push(p) - } else { - err := godirwalk.Walk(p, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - sp := p - if e.IsSymlink() { - // Ensure that any symlinks are properly resolved to their final destination. If - // that destination is outside the server directory skip over this entire item, otherwise - // use the resolved location for the rest of this function. - sp, err = fs.SafePath(p) - if err != nil { - if IsErrorCode(err, ErrCodePathResolution) { - return godirwalk.SkipThis - } - - return err - } - } - - if !e.IsDir() { - inc.Push(sp) - } - - return nil - }, - }) - - if err != nil { - return nil, err - } - } - } - - a := &backup.Archive{TrimPrefix: fs.Path(), Files: inc} - d := path.Join(cleanedRootDir, fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", ""))) - - if err := a.Create(d, context.Background()); err != nil { + if err := a.Create(d); err != nil { return nil, err } f, err := os.Stat(d) if err != nil { _ = os.Remove(d) - return nil, err } if err := fs.HasSpaceFor(f.Size()); err != nil { _ = os.Remove(d) - return nil, err }