de30e2fcc9
The stat call is operating against an unflushed file if called in the archive function, so you'll just get the emtpy archive size, rather than the final size. Plus, we only used the file stat in one place, so slight efficiency win?
133 lines
3.1 KiB
Go
133 lines
3.1 KiB
Go
package backup
|
|
|
|
import (
|
|
"archive/tar"
|
|
"context"
|
|
"github.com/apex/log"
|
|
gzip "github.com/klauspost/pgzip"
|
|
"github.com/pkg/errors"
|
|
"github.com/remeh/sizedwaitgroup"
|
|
"golang.org/x/sync/errgroup"
|
|
"io"
|
|
"os"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
type Archive struct {
|
|
sync.Mutex
|
|
|
|
TrimPrefix string
|
|
Files *IncludedFiles
|
|
}
|
|
|
|
// Creates an archive at dst with all of the files defined in the included files struct.
|
|
func (a *Archive) Create(dst string, ctx context.Context) error {
|
|
f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
return errors.WithStack(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
maxCpu := runtime.NumCPU() / 2
|
|
if maxCpu > 4 {
|
|
maxCpu = 4
|
|
}
|
|
|
|
gzw, _ := gzip.NewWriterLevel(f, gzip.BestSpeed)
|
|
_ = gzw.SetConcurrency(1<<20, maxCpu)
|
|
|
|
defer gzw.Flush()
|
|
defer gzw.Close()
|
|
|
|
tw := tar.NewWriter(gzw)
|
|
defer tw.Flush()
|
|
defer tw.Close()
|
|
|
|
wg := sizedwaitgroup.New(10)
|
|
g, ctx := errgroup.WithContext(ctx)
|
|
// Iterate over all of the files to be included and put them into the archive. This is
|
|
// done as a concurrent goroutine to speed things along. If an error is encountered at
|
|
// any step, the entire process is aborted.
|
|
for _, p := range a.Files.All() {
|
|
p := p
|
|
g.Go(func() error {
|
|
wg.Add()
|
|
defer wg.Done()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return errors.WithStack(ctx.Err())
|
|
default:
|
|
return a.addToArchive(p, tw)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Block until the entire routine is completed.
|
|
if err := g.Wait(); err != nil {
|
|
f.Close()
|
|
|
|
// Attempt to remove the archive if there is an error, report that error to
|
|
// the logger if it fails.
|
|
if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
|
|
log.WithField("location", dst).Warn("failed to delete corrupted backup archive")
|
|
}
|
|
|
|
return errors.WithStack(err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Adds a single file to the existing tar archive writer.
|
|
func (a *Archive) addToArchive(p string, w *tar.Writer) error {
|
|
f, err := os.Open(p)
|
|
if err != nil {
|
|
// If you try to backup something that no longer exists (got deleted somewhere during the process
|
|
// but not by this process), just skip over it and don't kill the entire backup.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
|
|
return errors.WithStack(err)
|
|
}
|
|
defer f.Close()
|
|
|
|
s, err := f.Stat()
|
|
if err != nil {
|
|
// Same as above, don't kill the process just because the file no longer exists.
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
|
|
return errors.WithStack(err)
|
|
}
|
|
|
|
header := &tar.Header{
|
|
// Trim the long server path from the name of the file so that the resulting
|
|
// archive is exactly how the user would see it in the panel file manager.
|
|
Name: strings.TrimPrefix(p, a.TrimPrefix),
|
|
Size: s.Size(),
|
|
Mode: int64(s.Mode()),
|
|
ModTime: s.ModTime(),
|
|
}
|
|
|
|
// These actions must occur sequentially, even if this function is called multiple
|
|
// in parallel. You'll get some nasty panic's otherwise.
|
|
a.Lock()
|
|
defer a.Unlock()
|
|
|
|
if err := w.WriteHeader(header); err != nil {
|
|
return errors.WithStack(err)
|
|
}
|
|
|
|
buf := make([]byte, 4*1024)
|
|
if _, err := io.CopyBuffer(w, f, buf); err != nil {
|
|
return errors.WithStack(err)
|
|
}
|
|
|
|
return nil
|
|
}
|