2021-03-07 18:02:03 +00:00
|
|
|
package filesystem
|
2020-12-25 19:52:57 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"archive/tar"
|
2021-01-18 05:05:51 +00:00
|
|
|
"io"
|
2021-07-12 16:17:56 +00:00
|
|
|
"io/fs"
|
2021-01-18 05:05:51 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
2022-10-05 02:35:48 +00:00
|
|
|
"sync/atomic"
|
2021-01-18 05:05:51 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
"emperror.dev/errors"
|
2020-12-28 00:30:00 +00:00
|
|
|
"github.com/apex/log"
|
2020-12-25 19:52:57 +00:00
|
|
|
"github.com/juju/ratelimit"
|
|
|
|
"github.com/karrick/godirwalk"
|
|
|
|
"github.com/klauspost/pgzip"
|
2021-08-02 21:07:00 +00:00
|
|
|
ignore "github.com/sabhiram/go-gitignore"
|
2021-07-12 16:17:56 +00:00
|
|
|
|
|
|
|
"github.com/pterodactyl/wings/config"
|
2022-10-05 02:35:48 +00:00
|
|
|
"github.com/pterodactyl/wings/system"
|
2020-12-25 19:52:57 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const memory = 4 * 1024
|
|
|
|
|
|
|
|
var pool = sync.Pool{
|
|
|
|
New: func() interface{} {
|
|
|
|
b := make([]byte, memory)
|
|
|
|
return b
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2022-11-06 20:38:30 +00:00
|
|
|
// TarProgress .
|
|
|
|
type TarProgress struct {
|
|
|
|
*tar.Writer
|
|
|
|
p *Progress
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewTarProgress .
|
|
|
|
func NewTarProgress(w *tar.Writer, p *Progress) *TarProgress {
|
|
|
|
if p != nil {
|
|
|
|
p.w = w
|
|
|
|
}
|
|
|
|
return &TarProgress{
|
|
|
|
Writer: w,
|
|
|
|
p: p,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *TarProgress) Write(v []byte) (int, error) {
|
|
|
|
if p.p == nil {
|
|
|
|
return p.Writer.Write(v)
|
|
|
|
}
|
|
|
|
return p.p.Write(v)
|
|
|
|
}
|
|
|
|
|
2022-10-05 02:35:48 +00:00
|
|
|
// Progress is used to track the progress of any I/O operation that are being
|
|
|
|
// performed.
|
|
|
|
type Progress struct {
|
|
|
|
// written is the total size of the files that have been written to the writer.
|
|
|
|
written int64
|
|
|
|
// Total is the total size of the archive in bytes.
|
|
|
|
total int64
|
|
|
|
// w .
|
|
|
|
w io.Writer
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewProgress .
|
|
|
|
func NewProgress(total int64) *Progress {
|
|
|
|
return &Progress{total: total}
|
|
|
|
}
|
|
|
|
|
2022-11-06 20:38:30 +00:00
|
|
|
// SetWriter sets the writer progress will forward writes to.
|
|
|
|
// NOTE: This function is not thread safe.
|
|
|
|
func (p *Progress) SetWriter(w io.Writer) {
|
|
|
|
p.w = w
|
|
|
|
}
|
|
|
|
|
2022-10-05 02:35:48 +00:00
|
|
|
// Written returns the total number of bytes written.
|
|
|
|
// This function should be used when the progress is tracking data being written.
|
|
|
|
func (p *Progress) Written() int64 {
|
|
|
|
return atomic.LoadInt64(&p.written)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Total returns the total size in bytes.
|
|
|
|
func (p *Progress) Total() int64 {
|
|
|
|
return atomic.LoadInt64(&p.total)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write totals the number of bytes that have been written to the writer.
|
|
|
|
func (p *Progress) Write(v []byte) (int, error) {
|
|
|
|
n := len(v)
|
|
|
|
atomic.AddInt64(&p.written, int64(n))
|
|
|
|
if p.w != nil {
|
|
|
|
return p.w.Write(v)
|
|
|
|
}
|
|
|
|
return n, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Progress returns a formatted progress string for the current progress.
|
|
|
|
func (p *Progress) Progress(width int) string {
|
2022-11-04 17:24:19 +00:00
|
|
|
// current = 100 (Progress, dynamic)
|
|
|
|
// total = 1000 (Content-Length, dynamic)
|
|
|
|
// width = 25 (Number of ticks to display, static)
|
|
|
|
// widthPercentage = 100 / width (What percentage does each tick represent, static)
|
|
|
|
//
|
|
|
|
// percentageDecimal = current / total = 0.1
|
|
|
|
// percentage = percentageDecimal * 100 = 10%
|
|
|
|
// ticks = percentage / widthPercentage = 2.5
|
|
|
|
//
|
|
|
|
// ticks is a float64, so we cast it to an int which rounds it down to 2.
|
|
|
|
|
|
|
|
// Values are cast to floats to prevent integer division.
|
2022-10-05 02:35:48 +00:00
|
|
|
current := p.Written()
|
|
|
|
total := p.Total()
|
2022-11-04 17:24:19 +00:00
|
|
|
// width := is passed as a parameter
|
|
|
|
widthPercentage := float64(100) / float64(width)
|
|
|
|
percentageDecimal := float64(current) / float64(total)
|
|
|
|
percentage := percentageDecimal * 100
|
|
|
|
ticks := int(percentage / widthPercentage)
|
|
|
|
|
|
|
|
// Ensure that we never get a negative number of ticks, this will prevent strings#Repeat
|
|
|
|
// from panicking. A negative number of ticks is likely to happen when the total size is
|
|
|
|
// inaccurate, such as when we are going off of rough disk usage calculation.
|
|
|
|
if ticks < 0 {
|
|
|
|
ticks = 0
|
|
|
|
} else if ticks > width {
|
|
|
|
ticks = width
|
|
|
|
}
|
2022-10-05 02:35:48 +00:00
|
|
|
|
2022-11-04 17:24:19 +00:00
|
|
|
bar := strings.Repeat("=", ticks) + strings.Repeat(" ", width-ticks)
|
2022-10-05 02:35:48 +00:00
|
|
|
return "[" + bar + "] " + system.FormatBytes(current) + " / " + system.FormatBytes(total)
|
|
|
|
}
|
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
type Archive struct {
|
|
|
|
// BasePath is the absolute path to create the archive from where Files and Ignore are
|
|
|
|
// relative to.
|
|
|
|
BasePath string
|
|
|
|
|
|
|
|
// Ignore is a gitignore string (most likely read from a file) of files to ignore
|
|
|
|
// from the archive.
|
|
|
|
Ignore string
|
|
|
|
|
|
|
|
// Files specifies the files to archive, this takes priority over the Ignore option, if
|
|
|
|
// unspecified, all files in the BasePath will be archived unless Ignore is set.
|
|
|
|
Files []string
|
2022-10-05 02:35:48 +00:00
|
|
|
|
|
|
|
// Progress wraps the writer of the archive to pass through the progress tracker.
|
|
|
|
Progress *Progress
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
2022-10-05 02:35:48 +00:00
|
|
|
// Create creates an archive at dst with all the files defined in the
|
|
|
|
// included Files array.
|
2020-12-25 19:52:57 +00:00
|
|
|
func (a *Archive) Create(dst string) error {
|
2021-11-15 17:37:56 +00:00
|
|
|
f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
2020-12-25 19:52:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
// Select a writer based off of the WriteLimit configuration option. If there is no
|
|
|
|
// write limit, use the file as the writer.
|
2020-12-27 19:21:26 +00:00
|
|
|
var writer io.Writer
|
2020-12-27 18:49:08 +00:00
|
|
|
if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 {
|
2020-12-25 19:52:57 +00:00
|
|
|
// Token bucket with a capacity of "writeLimit" MiB, adding "writeLimit" MiB/s
|
|
|
|
// and then wrap the file writer with the token bucket limiter.
|
|
|
|
writer = ratelimit.Writer(f, ratelimit.NewBucketWithRate(float64(writeLimit), writeLimit))
|
2020-12-27 19:21:26 +00:00
|
|
|
} else {
|
|
|
|
writer = f
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
2022-09-26 00:47:09 +00:00
|
|
|
// Choose which compression level to use based on the compression_level configuration option
|
2022-10-05 02:35:48 +00:00
|
|
|
var compressionLevel int
|
2022-09-26 00:47:09 +00:00
|
|
|
switch config.Get().System.Backups.CompressionLevel {
|
|
|
|
case "none":
|
2022-10-05 02:35:48 +00:00
|
|
|
compressionLevel = pgzip.NoCompression
|
2022-09-26 00:47:09 +00:00
|
|
|
case "best_compression":
|
2022-10-05 02:35:48 +00:00
|
|
|
compressionLevel = pgzip.BestCompression
|
|
|
|
case "best_speed":
|
|
|
|
fallthrough
|
|
|
|
default:
|
|
|
|
compressionLevel = pgzip.BestSpeed
|
2022-09-26 00:47:09 +00:00
|
|
|
}
|
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Create a new gzip writer around the file.
|
2022-10-05 02:35:48 +00:00
|
|
|
gw, _ := pgzip.NewWriterLevel(writer, compressionLevel)
|
2020-12-25 19:52:57 +00:00
|
|
|
_ = gw.SetConcurrency(1<<20, 1)
|
|
|
|
defer gw.Close()
|
|
|
|
|
|
|
|
// Create a new tar writer around the gzip writer.
|
2022-11-06 20:38:30 +00:00
|
|
|
tw := tar.NewWriter(gw)
|
2020-12-25 19:52:57 +00:00
|
|
|
defer tw.Close()
|
|
|
|
|
2022-11-06 20:38:30 +00:00
|
|
|
pw := NewTarProgress(tw, a.Progress)
|
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Configure godirwalk.
|
|
|
|
options := &godirwalk.Options{
|
|
|
|
FollowSymbolicLinks: false,
|
|
|
|
Unsorted: true,
|
2022-11-06 20:38:30 +00:00
|
|
|
Callback: a.callback(pw),
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If we're specifically looking for only certain files, or have requested
|
|
|
|
// that certain files be ignored we'll update the callback function to reflect
|
|
|
|
// that request.
|
|
|
|
if len(a.Files) == 0 && len(a.Ignore) > 0 {
|
|
|
|
i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...)
|
2020-12-27 20:55:58 +00:00
|
|
|
|
2022-11-06 20:38:30 +00:00
|
|
|
options.Callback = a.callback(pw, func(_ string, rp string) error {
|
2020-12-25 19:52:57 +00:00
|
|
|
if i.MatchesPath(rp) {
|
|
|
|
return godirwalk.SkipThis
|
|
|
|
}
|
2020-12-27 20:55:58 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
} else if len(a.Files) > 0 {
|
2022-11-06 20:38:30 +00:00
|
|
|
options.Callback = a.withFilesCallback(pw)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Recursively walk the path we are archiving.
|
|
|
|
return godirwalk.Walk(a.BasePath, options)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Callback function used to determine if a given file should be included in the archive
|
|
|
|
// being generated.
|
2022-11-06 20:38:30 +00:00
|
|
|
func (a *Archive) callback(tw *TarProgress, opts ...func(path string, relative string) error) func(path string, de *godirwalk.Dirent) error {
|
2020-12-25 19:52:57 +00:00
|
|
|
return func(path string, de *godirwalk.Dirent) error {
|
2022-10-05 02:35:48 +00:00
|
|
|
// Skip directories because we are walking them recursively.
|
2020-12-25 19:52:57 +00:00
|
|
|
if de.IsDir() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
relative := filepath.ToSlash(strings.TrimPrefix(path, a.BasePath+string(filepath.Separator)))
|
2020-12-27 20:55:58 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Call the additional options passed to this callback function. If any of them return
|
|
|
|
// a non-nil error we will exit immediately.
|
|
|
|
for _, opt := range opts {
|
|
|
|
if err := opt(path, relative); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add the file to the archive, if it is nested in a directory,
|
|
|
|
// the directory will be automatically "created" in the archive.
|
|
|
|
return a.addToArchive(path, relative, tw)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Pushes only files defined in the Files key to the final archive.
|
2022-11-06 20:38:30 +00:00
|
|
|
func (a *Archive) withFilesCallback(tw *TarProgress) func(path string, de *godirwalk.Dirent) error {
|
2020-12-25 19:52:57 +00:00
|
|
|
return a.callback(tw, func(p string, rp string) error {
|
|
|
|
for _, f := range a.Files {
|
|
|
|
// If the given doesn't match, or doesn't have the same prefix continue
|
|
|
|
// to the next item in the loop.
|
2022-05-12 22:00:55 +00:00
|
|
|
if p != f && !strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", f) {
|
2020-12-25 19:52:57 +00:00
|
|
|
continue
|
|
|
|
}
|
2020-12-27 20:55:58 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Once we have a match return a nil value here so that the loop stops and the
|
|
|
|
// call to this function will correctly include the file in the archive. If there
|
|
|
|
// are no matches we'll never make it to this line, and the final error returned
|
|
|
|
// will be the godirwalk.SkipThis error.
|
|
|
|
return nil
|
|
|
|
}
|
2020-12-27 20:55:58 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
return godirwalk.SkipThis
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// Adds a given file path to the final archive being created.
|
2022-11-06 20:38:30 +00:00
|
|
|
func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error {
|
2020-12-28 00:53:40 +00:00
|
|
|
// Lstat the file, this will give us the same information as Stat except that it will not
|
2022-09-25 19:34:28 +00:00
|
|
|
// follow a symlink to its target automatically. This is important to avoid including
|
2020-12-28 00:53:40 +00:00
|
|
|
// files that exist outside the server root unintentionally in the backup.
|
2020-12-25 19:52:57 +00:00
|
|
|
s, err := os.Lstat(p)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
}
|
2020-12-28 00:30:00 +00:00
|
|
|
return errors.WrapIff(err, "failed executing os.Lstat on '%s'", rp)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
2021-07-12 16:17:56 +00:00
|
|
|
// Skip socket files as they are unsupported by archive/tar.
|
|
|
|
// Error will come from tar#FileInfoHeader: "archive/tar: sockets not supported"
|
|
|
|
if s.Mode()&fs.ModeSocket != 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Resolve the symlink target if the file is a symlink.
|
|
|
|
var target string
|
2021-07-12 16:17:56 +00:00
|
|
|
if s.Mode()&fs.ModeSymlink != 0 {
|
2020-12-28 00:30:00 +00:00
|
|
|
// Read the target of the symlink. If there are any errors we will dump them out to
|
|
|
|
// the logs, but we're not going to stop the backup. There are far too many cases of
|
|
|
|
// symlinks causing all sorts of unnecessary pain in this process. Sucks to suck if
|
|
|
|
// it doesn't work.
|
2020-12-25 19:52:57 +00:00
|
|
|
target, err = os.Readlink(s.Name())
|
2020-12-27 19:54:18 +00:00
|
|
|
if err != nil {
|
2022-11-04 17:24:19 +00:00
|
|
|
// Ignore the not exist errors specifically, since there is nothing important about that.
|
2020-12-28 00:30:00 +00:00
|
|
|
if !os.IsNotExist(err) {
|
|
|
|
log.WithField("path", rp).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...")
|
2020-12-27 20:56:45 +00:00
|
|
|
}
|
2020-12-28 00:30:00 +00:00
|
|
|
return nil
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the tar FileInfoHeader in order to add the file to the archive.
|
|
|
|
header, err := tar.FileInfoHeader(s, filepath.ToSlash(target))
|
|
|
|
if err != nil {
|
2020-12-27 20:55:58 +00:00
|
|
|
return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", rp)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Fix the header name if the file is not a symlink.
|
2021-07-12 16:17:56 +00:00
|
|
|
if s.Mode()&fs.ModeSymlink == 0 {
|
2020-12-25 19:52:57 +00:00
|
|
|
header.Name = rp
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write the tar FileInfoHeader to the archive.
|
|
|
|
if err := w.WriteHeader(header); err != nil {
|
2020-12-27 20:55:58 +00:00
|
|
|
return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", rp)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If the size of the file is less than 1 (most likely for symlinks), skip writing the file.
|
|
|
|
if header.Size < 1 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// If the buffer size is larger than the file size, create a smaller buffer to hold the file.
|
|
|
|
var buf []byte
|
|
|
|
if header.Size < memory {
|
|
|
|
buf = make([]byte, header.Size)
|
|
|
|
} else {
|
|
|
|
// Get a fixed-size buffer from the pool to save on allocations.
|
|
|
|
buf = pool.Get().([]byte)
|
|
|
|
defer func() {
|
|
|
|
buf = make([]byte, memory)
|
|
|
|
pool.Put(buf)
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open the file.
|
|
|
|
f, err := os.Open(p)
|
|
|
|
if err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return nil
|
|
|
|
}
|
2020-12-27 20:55:58 +00:00
|
|
|
return errors.WrapIff(err, "failed to open '%s' for copying", header.Name)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
|
|
|
defer f.Close()
|
2020-12-27 18:49:08 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
// Copy the file's contents to the archive using our buffer.
|
|
|
|
if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil {
|
2020-12-27 20:55:58 +00:00
|
|
|
return errors.WrapIff(err, "failed to copy '%s' to archive", header.Name)
|
2020-12-25 19:52:57 +00:00
|
|
|
}
|
2020-12-27 18:49:08 +00:00
|
|
|
|
2020-12-25 19:52:57 +00:00
|
|
|
return nil
|
|
|
|
}
|