From 7871c0928f8f5f0082a53b7553ba7cc39534f40f Mon Sep 17 00:00:00 2001 From: Dane Everitt Date: Sun, 6 Dec 2020 15:57:03 -0800 Subject: [PATCH 1/5] Remove --brute; hella slow and not much of a difference in the end --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29481a7..ac22b06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - name: Compress binary and make it executable run: | - upx --brute build/wings_linux_amd64 && chmod +x build/wings_linux_amd64 + upx build/wings_linux_amd64 && chmod +x build/wings_linux_amd64 upx build/wings_linux_arm64 && chmod +x build/wings_linux_arm64 upx build/wings_linux_arm && chmod +x build/wings_linux_arm From cff76671559eefc3490f367aaf9aaa93f49f5b75 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Tue, 8 Dec 2020 09:11:15 -0700 Subject: [PATCH 2/5] fix build-test workflow running twice on pull_requests --- .github/workflows/build-test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c2f4596..43d05e3 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,10 +1,18 @@ name: Run Tests + on: push: + branches: + - 'develop' + branches-ignore: - 'master' - 'release/**' + pull_request: + branches: + - 'develop' + jobs: build: strategy: From fb3460f5f6a22cf3b0873cb6555510e6b5311182 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Tue, 8 Dec 2020 09:12:16 -0700 Subject: [PATCH 3/5] Fix build-test workflow --- .github/workflows/build-test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 43d05e3..6d15cab 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -5,10 +5,6 @@ on: branches: - 'develop' - branches-ignore: - - 'master' - - 'release/**' - pull_request: branches: - 'develop' From 2d4dd05ec9f1d871053213beab3bffadeee78397 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Tue, 8 Dec 2020 09:12:45 -0700 Subject: [PATCH 4/5] Remove arm from build-test workflow --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6d15cab..7f33fa5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,7 +19,7 @@ jobs: os: [ ubuntu-20.04 ] go: [ 1.15.6 ] goos: [ linux ] - goarch: [ amd64, arm, arm64 ] + goarch: [ amd64, arm64 ] runs-on: ${{ matrix.os }} From 8e29ffed50648926ec4e9ed7559ffdfa3eb0596c Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Tue, 8 Dec 2020 09:13:48 -0700 Subject: [PATCH 5/5] Add configurable disk write speed limit for backups (#74) * Add configurable disk write speed limit for backups --- config/config_system.go | 14 ++++++++++++++ go.mod | 1 + go.sum | 2 ++ server/backup/archiver.go | 24 ++++++++++++++++++++++-- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/config/config_system.go b/config/config_system.go index 848ddca..ac9021a 100644 --- a/config/config_system.go +++ b/config/config_system.go @@ -75,6 +75,8 @@ type SystemConfiguration struct { Sftp SftpConfiguration `yaml:"sftp"` CrashDetection CrashDetection `yaml:"crash_detection"` + + Backups Backups `yaml:"backups"` } type CrashDetection struct { @@ -89,6 +91,18 @@ type CrashDetection struct { Timeout int `default:"60" json:"timeout"` } +type Backups struct { + // WriteLimit imposes a Disk I/O write limit on backups to the disk, this affects all + // backup drivers as the archiver must first write the file to the disk in order to + // upload it to any external storage provider. + // + // If the value is less than 1, the write speed is unlimited, + // if the value is greater than 0, the write speed is the value in MB/s. + // + // Defaults to 0 (unlimited) + WriteLimit int `default:"0" yaml:"write_limit"` +} + // Ensures that all of the system directories exist on the system. These directories are // created so that only the owner can read the data, and no other users. func (sc *SystemConfiguration) ConfigureDirectories() error { diff --git a/go.mod b/go.mod index 746f147..f96f3ff 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/icza/dyno v0.0.0-20200205103839-49cb13720835 github.com/imdario/mergo v0.3.8 + github.com/juju/ratelimit v1.0.1 github.com/karrick/godirwalk v1.16.1 github.com/klauspost/compress v1.10.10 // indirect github.com/klauspost/pgzip v1.2.4 diff --git a/go.sum b/go.sum index db0dd5d..686af03 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,8 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= +github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= diff --git a/server/backup/archiver.go b/server/backup/archiver.go index bfb8d4a..c7ef45f 100644 --- a/server/backup/archiver.go +++ b/server/backup/archiver.go @@ -4,8 +4,10 @@ import ( "archive/tar" "context" "github.com/apex/log" + "github.com/juju/ratelimit" gzip "github.com/klauspost/pgzip" "github.com/pkg/errors" + "github.com/pterodactyl/wings/config" "github.com/remeh/sizedwaitgroup" "golang.org/x/sync/errgroup" "io" @@ -30,13 +32,31 @@ func (a *Archive) Create(dst string, ctx context.Context) error { } defer f.Close() + // Select a writer based off of the WriteLimit configuration option. + var writer io.Writer + if writeLimit := config.Get().System.Backups.WriteLimit; writeLimit < 1 { + // If there is no write limit, use the file as the writer. + writer = f + } else { + // Token bucket with a capacity of "writeLimit" MiB, adding "writeLimit" MiB/s + bucket := ratelimit.NewBucketWithRate(float64(writeLimit)*1024*1024, int64(writeLimit)*1024*1024) + + // Wrap the file writer with the token bucket limiter. + writer = ratelimit.Writer(f, bucket) + } + maxCpu := runtime.NumCPU() / 2 if maxCpu > 4 { maxCpu = 4 } - gzw, _ := gzip.NewWriterLevel(f, gzip.BestSpeed) - _ = gzw.SetConcurrency(1<<20, maxCpu) + gzw, err := gzip.NewWriterLevel(writer, gzip.BestSpeed) + if err != nil { + return errors.WithMessage(err, "failed to create gzip writer") + } + if err := gzw.SetConcurrency(1<<20, maxCpu); err != nil { + return errors.WithMessage(err, "failed to set gzip concurrency") + } defer gzw.Flush() defer gzw.Close()