Add internal logic to process activity events and send them to the panel

This commit is contained in:
DaneEveritt 2022-07-09 14:38:41 -04:00
parent 0380488cd2
commit 20e44bdc55
No known key found for this signature in database
GPG Key ID: EEA66103B3D71F53
11 changed files with 131 additions and 11 deletions

View File

@ -5,6 +5,7 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/pterodactyl/wings/internal/cron"
log2 "log" log2 "log"
"net/http" "net/http"
_ "net/http/pprof" _ "net/http/pprof"
@ -259,6 +260,13 @@ func rootCmdRun(cmd *cobra.Command, _ []string) {
} }
}() }()
if s, err := cron.Scheduler(manager); err != nil {
log.WithField("error", err).Fatal("failed to initialize cron system")
} else {
log.WithField("subsystem", "cron").Info("starting cron processes")
s.StartAsync()
}
go func() { go func() {
// Run the SFTP server. // Run the SFTP server.
if err := sftp.New(manager).Run(); err != nil { if err := sftp.New(manager).Run(); err != nil {

6
go.mod
View File

@ -37,7 +37,7 @@ require (
github.com/pkg/sftp v1.13.4 github.com/pkg/sftp v1.13.4
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/spf13/cobra v1.4.0 github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1 github.com/stretchr/testify v1.7.5
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
gopkg.in/ini.v1 v1.66.4 gopkg.in/ini.v1 v1.66.4
@ -66,6 +66,7 @@ require (
github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/gammazero/deque v0.1.1 // indirect github.com/gammazero/deque v0.1.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-co-op/gocron v1.15.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-playground/validator/v10 v10.10.1 // indirect github.com/go-playground/validator/v10 v10.10.1 // indirect
@ -97,6 +98,7 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect github.com/prometheus/procfs v0.7.3 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect github.com/ugorji/go/codec v1.2.7 // indirect
@ -115,5 +117,5 @@ require (
google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect
google.golang.org/grpc v1.45.0 // indirect google.golang.org/grpc v1.45.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
) )

9
go.sum
View File

@ -402,6 +402,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
github.com/go-co-op/gocron v1.15.0 h1:XmiPazahD9aq0/QdK5toCVHfgTXfrZ/s83RpAgzr6SM=
github.com/go-co-op/gocron v1.15.0/go.mod h1:On9zUZTv7FBeuj9D/cdYyAWcPUiLqqAx7nsPHd0EmKM=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@ -870,6 +872,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@ -934,6 +938,7 @@ github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@ -944,6 +949,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@ -1572,6 +1579,8 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

View File

@ -0,0 +1,55 @@
package cron
import (
"context"
"emperror.dev/errors"
"github.com/pterodactyl/wings/internal/database"
"github.com/pterodactyl/wings/server"
"github.com/pterodactyl/wings/system"
"github.com/xujiajun/nutsdb"
)
var processing system.AtomicBool
func processActivityLogs(m *server.Manager) error {
// Don't execute this cron if there is currently one running. Once this task is completed
// go ahead and mark it as no longer running.
if !processing.SwapIf(true) {
return nil
}
defer processing.Store(false)
var b [][]byte
err := database.DB().View(func(tx *nutsdb.Tx) error {
// Grab the oldest 100 activity events that have been logged and send them back to the
// Panel for processing. Once completed, delete those events from the database and then
// release the lock on this process.
list, err := tx.LRange(database.ServerActivityBucket, []byte("events"), 0, 1)
if err != nil {
// This error is returned when the bucket doesn't exist, which is likely on the
// first invocations of Wings since we haven't yet logged any data. There is nothing
// that needs to be done if this error occurs.
if errors.Is(err, nutsdb.ErrBucket) {
return nil
}
return errors.WithStackIf(err)
}
b = list
return nil
})
// If there is an error, return it. If there is no data to send to the Panel don't waste
// an API call, just return here. WithStackIf will return "nil" when the value provided to
// it is also nil.
if err != nil || len(b) == 0 {
return errors.WithStackIf(err)
}
if err := m.Client().SendActivityLogs(context.Background(), b); err != nil {
return errors.WrapIf(err, "cron: failed to send activity events to Panel")
}
return database.DB().Update(func(tx *nutsdb.Tx) error {
return tx.LTrim(database.ServerActivityBucket, []byte("events"), 2, -1)
})
}

36
internal/cron/cron.go Normal file
View File

@ -0,0 +1,36 @@
package cron
import (
"emperror.dev/errors"
"github.com/apex/log"
"github.com/go-co-op/gocron"
"github.com/pterodactyl/wings/config"
"github.com/pterodactyl/wings/server"
"github.com/pterodactyl/wings/system"
"time"
)
var o system.AtomicBool
// Scheduler configures the internal cronjob system for Wings and returns the scheduler
// instance to the caller. This should only be called once per application lifecycle, additional
// calls will result in an error being returned.
func Scheduler(m *server.Manager) (*gocron.Scheduler, error) {
if o.Load() {
return nil, errors.New("cron: cannot call scheduler more than once in application lifecycle")
}
o.Store(true)
l, err := time.LoadLocation(config.Get().System.Timezone)
if err != nil {
return nil, errors.Wrap(err, "cron: failed to parse configured system timezone")
}
s := gocron.NewScheduler(l)
_, _ = s.Every(5).Seconds().Do(func() {
if err := processActivityLogs(m); err != nil {
log.WithField("error", err).Error("cron: failed to process activity events")
}
})
return s, nil
}

View File

@ -12,8 +12,8 @@ import (
var db *nutsdb.DB var db *nutsdb.DB
var syncer sync.Once var syncer sync.Once
var ( const (
ServerEventsBucket = "server_events" ServerActivityBucket = "server_activity"
) )
func initialize() error { func initialize() error {

View File

@ -30,6 +30,7 @@ type Client interface {
SetInstallationStatus(ctx context.Context, uuid string, successful bool) error SetInstallationStatus(ctx context.Context, uuid string, successful bool) error
SetTransferStatus(ctx context.Context, uuid string, successful bool) error SetTransferStatus(ctx context.Context, uuid string, successful bool) error
ValidateSftpCredentials(ctx context.Context, request SftpAuthRequest) (SftpAuthResponse, error) ValidateSftpCredentials(ctx context.Context, request SftpAuthRequest) (SftpAuthResponse, error)
SendActivityLogs(ctx context.Context, activity [][]byte) error
} }
type client struct { type client struct {

View File

@ -178,6 +178,16 @@ func (c *client) SendRestorationStatus(ctx context.Context, backup string, succe
return nil return nil
} }
// SendActivityLogs sends activity logs back to the Panel for processing.
func (c *client) SendActivityLogs(ctx context.Context, activity [][]byte) error {
resp, err := c.Post(ctx, "/activty", d{"data": activity})
if err != nil {
return errors.WithStackIf(err)
}
_ = resp.Body.Close()
return nil
}
// getServersPaged returns a subset of servers from the Panel API using the // getServersPaged returns a subset of servers from the Panel API using the
// pagination query parameters. // pagination query parameters.
func (c *client) getServersPaged(ctx context.Context, page, limit int) ([]RawServerData, Pagination, error) { func (c *client) getServersPaged(ctx context.Context, page, limit int) ([]RawServerData, Pagination, error) {

View File

@ -2,11 +2,10 @@ package remote
import ( import (
"bytes" "bytes"
"regexp"
"strings"
"github.com/apex/log" "github.com/apex/log"
"github.com/goccy/go-json" "github.com/goccy/go-json"
"regexp"
"strings"
"github.com/pterodactyl/wings/parser" "github.com/pterodactyl/wings/parser"
) )

View File

@ -3,7 +3,7 @@ package router
import ( import (
"context" "context"
"github.com/goccy/go-json" "github.com/goccy/go-json"
"github.com/pterodactyl/wings/database" "github.com/pterodactyl/wings/internal/database"
"github.com/xujiajun/nutsdb" "github.com/xujiajun/nutsdb"
"net/http" "net/http"
"os" "os"
@ -55,7 +55,7 @@ func getServerActivityLogs(c *gin.Context) {
var out [][]byte var out [][]byte
err := database.DB().View(func(tx *nutsdb.Tx) error { err := database.DB().View(func(tx *nutsdb.Tx) error {
items, err := tx.LRange(database.ServerEventsBucket, []byte(s.ID()), 0, 10) items, err := tx.LRange(database.ServerActivityBucket, []byte(s.ID()), 0, 10)
if err != nil { if err != nil {
return err return err
} }

View File

@ -4,7 +4,7 @@ import (
"emperror.dev/errors" "emperror.dev/errors"
"github.com/apex/log" "github.com/apex/log"
"github.com/goccy/go-json" "github.com/goccy/go-json"
"github.com/pterodactyl/wings/database" "github.com/pterodactyl/wings/internal/database"
"github.com/xujiajun/nutsdb" "github.com/xujiajun/nutsdb"
"time" "time"
) )
@ -99,7 +99,7 @@ func (a Activity) Save() error {
WithFields(log.Fields{"server": a.Server, "user": a.User, "event": a.Event, "ip": a.IP}). WithFields(log.Fields{"server": a.Server, "user": a.User, "event": a.Event, "ip": a.IP}).
Debug("saving activity to database") Debug("saving activity to database")
if err := tx.RPush(database.ServerEventsBucket, []byte(a.Server), value); err != nil { if err := tx.RPush(database.ServerActivityBucket, []byte("events"), value); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
return nil return nil