Switch to SQLite for activity tracking

internal/cron/activity_cron.go

@@ -1,102 +1,102 @@
 package cron
 
 import (
+	"bytes"
 	"context"
+	"database/sql"
 	"emperror.dev/errors"
-	"github.com/apex/log"
-	"github.com/goccy/go-json"
-	"github.com/pterodactyl/wings/internal/database"
+	"encoding/gob"
+	"github.com/pterodactyl/wings/internal/sqlite"
 	"github.com/pterodactyl/wings/server"
 	"github.com/pterodactyl/wings/system"
-	"github.com/xujiajun/nutsdb"
+	"strings"
 )
 
-var key = []byte("events")
-var activityCron system.AtomicBool
+type activityCron struct {
+	mu      *system.AtomicBool
+	manager *server.Manager
+	max     int64
+}
 
-func processActivityLogs(m *server.Manager, c int64) error {
+const queryRegularActivity = `
+SELECT id, event, user_uuid, server_uuid, metadata, ip, timestamp FROM activity_logs
+	WHERE event NOT LIKE 'server:sftp.%'
+	ORDER BY timestamp
+	LIMIT ?
+`
+
+type QueriedActivity struct {
+	id int
+	b  []byte
+	server.Activity
+}
+
+// Parse parses the internal query results into the QueriedActivity type and then properly
+// sets the Metadata onto it. This also sets the ID that was returned to ensure we're able
+// to then delete all of the matching rows in the database after we're done.
+func (qa *QueriedActivity) Parse(r *sql.Rows) error {
+	if err := r.Scan(&qa.id, &qa.Event, &qa.User, &qa.Server, &qa.b, &qa.IP, &qa.Timestamp); err != nil {
+		return errors.Wrap(err, "cron: failed to parse activity log")
+	}
+	if err := gob.NewDecoder(bytes.NewBuffer(qa.b)).Decode(&qa.Metadata); err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+// Run executes the cronjob and ensures we fetch and send all of the stored activity to the
+// Panel instance. Once activity is sent it is deleted from the local database instance. Any
+// SFTP specific events are not handled in this cron, they're handled seperately to account
+// for de-duplication and event merging.
+func (ac *activityCron) Run(ctx context.Context) error {
 	// Don't execute this cron if there is currently one running. Once this task is completed
 	// go ahead and mark it as no longer running.
-	if !activityCron.SwapIf(true) {
-		log.WithField("subsystem", "cron").WithField("cron", "activity_logs").Warn("cron: process overlap detected, skipping this run")
+	if !ac.mu.SwapIf(true) {
+		return errors.WithStack(ErrCronRunning)
+	}
+	defer ac.mu.Store(false)
+
+	rows, err := sqlite.Instance().QueryContext(ctx, queryRegularActivity, ac.max)
+	if err != nil {
+		return errors.Wrap(err, "cron: failed to query activity logs")
+	}
+	defer rows.Close()
+
+	var logs []server.Activity
+	var ids []int
+	for rows.Next() {
+		var qa QueriedActivity
+		if err := qa.Parse(rows); err != nil {
+			return err
+		}
+		ids = append(ids, qa.id)
+		logs = append(logs, qa.Activity)
+	}
+
+	if err := rows.Err(); err != nil {
+		return errors.WithStack(err)
+	}
+	if len(logs) == 0 {
 		return nil
 	}
-	defer activityCron.Store(false)
-
-	var list [][]byte
-	err := database.DB().View(func(tx *nutsdb.Tx) error {
-		// Grab the oldest 100 activity events that have been logged and send them back to the
-		// Panel for processing. Once completed, delete those events from the database and then
-		// release the lock on this process.
-		end := int(c)
-		if s, err := tx.LSize(database.ServerActivityBucket, key); err != nil {
-			if errors.Is(err, nutsdb.ErrBucket) {
-				return nil
-			}
-			return errors.WithStackIf(err)
-		} else if s < end || s == 0 {
-			if s == 0 {
-				return nil
-			}
-			end = s
-		}
-		l, err := tx.LRange(database.ServerActivityBucket, key, 0, end)
-		if err != nil {
-			// This error is returned when the bucket doesn't exist, which is likely on the
-			// first invocations of Wings since we haven't yet logged any data. There is nothing
-			// that needs to be done if this error occurs.
-			if errors.Is(err, nutsdb.ErrBucket) {
-				return nil
-			}
-			return errors.WithStackIf(err)
-		}
-		list = l
-		return nil
-	})
-
-	if err != nil || len(list) == 0 {
-		return errors.WithStackIf(err)
-	}
-
-	var processed []json.RawMessage
-	for _, l := range list {
-		var v json.RawMessage
-		if err := json.Unmarshal(l, &v); err != nil {
-			log.WithField("error", errors.WithStack(err)).Warn("failed to parse activity event json, skipping entry")
-			continue
-		}
-		processed = append(processed, v)
-	}
-
-	if err := m.Client().SendActivityLogs(context.Background(), processed); err != nil {
+	if err := ac.manager.Client().SendActivityLogs(context.Background(), logs); err != nil {
 		return errors.WrapIf(err, "cron: failed to send activity events to Panel")
 	}
 
-	return database.DB().Update(func(tx *nutsdb.Tx) error {
-		if m, err := tx.LSize(database.ServerActivityBucket, key); err != nil {
-			return errors.WithStack(err)
-		} else if m > len(list) {
-			// As long as there are more elements than we have in the length of our list
-			// we can just use the existing `LTrim` functionality of nutsdb. This will remove
-			// all of the values we've already pulled and sent to the API.
-			return errors.WithStack(tx.LTrim(database.ServerActivityBucket, key, len(list), -1))
-		} else {
-			i := 0
-			// This is the only way I can figure out to actually empty the items out of the list
-			// because you cannot use `LTrim` (or I cannot for the life of me figure out how) to
-			// trim the slice down to 0 items without it triggering an internal logic error. Perhaps
-			// in a future release they'll have a function to do this (based on my skimming of issues
-			// on GitHub that I cannot read due to translation barriers).
-			for {
-				if i >= m {
-					break
-				}
-				if _, err := tx.LPop(database.ServerActivityBucket, key); err != nil {
-					return errors.WithStack(err)
-				}
-				i++
-			}
+	if tx, err := sqlite.Instance().Begin(); err != nil {
+		return err
+	} else {
+		t := make([]string, len(ids))
+		params := make([]interface{}, len(ids))
+		for i := 0; i < len(ids); i++ {
+			t[i] = "?"
+			params[i] = ids[i]
 		}
-		return nil
-	})
+		q := strings.Join(t, ",")
+		_, err := tx.Exec(`DELETE FROM activity_logs WHERE id IN(`+q+`)`, params...)
+		if err != nil {
+			return errors.Combine(errors.WithStack(err), tx.Rollback())
+		}
+		return errors.WithStack(tx.Commit())
+	}
 }