-
-
Notifications
You must be signed in to change notification settings - Fork 319
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Switch to SQLite for activity tracking
- Loading branch information
1 parent
e1e7916
commit 7bd11c1
Showing
16 changed files
with
264 additions
and
740 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,102 +1,102 @@ | ||
| package cron | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "database/sql" | ||
| "emperror.dev/errors" | ||
| "github.com/apex/log" | ||
| "github.com/goccy/go-json" | ||
| "github.com/pterodactyl/wings/internal/database" | ||
| "encoding/gob" | ||
| "github.com/pterodactyl/wings/internal/sqlite" | ||
| "github.com/pterodactyl/wings/server" | ||
| "github.com/pterodactyl/wings/system" | ||
| "github.com/xujiajun/nutsdb" | ||
| "strings" | ||
| ) | ||
|
|
||
| var key = []byte("events") | ||
| var activityCron system.AtomicBool | ||
| type activityCron struct { | ||
| mu *system.AtomicBool | ||
| manager *server.Manager | ||
| max int64 | ||
| } | ||
|
|
||
| const queryRegularActivity = ` | ||
| SELECT id, event, user_uuid, server_uuid, metadata, ip, timestamp FROM activity_logs | ||
| WHERE event NOT LIKE 'server:sftp.%' | ||
| ORDER BY timestamp | ||
| LIMIT ? | ||
| ` | ||
|
|
||
| type QueriedActivity struct { | ||
| id int | ||
| b []byte | ||
| server.Activity | ||
| } | ||
|
|
||
| func processActivityLogs(m *server.Manager, c int64) error { | ||
| // Parse parses the internal query results into the QueriedActivity type and then properly | ||
| // sets the Metadata onto it. This also sets the ID that was returned to ensure we're able | ||
| // to then delete all of the matching rows in the database after we're done. | ||
| func (qa *QueriedActivity) Parse(r *sql.Rows) error { | ||
| if err := r.Scan(&qa.id, &qa.Event, &qa.User, &qa.Server, &qa.b, &qa.IP, &qa.Timestamp); err != nil { | ||
| return errors.Wrap(err, "cron: failed to parse activity log") | ||
| } | ||
| if err := gob.NewDecoder(bytes.NewBuffer(qa.b)).Decode(&qa.Metadata); err != nil { | ||
| return errors.WithStack(err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Run executes the cronjob and ensures we fetch and send all of the stored activity to the | ||
| // Panel instance. Once activity is sent it is deleted from the local database instance. Any | ||
| // SFTP specific events are not handled in this cron, they're handled seperately to account | ||
| // for de-duplication and event merging. | ||
| func (ac *activityCron) Run(ctx context.Context) error { | ||
| // Don't execute this cron if there is currently one running. Once this task is completed | ||
| // go ahead and mark it as no longer running. | ||
| if !activityCron.SwapIf(true) { | ||
| log.WithField("subsystem", "cron").WithField("cron", "activity_logs").Warn("cron: process overlap detected, skipping this run") | ||
| return nil | ||
| if !ac.mu.SwapIf(true) { | ||
| return errors.WithStack(ErrCronRunning) | ||
| } | ||
| defer activityCron.Store(false) | ||
| defer ac.mu.Store(false) | ||
|
|
||
| var list [][]byte | ||
| err := database.DB().View(func(tx *nutsdb.Tx) error { | ||
| // Grab the oldest 100 activity events that have been logged and send them back to the | ||
| // Panel for processing. Once completed, delete those events from the database and then | ||
| // release the lock on this process. | ||
| end := int(c) | ||
| if s, err := tx.LSize(database.ServerActivityBucket, key); err != nil { | ||
| if errors.Is(err, nutsdb.ErrBucket) { | ||
| return nil | ||
| } | ||
| return errors.WithStackIf(err) | ||
| } else if s < end || s == 0 { | ||
| if s == 0 { | ||
| return nil | ||
| } | ||
| end = s | ||
| } | ||
| l, err := tx.LRange(database.ServerActivityBucket, key, 0, end) | ||
| if err != nil { | ||
| // This error is returned when the bucket doesn't exist, which is likely on the | ||
| // first invocations of Wings since we haven't yet logged any data. There is nothing | ||
| // that needs to be done if this error occurs. | ||
| if errors.Is(err, nutsdb.ErrBucket) { | ||
| return nil | ||
| } | ||
| return errors.WithStackIf(err) | ||
| } | ||
| list = l | ||
| return nil | ||
| }) | ||
|
|
||
| if err != nil || len(list) == 0 { | ||
| return errors.WithStackIf(err) | ||
| rows, err := sqlite.Instance().QueryContext(ctx, queryRegularActivity, ac.max) | ||
| if err != nil { | ||
| return errors.Wrap(err, "cron: failed to query activity logs") | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| var processed []json.RawMessage | ||
| for _, l := range list { | ||
| var v json.RawMessage | ||
| if err := json.Unmarshal(l, &v); err != nil { | ||
| log.WithField("error", errors.WithStack(err)).Warn("failed to parse activity event json, skipping entry") | ||
| continue | ||
| var logs []server.Activity | ||
| var ids []int | ||
| for rows.Next() { | ||
| var qa QueriedActivity | ||
| if err := qa.Parse(rows); err != nil { | ||
| return err | ||
| } | ||
| processed = append(processed, v) | ||
| ids = append(ids, qa.id) | ||
| logs = append(logs, qa.Activity) | ||
| } | ||
|
|
||
| if err := m.Client().SendActivityLogs(context.Background(), processed); err != nil { | ||
| if err := rows.Err(); err != nil { | ||
| return errors.WithStack(err) | ||
| } | ||
| if len(logs) == 0 { | ||
| return nil | ||
| } | ||
| if err := ac.manager.Client().SendActivityLogs(context.Background(), logs); err != nil { | ||
| return errors.WrapIf(err, "cron: failed to send activity events to Panel") | ||
| } | ||
|
|
||
| return database.DB().Update(func(tx *nutsdb.Tx) error { | ||
| if m, err := tx.LSize(database.ServerActivityBucket, key); err != nil { | ||
| return errors.WithStack(err) | ||
| } else if m > len(list) { | ||
| // As long as there are more elements than we have in the length of our list | ||
| // we can just use the existing `LTrim` functionality of nutsdb. This will remove | ||
| // all of the values we've already pulled and sent to the API. | ||
| return errors.WithStack(tx.LTrim(database.ServerActivityBucket, key, len(list), -1)) | ||
| } else { | ||
| i := 0 | ||
| // This is the only way I can figure out to actually empty the items out of the list | ||
| // because you cannot use `LTrim` (or I cannot for the life of me figure out how) to | ||
| // trim the slice down to 0 items without it triggering an internal logic error. Perhaps | ||
| // in a future release they'll have a function to do this (based on my skimming of issues | ||
| // on GitHub that I cannot read due to translation barriers). | ||
| for { | ||
| if i >= m { | ||
| break | ||
| } | ||
| if _, err := tx.LPop(database.ServerActivityBucket, key); err != nil { | ||
| return errors.WithStack(err) | ||
| } | ||
| i++ | ||
| } | ||
| if tx, err := sqlite.Instance().Begin(); err != nil { | ||
| return err | ||
| } else { | ||
| t := make([]string, len(ids)) | ||
| params := make([]interface{}, len(ids)) | ||
| for i := 0; i < len(ids); i++ { | ||
| t[i] = "?" | ||
| params[i] = ids[i] | ||
| } | ||
| return nil | ||
| }) | ||
| q := strings.Join(t, ",") | ||
| _, err := tx.Exec(`DELETE FROM activity_logs WHERE id IN(`+q+`)`, params...) | ||
| if err != nil { | ||
| return errors.Combine(errors.WithStack(err), tx.Rollback()) | ||
| } | ||
| return errors.WithStack(tx.Commit()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.