From ee5ea87e836f1dc95813b6d2f3481fe0e16962bf Mon Sep 17 00:00:00 2001 From: Max Sandholm Date: Fri, 14 Apr 2023 18:48:35 +0300 Subject: [PATCH 1/3] Forward fill missing messages on startup --- database/message.go | 5 +++++ portal.go | 40 ++++++++++++++++++++++++++++++++++++++++ user.go | 2 ++ 3 files changed, 47 insertions(+) diff --git a/database/message.go b/database/message.go index e120c2b..335fbb2 100644 --- a/database/message.go +++ b/database/message.go @@ -70,6 +70,11 @@ func (mq *MessageQuery) GetLastInThread(key PortalKey, threadID string) *Message return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver, threadID)) } +func (mq *MessageQuery) GetLast(key PortalKey) *Message { + query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dc_edit_index=0 ORDER BY timestamp DESC LIMIT 1" + return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver)) +} + func (mq *MessageQuery) DeleteAll(key PortalKey) { query := "DELETE FROM message WHERE dc_chan_id=$1 AND dc_chan_receiver=$2" _, err := mq.db.Exec(query, key.ChannelID, key.Receiver) diff --git a/portal.go b/portal.go index 0f6ac84..8838a5b 100644 --- a/portal.go +++ b/portal.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "reflect" + "sort" "strconv" "strings" "sync" @@ -1966,3 +1967,42 @@ func (portal *Portal) UpdateInfo(source *User, meta *discordgo.Channel) *discord } return meta } + +func (portal *Portal) ForwardBackfill(source *User) error { + portal.log.Debugln("Checking for missing messages to fill") + lastMessage := portal.bridge.DB.Message.GetLast(portal.Key) + if lastMessage == nil { + portal.log.Debugln("No last message in portal, can't forward backfill") + return nil + } + + // Get up to 100 messages at a time until everything is fetched + for { + messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, 100, "", lastMessage.DiscordID, "") + if err != nil { + portal.log.Debugln("Error getting messages to forward backfill", err) + return err + } + // Discord seems to return messages in reverse order, + // but no specific order is guaranteed by their API docs? + sort.Slice(messages, func(i, j int) bool { + return messages[i].Timestamp.Before(messages[j].Timestamp) + }) + + for _, msg := range messages { + portal.handleDiscordMessageCreate(source, msg, nil) + } + + if len(messages) < 100 { + // Assume that was all the missing messages + break + } + lastMessage = portal.bridge.DB.Message.GetLast(portal.Key) + if lastMessage == nil { + portal.log.Debugln("No last message in portal, can't forward backfill") + return nil + } + } + + return nil +} diff --git a/user.go b/user.go index 6b300cb..a1863a7 100644 --- a/user.go +++ b/user.go @@ -728,6 +728,7 @@ func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel, } } else { portal.UpdateInfo(user, meta) + portal.ForwardBackfill(user) } user.MarkInPortal(database.UserPortal{ DiscordID: portal.Key.ChannelID, @@ -842,6 +843,7 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp } } else { portal.UpdateInfo(user, ch) + portal.ForwardBackfill(user) } } } From ef1142c61456230249ffdf544e1c01ad8961dc5d Mon Sep 17 00:00:00 2001 From: Max Sandholm Date: Fri, 14 Apr 2023 18:59:17 +0300 Subject: [PATCH 2/3] Get 50 instead of 100 messages at a time --- portal.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/portal.go b/portal.go index 8838a5b..58be0cf 100644 --- a/portal.go +++ b/portal.go @@ -1976,9 +1976,9 @@ func (portal *Portal) ForwardBackfill(source *User) error { return nil } - // Get up to 100 messages at a time until everything is fetched + // Get up to 50 messages at a time until everything is fetched for { - messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, 100, "", lastMessage.DiscordID, "") + messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, 50, "", lastMessage.DiscordID, "") if err != nil { portal.log.Debugln("Error getting messages to forward backfill", err) return err From 693fe49a9ad93a34a67718db60318046bb4f89ed Mon Sep 17 00:00:00 2001 From: Max Sandholm Date: Fri, 14 Apr 2023 23:09:59 +0300 Subject: [PATCH 3/3] Check last message ID before attempting backfill --- portal.go | 22 ++++++++++++++++------ user.go | 4 ++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/portal.go b/portal.go index 58be0cf..7a2a85b 100644 --- a/portal.go +++ b/portal.go @@ -1968,11 +1968,24 @@ func (portal *Portal) UpdateInfo(source *User, meta *discordgo.Channel) *discord return meta } -func (portal *Portal) ForwardBackfill(source *User) error { +func (portal *Portal) ForwardBackfill(source *User, meta *discordgo.Channel) error { portal.log.Debugln("Checking for missing messages to fill") lastMessage := portal.bridge.DB.Message.GetLast(portal.Key) if lastMessage == nil { - portal.log.Debugln("No last message in portal, can't forward backfill") + return nil + } + + metaLastMessageID, err := strconv.ParseInt(meta.LastMessageID, 10, 0) + if err != nil { + portal.log.Errorfln("Last message ID %s isn't integer", meta.LastMessageID) + return err + } + dbLastMessageID, err := strconv.ParseInt(lastMessage.DiscordID, 10, 0) + if err != nil { + portal.log.Errorfln("Last message ID %s isn't integer", lastMessage.DiscordID) + return err + } + if metaLastMessageID <= dbLastMessageID { return nil } @@ -1995,14 +2008,11 @@ func (portal *Portal) ForwardBackfill(source *User) error { if len(messages) < 100 { // Assume that was all the missing messages - break + return nil } lastMessage = portal.bridge.DB.Message.GetLast(portal.Key) if lastMessage == nil { - portal.log.Debugln("No last message in portal, can't forward backfill") return nil } } - - return nil } diff --git a/user.go b/user.go index a1863a7..09fd639 100644 --- a/user.go +++ b/user.go @@ -728,7 +728,7 @@ func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel, } } else { portal.UpdateInfo(user, meta) - portal.ForwardBackfill(user) + portal.ForwardBackfill(user, meta) } user.MarkInPortal(database.UserPortal{ DiscordID: portal.Key.ChannelID, @@ -843,7 +843,7 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp } } else { portal.UpdateInfo(user, ch) - portal.ForwardBackfill(user) + portal.ForwardBackfill(user, ch) } } }