Bump version to v0.4.0

Fixes #87

CHANGELOG.md

@@ -1,3 +1,30 @@
+# v0.4.0 (2023-05-16)
+
+* Added bridging of friend nicks into DM room names.
+* Added option to bypass homeserver for Discord media.
+  See [docs](https://docs.mau.fi/bridges/go/discord/direct-media.html) for more info.
+* Added conversion of replies to embeds when sending messages via webhook.
+* Added option to disable caching reuploaded media. This may be necessary when
+  using a media repo that doesn't create a unique mxc URI for each upload.
+* Improved formatting of error messages returned by Discord.
+* Enabled discordgo info logs by default.
+* Fixed limited backfill always stopping after 50 messages
+  (thanks to [@odrling] in [#81]).
+* Fixed startup sync to sync most recent private channels first.
+* Fixed syncing group DM participants when they change.
+* Possibly fixed inviting to portal rooms when multiple Matrix users use the
+  bridge.
+
+[@odrling]: https://github.com/odrling
+[#81]: https://github.com/mautrix/discord/pull/81
+
+# v0.3.0 (2023-04-16)
+
+* Added support for backfilling on room creation and missed messages on startup.
+* Added options to automatically ratchet/delete megolm sessions to minimize
+  access to old messages.
+* Added basic support for incoming voice messages.
+
 # v0.2.0 (2023-03-16)
 
 * Switched to zerolog for logging.

attachments.go

@@ -246,7 +246,7 @@ func (br *DiscordBridge) convertLottie(data []byte) ([]byte, string, error) {
 }
 
 func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, url string, encrypt bool, meta AttachmentMeta) (returnDBFile *database.File, returnErr error) {
-	isCacheable := !encrypt
+	isCacheable := br.Config.Bridge.CacheMedia != "never" && (br.Config.Bridge.CacheMedia == "always" || !encrypt)
 	returnDBFile = br.DB.File.Get(url, encrypt)
 	if returnDBFile == nil {
 		transferKey := attachmentKey{url, encrypt}
@@ -288,13 +288,19 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur
 }
 
 func (portal *Portal) getEmojiMXCByDiscordID(emojiID, name string, animated bool) id.ContentURI {
-	var url, mimeType string
+	var url, mimeType, ext string
 	if animated {
 		url = discordgo.EndpointEmojiAnimated(emojiID)
 		mimeType = "image/gif"
+		ext = "gif"
 	} else {
 		url = discordgo.EndpointEmoji(emojiID)
 		mimeType = "image/png"
+		ext = "png"
+	}
+	mxc := portal.bridge.Config.Bridge.MediaPatterns.Emoji(emojiID, ext)
+	if !mxc.IsEmpty() {
+		return mxc
 	}
 	dbFile, err := portal.bridge.copyAttachmentToMatrix(portal.MainIntent(), url, false, AttachmentMeta{
 		AttachmentID: emojiID,
@@ -302,7 +308,7 @@ func (portal *Portal) getEmojiMXCByDiscordID(emojiID, name string, animated bool
 		EmojiName:    name,
 	})
 	if err != nil {
-		portal.log.Warnfln("Failed to download emoji %s from discord: %v", emojiID, err)
+		portal.log.Warn().Err(err).Str("emoji_id", emojiID).Msg("Failed to copy emoji to Matrix")
 		return id.ContentURI{}
 	}
 	return dbFile.MXC

backfill.go

@@ -0,0 +1,315 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"sort"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/rs/zerolog"
+	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/bridge/bridgeconfig"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+
+	"go.mau.fi/mautrix-discord/database"
+)
+
+func (portal *Portal) forwardBackfillInitial(source *User) {
+	defer portal.forwardBackfillLock.Unlock()
+	// This should only be called from CreateMatrixRoom which locks forwardBackfillLock before creating the room.
+	if portal.forwardBackfillLock.TryLock() {
+		panic("forwardBackfillInitial() called without locking forwardBackfillLock")
+	}
+
+	limit := portal.bridge.Config.Bridge.Backfill.Limits.Initial.Channel
+	if portal.GuildID == "" {
+		limit = portal.bridge.Config.Bridge.Backfill.Limits.Initial.DM
+	}
+	if limit == 0 {
+		return
+	}
+
+	log := portal.log.With().
+		Str("action", "initial backfill").
+		Str("room_id", portal.MXID.String()).
+		Int("limit", limit).
+		Logger()
+
+	portal.backfillLimited(log, source, limit, "")
+}
+
+func (portal *Portal) ForwardBackfillMissed(source *User, meta *discordgo.Channel) {
+	if portal.MXID == "" {
+		return
+	}
+
+	limit := portal.bridge.Config.Bridge.Backfill.Limits.Missed.Channel
+	if portal.GuildID == "" {
+		limit = portal.bridge.Config.Bridge.Backfill.Limits.Missed.DM
+	}
+	if limit == 0 {
+		return
+	}
+	log := portal.log.With().
+		Str("action", "missed event backfill").
+		Str("room_id", portal.MXID.String()).
+		Int("limit", limit).
+		Logger()
+
+	portal.forwardBackfillLock.Lock()
+	defer portal.forwardBackfillLock.Unlock()
+
+	lastMessage := portal.bridge.DB.Message.GetLast(portal.Key)
+	if lastMessage == nil || meta.LastMessageID == "" {
+		log.Debug().Msg("Not backfilling, no last message in database or no last message in metadata")
+		return
+	} else if !shouldBackfill(lastMessage.DiscordID, meta.LastMessageID) {
+		log.Debug().
+			Str("last_bridged_message", lastMessage.DiscordID).
+			Str("last_server_message", meta.LastMessageID).
+			Msg("Not backfilling, last message in database is newer than last message in metadata")
+		return
+	}
+	log.Debug().
+		Str("last_bridged_message", lastMessage.DiscordID).
+		Str("last_server_message", meta.LastMessageID).
+		Msg("Backfilling missed messages")
+	if limit < 0 {
+		portal.backfillUnlimitedMissed(log, source, lastMessage.DiscordID)
+	} else {
+		portal.backfillLimited(log, source, limit, lastMessage.DiscordID)
+	}
+}
+
+const messageFetchChunkSize = 50
+
+func (portal *Portal) collectBackfillMessages(log zerolog.Logger, source *User, limit int, until string) ([]*discordgo.Message, bool, error) {
+	var messages []*discordgo.Message
+	var before string
+	var foundAll bool
+	for {
+		log.Debug().Str("before_id", before).Msg("Fetching messages for backfill")
+		newMessages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, before, "", "")
+		if err != nil {
+			return nil, false, err
+		}
+		if until != "" {
+			for i, msg := range newMessages {
+				if compareMessageIDs(msg.ID, until) <= 0 {
+					log.Debug().
+						Str("message_id", msg.ID).
+						Str("until_id", until).
+						Msg("Found message that was already bridged")
+					newMessages = newMessages[:i]
+					foundAll = true
+					break
+				}
+			}
+		}
+		messages = append(messages, newMessages...)
+		log.Debug().Int("count", len(newMessages)).Msg("Added messages to backfill collection")
+		if len(newMessages) < messageFetchChunkSize || len(messages) >= limit {
+			break
+		}
+		before = newMessages[len(newMessages)-1].ID
+	}
+	if len(messages) > limit {
+		foundAll = false
+		messages = messages[:limit]
+	}
+	return messages, foundAll, nil
+}
+
+func (portal *Portal) backfillLimited(log zerolog.Logger, source *User, limit int, after string) {
+	messages, foundAll, err := portal.collectBackfillMessages(log, source, limit, after)
+	if err != nil {
+		log.Err(err).Msg("Error collecting messages to forward backfill")
+		return
+	}
+	log.Info().
+		Int("count", len(messages)).
+		Bool("found_all", foundAll).
+		Msg("Collected messages to backfill")
+	sort.Sort(MessageSlice(messages))
+	if !foundAll {
+		_, err = portal.sendMatrixMessage(portal.MainIntent(), event.EventMessage, &event.MessageEventContent{
+			MsgType: event.MsgNotice,
+			Body:    "Some messages may have been missed here while the bridge was offline.",
+		}, nil, 0)
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to send missed message warning")
+		} else {
+			log.Debug().Msg("Sent warning about possibly missed messages")
+		}
+	}
+	portal.sendBackfillBatch(log, source, messages)
+}
+
+func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User, after string) {
+	for {
+		log.Debug().Str("after_id", after).Msg("Fetching chunk of messages to backfill")
+		messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, "", after, "")
+		if err != nil {
+			log.Err(err).Msg("Error fetching chunk of messages to forward backfill")
+			return
+		}
+		log.Debug().Int("count", len(messages)).Msg("Fetched chunk of messages to backfill")
+		sort.Sort(MessageSlice(messages))
+
+		portal.sendBackfillBatch(log, source, messages)
+
+		if len(messages) < messageFetchChunkSize {
+			// Assume that was all the missing messages
+			log.Debug().Msg("Chunk had less than 50 messages, stopping backfill")
+			return
+		}
+		after = messages[len(messages)-1].ID
+	}
+}
+
+func (portal *Portal) sendBackfillBatch(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+	if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry {
+		log.Debug().Msg("Using hungryserv, sending messages with batch send endpoint")
+		portal.forwardBatchSend(log, source, messages)
+	} else {
+		log.Debug().Msg("Not using hungryserv, sending messages one by one")
+		for _, msg := range messages {
+			portal.handleDiscordMessageCreate(source, msg, nil)
+		}
+	}
+}
+
+func (portal *Portal) forwardBatchSend(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+	evts, dbMessages := portal.convertMessageBatch(log, source, messages)
+	if len(evts) == 0 {
+		log.Warn().Msg("Didn't get any events to backfill")
+		return
+	}
+	log.Info().Int("events", len(evts)).Msg("Converted messages to backfill")
+	resp, err := portal.MainIntent().BatchSend(portal.MXID, &mautrix.ReqBatchSend{
+		BeeperNewMessages: true,
+		Events:            evts,
+	})
+	if err != nil {
+		log.Err(err).Msg("Error sending backfill batch")
+		return
+	}
+	for i, evtID := range resp.EventIDs {
+		dbMessages[i].MXID = evtID
+	}
+	portal.bridge.DB.Message.MassInsert(portal.Key, dbMessages)
+	log.Info().Msg("Inserted backfilled batch to database")
+}
+
+func (portal *Portal) convertMessageBatch(log zerolog.Logger, source *User, messages []*discordgo.Message) ([]*event.Event, []database.Message) {
+	evts := make([]*event.Event, 0, len(messages))
+	dbMessages := make([]database.Message, 0, len(messages))
+	ctx := context.Background()
+	for _, msg := range messages {
+		for _, mention := range msg.Mentions {
+			puppet := portal.bridge.GetPuppetByID(mention.ID)
+			puppet.UpdateInfo(nil, mention)
+		}
+
+		puppet := portal.bridge.GetPuppetByID(msg.Author.ID)
+		puppet.UpdateInfo(source, msg.Author)
+		intent := puppet.IntentFor(portal)
+		replyTo := portal.getReplyTarget(source, "", msg.MessageReference, msg.Embeds, true)
+
+		ts, _ := discordgo.SnowflakeTimestamp(msg.ID)
+		log := log.With().
+			Str("message_id", msg.ID).
+			Int("message_type", int(msg.Type)).
+			Str("author_id", msg.Author.ID).
+			Logger()
+		parts := portal.convertDiscordMessage(log.WithContext(ctx), intent, msg)
+		for i, part := range parts {
+			if replyTo != nil {
+				part.Content.RelatesTo = &event.RelatesTo{InReplyTo: replyTo}
+				// Only set reply for first event
+				replyTo = nil
+			}
+			partName := part.AttachmentID
+			// Always use blank part name for first part so that replies and other things
+			// can reference it without knowing about attachments.
+			if i == 0 {
+				partName = ""
+			}
+			evt := &event.Event{
+				ID:        portal.deterministicEventID(msg.ID, partName),
+				Type:      part.Type,
+				Sender:    intent.UserID,
+				Timestamp: ts.UnixMilli(),
+				Content: event.Content{
+					Parsed: part.Content,
+					Raw:    part.Extra,
+				},
+			}
+			var err error
+			evt.Type, err = portal.encrypt(intent, &evt.Content, evt.Type)
+			if err != nil {
+				log.Err(err).Msg("Failed to encrypt event")
+				continue
+			}
+			intent.AddDoublePuppetValue(&evt.Content)
+			evts = append(evts, evt)
+			dbMessages = append(dbMessages, database.Message{
+				Channel:      portal.Key,
+				DiscordID:    msg.ID,
+				SenderID:     msg.Author.ID,
+				Timestamp:    ts,
+				AttachmentID: part.AttachmentID,
+			})
+		}
+	}
+	return evts, dbMessages
+}
+
+func (portal *Portal) deterministicEventID(messageID, partName string) id.EventID {
+	data := fmt.Sprintf("%s/discord/%s/%s", portal.MXID, messageID, partName)
+	sum := sha256.Sum256([]byte(data))
+	return id.EventID(fmt.Sprintf("$%s:discord.com", base64.RawURLEncoding.EncodeToString(sum[:])))
+}
+
+// compareMessageIDs compares two Discord message IDs.
+//
+// If the first ID is lower, -1 is returned.
+// If the second ID is lower, 1 is returned.
+// If the IDs are equal, 0 is returned.
+func compareMessageIDs(id1, id2 string) int {
+	if id1 == id2 {
+		return 0
+	}
+	if len(id1) < len(id2) {
+		return -1
+	} else if len(id2) < len(id1) {
+		return 1
+	}
+	if id1 < id2 {
+		return -1
+	}
+	return 1
+}
+
+func shouldBackfill(latestBridgedIDStr, latestIDFromServerStr string) bool {
+	return compareMessageIDs(latestBridgedIDStr, latestIDFromServerStr) == -1
+}
+
+type MessageSlice []*discordgo.Message
+
+var _ sort.Interface = (MessageSlice)(nil)
+
+func (a MessageSlice) Len() int {
+	return len(a)
+}
+
+func (a MessageSlice) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a MessageSlice) Less(i, j int) bool {
+	return compareMessageIDs(a[i].ID, a[j].ID) == -1
+}

commands.go

@@ -371,10 +371,10 @@ func fnRejoinSpace(ce *WrappedCommandEvent) {
 	}
 	user := ce.User
 	if ce.Args[0] == "main" {
-		user.ensureInvited(nil, user.GetSpaceRoom(), false)
+		user.ensureInvited(nil, user.GetSpaceRoom(), false, true)
 		ce.Reply("Invited you to your main space ([link](%s))", user.GetSpaceRoom().URI(ce.Bridge.AS.HomeserverDomain).MatrixToURL())
 	} else if ce.Args[0] == "dms" {
-		user.ensureInvited(nil, user.GetDMSpaceRoom(), false)
+		user.ensureInvited(nil, user.GetDMSpaceRoom(), false, true)
 		ce.Reply("Invited you to your DM space ([link](%s))", user.GetDMSpaceRoom().URI(ce.Bridge.AS.HomeserverDomain).MatrixToURL())
 	} else if _, err := strconv.Atoi(ce.Args[0]); err == nil {
 		ce.Reply("Rejoining guild spaces is not yet implemented")

config/bridge.go

@@ -25,6 +25,7 @@ import (
 	"github.com/bwmarrin/discordgo"
 
 	"maunium.net/go/mautrix/bridge/bridgeconfig"
+	"maunium.net/go/mautrix/id"
 )
 
 type BridgeConfig struct {
@@ -32,7 +33,7 @@ type BridgeConfig struct {
 	DisplaynameTemplate       string `yaml:"displayname_template"`
 	ChannelNameTemplate       string `yaml:"channel_name_template"`
 	GuildNameTemplate         string `yaml:"guild_name_template"`
-	PrivateChatPortalMeta     bool   `yaml:"private_chat_portal_meta"`
+	PrivateChatPortalMeta     string `yaml:"private_chat_portal_meta"`
 	PrivateChannelCreateLimit int    `yaml:"startup_private_channel_create_limit"`
 
 	PortalMessageBuffer int `yaml:"portal_message_buffer"`
@@ -50,7 +51,12 @@ type BridgeConfig struct {
 	DeletePortalOnChannelDelete bool `yaml:"delete_portal_on_channel_delete"`
 	DeleteGuildOnLeave          bool `yaml:"delete_guild_on_leave"`
 	FederateRooms               bool `yaml:"federate_rooms"`
-	AnimatedSticker             struct {
+	UseDiscordCDNUpload         bool `yaml:"use_discord_cdn_upload"`
+
+	CacheMedia    string        `yaml:"cache_media"`
+	MediaPatterns MediaPatterns `yaml:"media_patterns"`
+
+	AnimatedSticker struct {
 		Target string `yaml:"target"`
 		Args   struct {
 			Width  int `yaml:"width"`
@@ -66,6 +72,14 @@ type BridgeConfig struct {
 	CommandPrefix      string                           `yaml:"command_prefix"`
 	ManagementRoomText bridgeconfig.ManagementRoomTexts `yaml:"management_room_text"`
 
+	Backfill struct {
+		Limits struct {
+			Initial BackfillLimitPart `yaml:"initial"`
+			Missed  BackfillLimitPart `yaml:"missed"`
+		} `yaml:"forward_limits"`
+		MaxGuildMembers int `yaml:"max_guild_members"`
+	} `yaml:"backfill"`
+
 	Encryption bridgeconfig.EncryptionConfig `yaml:"encryption"`
 
 	Provisioning struct {
@@ -81,6 +95,118 @@ type BridgeConfig struct {
 	guildNameTemplate   *template.Template `yaml:"-"`
 }
 
+type MediaPatterns struct {
+	Enabled        bool   `yaml:"enabled"`
+	TplAttachments string `yaml:"attachments"`
+	TplEmojis      string `yaml:"emojis"`
+	TplStickers    string `yaml:"stickers"`
+	TplAvatars     string `yaml:"avatars"`
+
+	attachments *template.Template `yaml:"-"`
+	emojis      *template.Template `yaml:"-"`
+	stickers    *template.Template `yaml:"-"`
+	avatars     *template.Template `yaml:"-"`
+}
+
+type umMediaPatterns MediaPatterns
+
+func (mp *MediaPatterns) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	err := unmarshal((*umMediaPatterns)(mp))
+	if err != nil {
+		return err
+	}
+	tpl := template.New("media_patterns")
+
+	pairs := []struct {
+		ptr      **template.Template
+		name     string
+		template string
+	}{
+		{&mp.attachments, "attachments", mp.TplAttachments},
+		{&mp.emojis, "emojis", mp.TplEmojis},
+		{&mp.stickers, "stickers", mp.TplStickers},
+		{&mp.avatars, "avatars", mp.TplAvatars},
+	}
+	for _, pair := range pairs {
+		if pair.template == "" {
+			continue
+		}
+		*pair.ptr, err = tpl.New(pair.name).Parse(pair.template)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type attachmentParams struct {
+	ChannelID    string
+	AttachmentID string
+	FileName     string
+}
+
+type emojiStickerParams struct {
+	ID  string
+	Ext string
+}
+
+type avatarParams struct {
+	UserID   string
+	AvatarID string
+	Ext      string
+}
+
+func (mp *MediaPatterns) execute(tpl *template.Template, params any) id.ContentURI {
+	if tpl == nil || !mp.Enabled {
+		return id.ContentURI{}
+	}
+	var out strings.Builder
+	err := tpl.Execute(&out, params)
+	if err != nil {
+		panic(err)
+	}
+	uri, err := id.ParseContentURI(out.String())
+	if err != nil {
+		panic(err)
+	}
+	return uri
+}
+
+func (mp *MediaPatterns) Attachment(channelID, attachmentID, filename string) id.ContentURI {
+	return mp.execute(mp.attachments, attachmentParams{
+		ChannelID:    channelID,
+		AttachmentID: attachmentID,
+		FileName:     filename,
+	})
+}
+
+func (mp *MediaPatterns) Emoji(emojiID, ext string) id.ContentURI {
+	return mp.execute(mp.emojis, emojiStickerParams{
+		ID:  emojiID,
+		Ext: ext,
+	})
+}
+
+func (mp *MediaPatterns) Sticker(stickerID, ext string) id.ContentURI {
+	return mp.execute(mp.stickers, emojiStickerParams{
+		ID:  stickerID,
+		Ext: ext,
+	})
+}
+
+func (mp *MediaPatterns) Avatar(userID, avatarID, ext string) id.ContentURI {
+	return mp.execute(mp.avatars, avatarParams{
+		UserID:   userID,
+		AvatarID: avatarID,
+		Ext:      ext,
+	})
+}
+
+type BackfillLimitPart struct {
+	DM      int `yaml:"dm"`
+	Channel int `yaml:"channel"`
+}
+
 func (bc *BridgeConfig) GetResendBridgeInfo() bool {
 	return bc.ResendBridgeInfo
 }

config/upgrade.go

@@ -1,5 +1,5 @@
 // mautrix-discord - A Matrix-Discord puppeting bridge.
-// Copyright (C) 2022 Tulir Asokan
+// Copyright (C) 2023 Tulir Asokan
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -31,7 +31,15 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Str, "bridge", "displayname_template")
 	helper.Copy(up.Str, "bridge", "channel_name_template")
 	helper.Copy(up.Str, "bridge", "guild_name_template")
-	helper.Copy(up.Bool, "bridge", "private_chat_portal_meta")
+	if legacyPrivateChatPortalMeta, ok := helper.Get(up.Bool, "bridge", "private_chat_portal_meta"); ok {
+		updatedPrivateChatPortalMeta := "default"
+		if legacyPrivateChatPortalMeta == "true" {
+			updatedPrivateChatPortalMeta = "always"
+		}
+		helper.Set(up.Str, updatedPrivateChatPortalMeta, "bridge", "private_chat_portal_meta")
+	} else {
+		helper.Copy(up.Str, "bridge", "private_chat_portal_meta")
+	}
 	helper.Copy(up.Int, "bridge", "startup_private_channel_create_limit")
 	helper.Copy(up.Int, "bridge", "portal_message_buffer")
 	helper.Copy(up.Bool, "bridge", "delivery_receipts")
@@ -47,6 +55,13 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Bool, "bridge", "delete_portal_on_channel_delete")
 	helper.Copy(up.Bool, "bridge", "delete_guild_on_leave")
 	helper.Copy(up.Bool, "bridge", "federate_rooms")
+	helper.Copy(up.Bool, "bridge", "use_discord_cdn_upload")
+	helper.Copy(up.Bool, "bridge", "media_patterns", "enabled")
+	helper.Copy(up.Str, "bridge", "cache_media")
+	helper.Copy(up.Str|up.Null, "bridge", "media_patterns", "attachments")
+	helper.Copy(up.Str|up.Null, "bridge", "media_patterns", "emojis")
+	helper.Copy(up.Str|up.Null, "bridge", "media_patterns", "stickers")
+	helper.Copy(up.Str|up.Null, "bridge", "media_patterns", "avatars")
 	helper.Copy(up.Str, "bridge", "animated_sticker", "target")
 	helper.Copy(up.Int, "bridge", "animated_sticker", "args", "width")
 	helper.Copy(up.Int, "bridge", "animated_sticker", "args", "height")
@@ -59,11 +74,24 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Str, "bridge", "management_room_text", "welcome_connected")
 	helper.Copy(up.Str, "bridge", "management_room_text", "welcome_unconnected")
 	helper.Copy(up.Str|up.Null, "bridge", "management_room_text", "additional_help")
+	helper.Copy(up.Bool, "bridge", "backfill", "enabled")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "dm")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "dm")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "max_guild_members")
 	helper.Copy(up.Bool, "bridge", "encryption", "allow")
 	helper.Copy(up.Bool, "bridge", "encryption", "default")
 	helper.Copy(up.Bool, "bridge", "encryption", "require")
 	helper.Copy(up.Bool, "bridge", "encryption", "appservice")
 	helper.Copy(up.Bool, "bridge", "encryption", "allow_key_sharing")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_outbound_on_ack")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "dont_store_outbound")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "ratchet_on_decrypt")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_fully_used_on_decrypt")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_prev_on_new_session")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_on_device_delete")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "periodically_delete_expired")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "receive")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "send")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "share")

database/database.go

@@ -68,9 +68,10 @@ func New(baseDB *dbutil.Database, log maulogger.Logger) *Database {
 	return db
 }
 
-func strPtr(val string) *string {
+func strPtr[T ~string](val T) *string {
 	if val == "" {
 		return nil
 	}
-	return &val
+	valStr := string(val)
+	return &valStr
 }

database/file.go

@@ -79,7 +79,7 @@ func (f *File) Scan(row dbutil.Scannable) *File {
 	}
 	f.ID = fileID.String
 	f.EmojiName = emojiName.String
-	f.Timestamp = time.UnixMilli(timestamp)
+	f.Timestamp = time.UnixMilli(timestamp).UTC()
 	f.Width = int(width.Int32)
 	f.Height = int(height.Int32)
 	f.MXC, err = id.ParseContentURI(mxc)

database/message.go

@@ -19,7 +19,7 @@ type MessageQuery struct {
 }
 
 const (
-	messageSelect = "SELECT dcid, dc_attachment_id, dc_edit_index, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, dc_thread_id, mxid FROM message"
+	messageSelect = "SELECT dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, dc_edit_timestamp, dc_thread_id, mxid FROM message"
 )
 
 func (mq *MessageQuery) New() *Message {
@@ -46,17 +46,17 @@ func (mq *MessageQuery) scanAll(rows dbutil.Rows, err error) []*Message {
 }
 
 func (mq *MessageQuery) GetByDiscordID(key PortalKey, discordID string) []*Message {
-	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 AND dc_edit_index=0 ORDER BY dc_attachment_id ASC"
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 ORDER BY dc_attachment_id ASC"
 	return mq.scanAll(mq.db.Query(query, key.ChannelID, key.Receiver, discordID))
 }
 
 func (mq *MessageQuery) GetFirstByDiscordID(key PortalKey, discordID string) *Message {
-	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 AND dc_edit_index=0 ORDER BY dc_attachment_id ASC LIMIT 1"
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 ORDER BY dc_attachment_id ASC LIMIT 1"
 	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver, discordID))
 }
 
 func (mq *MessageQuery) GetLastByDiscordID(key PortalKey, discordID string) *Message {
-	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 AND dc_edit_index=0 ORDER BY dc_attachment_id DESC LIMIT 1"
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dcid=$3 ORDER BY dc_attachment_id DESC LIMIT 1"
 	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver, discordID))
 }
 
@@ -66,10 +66,15 @@ func (mq *MessageQuery) GetClosestBefore(key PortalKey, threadID string, ts time
 }
 
 func (mq *MessageQuery) GetLastInThread(key PortalKey, threadID string) *Message {
-	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dc_thread_id=$3 AND dc_edit_index=0 ORDER BY timestamp DESC, dc_attachment_id DESC LIMIT 1"
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dc_thread_id=$3 ORDER BY timestamp DESC, dc_attachment_id DESC LIMIT 1"
 	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver, threadID))
 }
 
+func (mq *MessageQuery) GetLast(key PortalKey) *Message {
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 ORDER BY timestamp DESC LIMIT 1"
+	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver))
+}
+
 func (mq *MessageQuery) DeleteAll(key PortalKey) {
 	query := "DELETE FROM message WHERE dc_chan_id=$1 AND dc_chan_receiver=$2"
 	_, err := mq.db.Exec(query, key.ChannelID, key.Receiver)
@@ -90,17 +95,47 @@ func (mq *MessageQuery) GetByMXID(key PortalKey, mxid id.EventID) *Message {
 	return mq.New().Scan(row)
 }
 
+func (mq *MessageQuery) MassInsert(key PortalKey, msgs []Message) {
+	if len(msgs) == 0 {
+		return
+	}
+	valueStringFormat := "($%d, $%d, $1, $2, $%d, $%d, $%d, $%d, $%d)"
+	if mq.db.Dialect == dbutil.SQLite {
+		valueStringFormat = strings.ReplaceAll(valueStringFormat, "$", "?")
+	}
+	params := make([]interface{}, 2+len(msgs)*7)
+	placeholders := make([]string, len(msgs))
+	params[0] = key.ChannelID
+	params[1] = key.Receiver
+	for i, msg := range msgs {
+		baseIndex := 2 + i*7
+		params[baseIndex] = msg.DiscordID
+		params[baseIndex+1] = msg.AttachmentID
+		params[baseIndex+2] = msg.SenderID
+		params[baseIndex+3] = msg.Timestamp.UnixMilli()
+		params[baseIndex+4] = msg.editTimestampVal()
+		params[baseIndex+5] = msg.ThreadID
+		params[baseIndex+6] = msg.MXID
+		placeholders[i] = fmt.Sprintf(valueStringFormat, baseIndex+1, baseIndex+2, baseIndex+3, baseIndex+4, baseIndex+5, baseIndex+6, baseIndex+7)
+	}
+	_, err := mq.db.Exec(fmt.Sprintf(messageMassInsertTemplate, strings.Join(placeholders, ", ")), params...)
+	if err != nil {
+		mq.log.Warnfln("Failed to insert %d messages: %v", len(msgs), err)
+		panic(err)
+	}
+}
+
 type Message struct {
 	db  *Database
 	log log.Logger
 
-	DiscordID    string
+	DiscordID     string
-	AttachmentID string
+	AttachmentID  string
-	EditIndex    int
+	Channel       PortalKey
-	Channel      PortalKey
+	SenderID      string
-	SenderID     string
+	Timestamp     time.Time
-	Timestamp    time.Time
+	EditTimestamp time.Time
-	ThreadID     string
+	ThreadID      string
 
 	MXID id.EventID
 }
@@ -114,9 +149,9 @@ func (m *Message) DiscordProtoChannelID() string {
 }
 
 func (m *Message) Scan(row dbutil.Scannable) *Message {
-	var ts int64
+	var ts, editTS int64
 
-	err := row.Scan(&m.DiscordID, &m.AttachmentID, &m.EditIndex, &m.Channel.ChannelID, &m.Channel.Receiver, &m.SenderID, &ts, &m.ThreadID, &m.MXID)
+	err := row.Scan(&m.DiscordID, &m.AttachmentID, &m.Channel.ChannelID, &m.Channel.Receiver, &m.SenderID, &ts, &editTS, &m.ThreadID, &m.MXID)
 	if err != nil {
 		if !errors.Is(err, sql.ErrNoRows) {
 			m.log.Errorln("Database scan failed:", err)
@@ -127,7 +162,10 @@ func (m *Message) Scan(row dbutil.Scannable) *Message {
 	}
 
 	if ts != 0 {
-		m.Timestamp = time.UnixMilli(ts)
+		m.Timestamp = time.UnixMilli(ts).UTC()
+	}
+	if editTS != 0 {
+		m.EditTimestamp = time.Unix(0, editTS).UTC()
 	}
 
 	return m
@@ -135,7 +173,7 @@ func (m *Message) Scan(row dbutil.Scannable) *Message {
 
 const messageInsertQuery = `
 	INSERT INTO message (
-		dcid, dc_attachment_id, dc_edit_index, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, dc_thread_id, mxid
+		dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, dc_edit_timestamp, dc_thread_id, mxid
 	)
 	VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
 `
@@ -147,7 +185,14 @@ type MessagePart struct {
 	MXID         id.EventID
 }
 
-func (m *Message) MassInsert(msgs []MessagePart) {
+func (m *Message) editTimestampVal() int64 {
+	if m.EditTimestamp.IsZero() {
+		return 0
+	}
+	return m.EditTimestamp.UnixNano()
+}
+
+func (m *Message) MassInsertParts(msgs []MessagePart) {
 	if len(msgs) == 0 {
 		return
 	}
@@ -158,11 +203,11 @@ func (m *Message) MassInsert(msgs []MessagePart) {
 	params := make([]interface{}, 7+len(msgs)*2)
 	placeholders := make([]string, len(msgs))
 	params[0] = m.DiscordID
-	params[1] = m.EditIndex
+	params[1] = m.Channel.ChannelID
-	params[2] = m.Channel.ChannelID
+	params[2] = m.Channel.Receiver
-	params[3] = m.Channel.Receiver
+	params[3] = m.SenderID
-	params[4] = m.SenderID
+	params[4] = m.Timestamp.UnixMilli()
-	params[5] = m.Timestamp.UnixMilli()
+	params[5] = m.editTimestampVal()
 	params[6] = m.ThreadID
 	for i, msg := range msgs {
 		params[7+i*2] = msg.AttachmentID
@@ -178,8 +223,8 @@ func (m *Message) MassInsert(msgs []MessagePart) {
 
 func (m *Message) Insert() {
 	_, err := m.db.Exec(messageInsertQuery,
-		m.DiscordID, m.AttachmentID, m.EditIndex, m.Channel.ChannelID, m.Channel.Receiver, m.SenderID,
+		m.DiscordID, m.AttachmentID, m.Channel.ChannelID, m.Channel.Receiver, m.SenderID,
-		m.Timestamp.UnixMilli(), m.ThreadID, m.MXID)
+		m.Timestamp.UnixMilli(), m.editTimestampVal(), m.ThreadID, m.MXID)
 
 	if err != nil {
 		m.log.Warnfln("Failed to insert %s@%s: %v", m.DiscordID, m.Channel, err)
@@ -187,6 +232,20 @@ func (m *Message) Insert() {
 	}
 }
 
+const editUpdateQuery = `
+	UPDATE message
+	SET dc_edit_timestamp=$1
+	WHERE dcid=$2 AND dc_attachment_id=$3 AND dc_chan_id=$4 AND dc_chan_receiver=$5 AND dc_edit_timestamp<$1
+`
+
+func (m *Message) UpdateEditTimestamp(ts time.Time) {
+	_, err := m.db.Exec(editUpdateQuery, ts.UnixNano(), m.DiscordID, m.AttachmentID, m.Channel.ChannelID, m.Channel.Receiver)
+	if err != nil {
+		m.log.Warnfln("Failed to update edit timestamp of %s@%s: %v", m.DiscordID, m.Channel, err)
+		panic(err)
+	}
+}
+
 func (m *Message) Delete() {
 	query := "DELETE FROM message WHERE dcid=$1 AND dc_chan_id=$2 AND dc_chan_receiver=$3 AND dc_attachment_id=$4"
 	_, err := m.db.Exec(query, m.DiscordID, m.Channel.ChannelID, m.Channel.Receiver, m.AttachmentID)

database/portal.go

@@ -15,7 +15,7 @@ import (
 const (
 	portalSelect = `
 		SELECT dcid, receiver, type, other_user_id, dc_guild_id, dc_parent_id, mxid,
-		       plain_name, name, name_set, topic, topic_set, avatar, avatar_url, avatar_set,
+		       plain_name, name, name_set, friend_nick, topic, topic_set, avatar, avatar_url, avatar_set,
 		       encrypted, in_space, first_event_id, relay_webhook_id, relay_webhook_secret
 		FROM portal
 	`
@@ -68,6 +68,10 @@ func (pq *PortalQuery) GetByMXID(mxid id.RoomID) *Portal {
 	return pq.get(portalSelect+" WHERE mxid=$1", mxid)
 }
 
+func (pq *PortalQuery) FindPrivateChatBetween(id, receiver string) *Portal {
+	return pq.get(portalSelect+" WHERE other_user_id=$1 AND receiver=$2 AND type=$3", id, receiver, discordgo.ChannelTypeDM)
+}
+
 func (pq *PortalQuery) FindPrivateChatsWith(id string) []*Portal {
 	return pq.getAll(portalSelect+" WHERE other_user_id=$1 AND type=$2", id, discordgo.ChannelTypeDM)
 }
@@ -109,16 +113,17 @@ type Portal struct {
 
 	MXID id.RoomID
 
-	PlainName string
+	PlainName  string
-	Name      string
+	Name       string
-	NameSet   bool
+	NameSet    bool
-	Topic     string
+	FriendNick bool
-	TopicSet  bool
+	Topic      string
-	Avatar    string
+	TopicSet   bool
-	AvatarURL id.ContentURI
+	Avatar     string
-	AvatarSet bool
+	AvatarURL  id.ContentURI
-	Encrypted bool
+	AvatarSet  bool
-	InSpace   id.RoomID
+	Encrypted  bool
+	InSpace    id.RoomID
 
 	FirstEventID id.EventID
 
@@ -132,7 +137,7 @@ func (p *Portal) Scan(row dbutil.Scannable) *Portal {
 	var avatarURL string
 
 	err := row.Scan(&p.Key.ChannelID, &p.Key.Receiver, &chanType, &otherUserID, &guildID, &parentID,
-		&mxid, &p.PlainName, &p.Name, &p.NameSet, &p.Topic, &p.TopicSet, &p.Avatar, &avatarURL, &p.AvatarSet,
+		&mxid, &p.PlainName, &p.Name, &p.NameSet, &p.FriendNick, &p.Topic, &p.TopicSet, &p.Avatar, &avatarURL, &p.AvatarSet,
 		&p.Encrypted, &p.InSpace, &firstEventID, &relayWebhookID, &relayWebhookSecret)
 
 	if err != nil {
@@ -160,13 +165,13 @@ func (p *Portal) Scan(row dbutil.Scannable) *Portal {
 func (p *Portal) Insert() {
 	query := `
 		INSERT INTO portal (dcid, receiver, type, other_user_id, dc_guild_id, dc_parent_id, mxid,
-		                    plain_name, name, name_set, topic, topic_set, avatar, avatar_url, avatar_set,
+		                    plain_name, name, name_set, friend_nick, topic, topic_set, avatar, avatar_url, avatar_set,
 		                    encrypted, in_space, first_event_id, relay_webhook_id, relay_webhook_secret)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)
 	`
 	_, err := p.db.Exec(query, p.Key.ChannelID, p.Key.Receiver, p.Type,
 		strPtr(p.OtherUserID), strPtr(p.GuildID), strPtr(p.ParentID), strPtr(string(p.MXID)),
-		p.PlainName, p.Name, p.NameSet, p.Topic, p.TopicSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet,
+		p.PlainName, p.Name, p.NameSet, p.FriendNick, p.Topic, p.TopicSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet,
 		p.Encrypted, p.InSpace, p.FirstEventID.String(), strPtr(p.RelayWebhookID), strPtr(p.RelayWebhookSecret))
 
 	if err != nil {
@@ -179,14 +184,16 @@ func (p *Portal) Update() {
 	query := `
 		UPDATE portal
 		SET type=$1, other_user_id=$2, dc_guild_id=$3, dc_parent_id=$4, mxid=$5,
-			plain_name=$6, name=$7, name_set=$8, topic=$9, topic_set=$10, avatar=$11, avatar_url=$12, avatar_set=$13,
+			plain_name=$6, name=$7, name_set=$8, friend_nick=$9, topic=$10, topic_set=$11,
-			encrypted=$14, in_space=$15, first_event_id=$16, relay_webhook_id=$17, relay_webhook_secret=$18
+			avatar=$12, avatar_url=$13, avatar_set=$14, encrypted=$15, in_space=$16, first_event_id=$17,
-		WHERE dcid=$19 AND receiver=$20
+			relay_webhook_id=$18, relay_webhook_secret=$19
+		WHERE dcid=$20 AND receiver=$21
 	`
 	_, err := p.db.Exec(query,
 		p.Type, strPtr(p.OtherUserID), strPtr(p.GuildID), strPtr(p.ParentID), strPtr(string(p.MXID)),
-		p.PlainName, p.Name, p.NameSet, p.Topic, p.TopicSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet,
+		p.PlainName, p.Name, p.NameSet, p.FriendNick, p.Topic, p.TopicSet,
-		p.Encrypted, p.InSpace, p.FirstEventID.String(), strPtr(p.RelayWebhookID), strPtr(p.RelayWebhookSecret),
+		p.Avatar, p.AvatarURL.String(), p.AvatarSet, p.Encrypted, p.InSpace, p.FirstEventID.String(),
+		strPtr(p.RelayWebhookID), strPtr(p.RelayWebhookSecret),
 		p.Key.ChannelID, p.Key.Receiver)
 
 	if err != nil {

database/puppet.go

@@ -11,7 +11,7 @@ import (
 
 const (
 	puppetSelect = "SELECT id, name, name_set, avatar, avatar_url, avatar_set," +
-		" custom_mxid, access_token, next_batch" +
+		" contact_info_set, username, discriminator, is_bot, custom_mxid, access_token, next_batch" +
 		" FROM puppet "
 )
 
@@ -73,6 +73,12 @@ type Puppet struct {
 	AvatarURL id.ContentURI
 	AvatarSet bool
 
+	ContactInfoSet bool
+
+	Username      string
+	Discriminator string
+	IsBot         bool
+
 	CustomMXID  id.UserID
 	AccessToken string
 	NextBatch   string
@@ -82,8 +88,8 @@ func (p *Puppet) Scan(row dbutil.Scannable) *Puppet {
 	var avatarURL string
 	var customMXID, accessToken, nextBatch sql.NullString
 
-	err := row.Scan(&p.ID, &p.Name, &p.NameSet, &p.Avatar, &avatarURL, &p.AvatarSet,
+	err := row.Scan(&p.ID, &p.Name, &p.NameSet, &p.Avatar, &avatarURL, &p.AvatarSet, &p.ContactInfoSet,
-		&customMXID, &accessToken, &nextBatch)
+		&p.Username, &p.Discriminator, &p.IsBot, &customMXID, &accessToken, &nextBatch)
 
 	if err != nil {
 		if err != sql.ErrNoRows {
@@ -104,11 +110,11 @@ func (p *Puppet) Scan(row dbutil.Scannable) *Puppet {
 
 func (p *Puppet) Insert() {
 	query := `
-		INSERT INTO puppet (id, name, name_set, avatar, avatar_url, avatar_set, custom_mxid, access_token, next_batch)
+		INSERT INTO puppet (id, name, name_set, avatar, avatar_url, avatar_set, contact_info_set, username, discriminator, is_bot, custom_mxid, access_token, next_batch)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 	`
-	_, err := p.db.Exec(query, p.ID, p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet,
+	_, err := p.db.Exec(query, p.ID, p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet, p.ContactInfoSet,
-		strPtr(string(p.CustomMXID)), strPtr(p.AccessToken), strPtr(p.NextBatch))
+		p.Username, p.Discriminator, p.IsBot, strPtr(p.CustomMXID), strPtr(p.AccessToken), strPtr(p.NextBatch))
 
 	if err != nil {
 		p.log.Warnfln("Failed to insert %s: %v", p.ID, err)
@@ -118,12 +124,12 @@ func (p *Puppet) Insert() {
 
 func (p *Puppet) Update() {
 	query := `
-		UPDATE puppet SET name=$1, name_set=$2, avatar=$3, avatar_url=$4, avatar_set=$5,
+		UPDATE puppet SET name=$1, name_set=$2, avatar=$3, avatar_url=$4, avatar_set=$5, contact_info_set=$6,
-		                  custom_mxid=$6, access_token=$7, next_batch=$8
+		                  username=$7, discriminator=$8, is_bot=$9, custom_mxid=$10, access_token=$11, next_batch=$12
-		WHERE id=$9
+		WHERE id=$13
 	`
-	_, err := p.db.Exec(query, p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet,
+	_, err := p.db.Exec(query, p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet, p.ContactInfoSet,
-		strPtr(string(p.CustomMXID)), strPtr(p.AccessToken), strPtr(p.NextBatch),
+		p.Username, p.Discriminator, p.IsBot, strPtr(p.CustomMXID), strPtr(p.AccessToken), strPtr(p.NextBatch),
 		p.ID)
 
 	if err != nil {

database/upgrades/00-latest-revision.sql

@@ -1,4 +1,4 @@
--- v0 -> v15: Latest revision
+-- v0 -> v19: Latest revision
 
 CREATE TABLE guild (
     dcid       TEXT PRIMARY KEY,
@@ -29,6 +29,7 @@ CREATE TABLE portal (
     plain_name TEXT NOT NULL,
     name       TEXT NOT NULL,
     name_set   BOOLEAN NOT NULL,
+    friend_nick BOOLEAN NOT NULL,
     topic      TEXT NOT NULL,
     topic_set  BOOLEAN NOT NULL,
     avatar     TEXT NOT NULL,
@@ -62,11 +63,17 @@ CREATE TABLE thread (
 CREATE TABLE puppet (
     id TEXT PRIMARY KEY,
 
-    name       TEXT NOT NULL,
+    name             TEXT NOT NULL,
-    name_set   BOOLEAN NOT NULL,
+    name_set         BOOLEAN NOT NULL DEFAULT false,
-    avatar     TEXT NOT NULL,
+    avatar           TEXT NOT NULL,
-    avatar_url TEXT NOT NULL,
+    avatar_url       TEXT NOT NULL,
-    avatar_set BOOLEAN NOT NULL,
+    avatar_set       BOOLEAN NOT NULL DEFAULT false,
+
+    contact_info_set BOOLEAN NOT NULL DEFAULT false,
+
+    username      TEXT NOT NULL DEFAULT '',
+    discriminator TEXT NOT NULL DEFAULT '',
+    is_bot        BOOLEAN NOT NULL DEFAULT false,
 
     custom_mxid  TEXT,
     access_token TEXT,
@@ -97,18 +104,18 @@ CREATE TABLE user_portal (
 );
 
 CREATE TABLE message (
-    dcid             TEXT,
+    dcid              TEXT,
-    dc_attachment_id TEXT,
+    dc_attachment_id  TEXT,
-    dc_edit_index    INTEGER,
+    dc_chan_id        TEXT,
-    dc_chan_id       TEXT,
+    dc_chan_receiver  TEXT,
-    dc_chan_receiver TEXT,
+    dc_sender         TEXT   NOT NULL,
-    dc_sender        TEXT   NOT NULL,
+    timestamp         BIGINT NOT NULL,
-    timestamp        BIGINT NOT NULL,
+    dc_edit_timestamp BIGINT NOT NULL,
-    dc_thread_id     TEXT   NOT NULL,
+    dc_thread_id      TEXT   NOT NULL,
 
     mxid TEXT NOT NULL UNIQUE,
 
-    PRIMARY KEY (dcid, dc_attachment_id, dc_edit_index, dc_chan_id, dc_chan_receiver),
+    PRIMARY KEY (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver),
     CONSTRAINT message_portal_fkey FOREIGN KEY (dc_chan_id, dc_chan_receiver) REFERENCES portal (dcid, receiver) ON DELETE CASCADE
 );
 
@@ -120,13 +127,12 @@ CREATE TABLE reaction (
     dc_emoji_name    TEXT,
     dc_thread_id     TEXT NOT NULL,
 
-    dc_first_attachment_id TEXT    NOT NULL,
+    dc_first_attachment_id TEXT NOT NULL,
-    _dc_first_edit_index   INTEGER NOT NULL DEFAULT 0,
 
     mxid TEXT NOT NULL UNIQUE,
 
     PRIMARY KEY (dc_chan_id, dc_chan_receiver, dc_msg_id, dc_sender, dc_emoji_name),
-    CONSTRAINT reaction_message_fkey FOREIGN KEY (dc_msg_id, dc_first_attachment_id, _dc_first_edit_index, dc_chan_id, dc_chan_receiver) REFERENCES message (dcid, dc_attachment_id, dc_edit_index, dc_chan_id, dc_chan_receiver) ON DELETE CASCADE
+    CONSTRAINT reaction_message_fkey FOREIGN KEY (dc_msg_id, dc_first_attachment_id, dc_chan_id, dc_chan_receiver) REFERENCES message (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver) ON DELETE CASCADE
 );
 
 CREATE TABLE role (

database/upgrades/16-add-contact-info.sql

@@ -0,0 +1,3 @@
+-- v16: Store whether custom contact info has been set for the puppet
+
+ALTER TABLE puppet ADD COLUMN contact_info_set BOOLEAN NOT NULL DEFAULT false;

database/upgrades/17-dm-portal-friend-nick.sql

@@ -0,0 +1,2 @@
+-- v17: Store whether DM portal name is a friend nickname
+ALTER TABLE portal ADD COLUMN friend_nick BOOLEAN NOT NULL DEFAULT false;

database/upgrades/18-extra-ghost-metadata.sql

@@ -0,0 +1,4 @@
+-- v18 (compatible with v15+): Store additional metadata for ghosts
+ALTER TABLE puppet ADD COLUMN username TEXT NOT NULL DEFAULT '';
+ALTER TABLE puppet ADD COLUMN discriminator TEXT NOT NULL DEFAULT '';
+ALTER TABLE puppet ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false;

database/upgrades/19-message-edit-ts.postgres.sql

@@ -0,0 +1,15 @@
+-- v19: Replace dc_edit_index with dc_edit_timestamp
+-- transaction: off
+BEGIN;
+
+ALTER TABLE reaction DROP CONSTRAINT reaction_message_fkey;
+ALTER TABLE message DROP CONSTRAINT message_pkey;
+ALTER TABLE message DROP COLUMN dc_edit_index;
+ALTER TABLE reaction DROP COLUMN _dc_first_edit_index;
+ALTER TABLE message ADD PRIMARY KEY (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver);
+ALTER TABLE reaction ADD CONSTRAINT reaction_message_fkey FOREIGN KEY (dc_msg_id, dc_first_attachment_id, dc_chan_id, dc_chan_receiver) REFERENCES message (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver) ON DELETE CASCADE;
+
+ALTER TABLE message ADD COLUMN dc_edit_timestamp BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE message ALTER COLUMN dc_edit_timestamp DROP DEFAULT;
+
+COMMIT;

database/upgrades/19-message-edit-ts.sqlite.sql

@@ -0,0 +1,48 @@
+-- v19: Replace dc_edit_index with dc_edit_timestamp
+-- transaction: off
+PRAGMA foreign_keys = OFF;
+BEGIN;
+
+CREATE TABLE message_new (
+    dcid              TEXT,
+    dc_attachment_id  TEXT,
+    dc_chan_id        TEXT,
+    dc_chan_receiver  TEXT,
+    dc_sender         TEXT   NOT NULL,
+    timestamp         BIGINT NOT NULL,
+    dc_edit_timestamp BIGINT NOT NULL,
+    dc_thread_id      TEXT   NOT NULL,
+
+    mxid TEXT NOT NULL UNIQUE,
+
+    PRIMARY KEY (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver),
+    CONSTRAINT message_portal_fkey FOREIGN KEY (dc_chan_id, dc_chan_receiver) REFERENCES portal (dcid, receiver) ON DELETE CASCADE
+);
+INSERT INTO message_new (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, dc_edit_timestamp, dc_thread_id, mxid)
+    SELECT dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver, dc_sender, timestamp, 0, dc_thread_id, mxid FROM message;
+DROP TABLE message;
+ALTER TABLE message_new RENAME TO message;
+
+CREATE TABLE reaction_new (
+    dc_chan_id       TEXT,
+    dc_chan_receiver TEXT,
+    dc_msg_id        TEXT,
+    dc_sender        TEXT,
+    dc_emoji_name    TEXT,
+    dc_thread_id     TEXT NOT NULL,
+
+    dc_first_attachment_id TEXT NOT NULL,
+
+    mxid TEXT NOT NULL UNIQUE,
+
+    PRIMARY KEY (dc_chan_id, dc_chan_receiver, dc_msg_id, dc_sender, dc_emoji_name),
+    CONSTRAINT reaction_message_fkey FOREIGN KEY (dc_msg_id, dc_first_attachment_id, dc_chan_id, dc_chan_receiver) REFERENCES message (dcid, dc_attachment_id, dc_chan_id, dc_chan_receiver) ON DELETE CASCADE
+);
+INSERT INTO reaction_new (dc_chan_id, dc_chan_receiver, dc_msg_id, dc_sender, dc_emoji_name, dc_thread_id, dc_first_attachment_id, mxid)
+    SELECT dc_chan_id, dc_chan_receiver, dc_msg_id, dc_sender, dc_emoji_name, COALESCE(dc_thread_id, ''), dc_first_attachment_id, mxid FROM reaction;
+DROP TABLE reaction;
+ALTER TABLE reaction_new RENAME TO reaction;
+
+PRAGMA foreign_key_check;
+COMMIT;
+PRAGMA foreign_keys = ON;

database/userportal.go

@@ -29,7 +29,7 @@ func (up UserPortal) Scan(l log.Logger, row dbutil.Scannable) *UserPortal {
 		l.Errorln("Error scanning user portal:", err)
 		panic(err)
 	}
-	up.Timestamp = time.UnixMilli(ts)
+	up.Timestamp = time.UnixMilli(ts).UTC()
 	return &up
 }
 

example-config.yaml

@@ -97,9 +97,11 @@ bridge:
     # Available variables:
     #   .Name - Guild name
     guild_name_template: '{{.Name}}'
-    # Should the bridge explicitly set the avatar and room name for DM portal rooms?
+    # Whether to explicitly set the avatar and room name for private chat portal rooms.
-    # This is implicitly enabled in encrypted rooms.
+    # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms.
-    private_chat_portal_meta: false
+    # If set to `always`, all DM rooms will have explicit names and avatars set.
+    # If set to `never`, DM rooms will never have names and avatars set.
+    private_chat_portal_meta: default
 
     portal_message_buffer: 128
 
@@ -143,6 +145,28 @@ bridge:
     # Whether or not created rooms should have federation enabled.
     # If false, created portal rooms will never be federated.
     federate_rooms: true
+    # Should the bridge upload media to the Discord CDN directly before sending the message when using a user token,
+    # like the official client does? The other option is sending the media in the message send request as a form part
+    # (which is always used by bots and webhooks).
+    use_discord_cdn_upload: true
+    # Should mxc uris copied from Discord be cached?
+    # This can be `never` to never cache, `unencrypted` to only cache unencrypted mxc uris, or `always` to cache everything.
+    # If you have a media repo that generates non-unique mxc uris, you should set this to never.
+    cache_media: unencrypted
+    # Patterns for converting Discord media to custom mxc:// URIs instead of reuploading.
+    # Each of the patterns can be set to null to disable custom URIs for that type of media.
+    # More details can be found at https://docs.mau.fi/bridges/go/discord/direct-media.html
+    media_patterns:
+        # Should custom mxc:// URIs be used instead of reuploading media?
+        enabled: false
+        # Pattern for normal message attachments.
+        attachments: mxc://discord-media.mau.dev/attachments|{{.ChannelID}}|{{.AttachmentID}}|{{.FileName}}
+        # Pattern for custom emojis.
+        emojis: mxc://discord-media.mau.dev/emojis|{{.ID}}.{{.Ext}}
+        # Pattern for stickers. Note that animated lottie stickers will not be converted if this is enabled.
+        stickers: mxc://discord-media.mau.dev/stickers|{{.ID}}.{{.Ext}}
+        # Pattern for static user avatars.
+        avatars: mxc://discord-media.mau.dev/avatars|{{.UserID}}|{{.AvatarID}}.{{.Ext}}
     # Settings for converting animated stickers.
     animated_sticker:
         # Format to which animated stickers should be converted.
@@ -184,6 +208,28 @@ bridge:
         # Optional extra text sent when joining a management room.
         additional_help: ""
 
+    # Settings for backfilling messages.
+    backfill:
+        # Limits for forward backfilling.
+        forward_limits:
+            # Initial backfill (when creating portal). 0 means backfill is disabled.
+            # A special unlimited value is not supported, you must set a limit. Initial backfill will
+            # fetch all messages first before backfilling anything, so high limits can take a lot of time.
+            initial:
+                dm: 0
+                channel: 0
+            # Missed message backfill (on startup).
+            # 0 means backfill is disabled, -1 means fetch all messages since last bridged message.
+            # When using unlimited backfill (-1), messages are backfilled as they are fetched.
+            # With limits, all messages up to the limit are fetched first and backfilled afterwards.
+            missed:
+                dm: 0
+                channel: 0
+        # Maximum members in a guild to enable backfilling. Set to -1 to disable limit.
+        # This can be used as a rough heuristic to disable backfilling in channels that are too active.
+        # Currently only applies to missed message backfill.
+        max_guild_members: -1
+
     # End-to-bridge encryption support options.
     #
     # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
@@ -200,6 +246,23 @@ bridge:
         # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
         # You must use a client that supports requesting keys from other users to use this feature.
         allow_key_sharing: false
+        # Options for deleting megolm sessions from the bridge.
+        delete_keys:
+            # Beeper-specific: delete outbound sessions when hungryserv confirms
+            # that the user has uploaded the key to key backup.
+            delete_outbound_on_ack: false
+            # Don't store outbound sessions in the inbound table.
+            dont_store_outbound: false
+            # Ratchet megolm sessions forward after decrypting messages.
+            ratchet_on_decrypt: false
+            # Delete fully used keys (index >= max_messages) after decrypting messages.
+            delete_fully_used_on_decrypt: false
+            # Delete previous megolm sessions from same device when receiving a new one.
+            delete_prev_on_new_session: false
+            # Delete megolm sessions received from a device when the device is deleted.
+            delete_on_device_delete: false
+            # Periodically delete megolm sessions when 2x max_age has passed since receiving the session.
+            periodically_delete_expired: false
         # What level of device verification should be required from users?
         #
         # Valid levels:

formatter_tag.go

@@ -192,7 +192,7 @@ func (s *discordTagParser) Parse(parent ast.Node, block text.Reader, pc parser.C
 	case strings.HasPrefix(tagName, ":"):
 		return &astDiscordCustomEmoji{name: tagName, astDiscordTag: tag}
 	case strings.HasPrefix(tagName, "a:"):
-		return &astDiscordCustomEmoji{name: tagName[1:], astDiscordTag: tag}
+		return &astDiscordCustomEmoji{name: tagName[1:], astDiscordTag: tag, animated: true}
 	default:
 		return nil
 	}
@@ -263,9 +263,9 @@ func (r *discordTagHTMLRenderer) renderDiscordMention(w util.BufWriter, source [
 	switch node := n.(type) {
 	case *astDiscordUserMention:
 		if user := node.portal.bridge.GetUserByID(strconv.FormatInt(node.id, 10)); user != nil {
-			_, _ = fmt.Fprintf(w, `<a href="https://matrix.to/#/%[1]s">%[1]s</a>`, user.MXID)
+			_, _ = fmt.Fprintf(w, `<a href="%s">%s</a>`, user.MXID.URI().MatrixToURL(), user.MXID)
 		} else if puppet := node.portal.bridge.GetPuppetByID(strconv.FormatInt(node.id, 10)); puppet != nil {
-			_, _ = fmt.Fprintf(w, `<a href="https://matrix.to/#/%s">%s</a>`, puppet.MXID, puppet.Name)
+			_, _ = fmt.Fprintf(w, `<a href="%s">%s</a>`, puppet.MXID.URI().MatrixToURL(), puppet.Name)
 		}
 		return
 	case *astDiscordRoleMention:
@@ -281,7 +281,7 @@ func (r *discordTagHTMLRenderer) renderDiscordMention(w util.BufWriter, source [
 		})
 		if portal != nil {
 			if portal.MXID != "" {
-				_, _ = fmt.Fprintf(w, `<a href="https://matrix.to/#/%s?via=%s">%s</a>`, portal.MXID, portal.bridge.AS.HomeserverDomain, portal.Name)
+				_, _ = fmt.Fprintf(w, `<a href="%s">%s</a>`, portal.MXID.URI(portal.bridge.AS.HomeserverDomain).MatrixToURL(), portal.Name)
 			} else {
 				_, _ = w.WriteString(portal.Name)
 			}
@@ -290,7 +290,11 @@ func (r *discordTagHTMLRenderer) renderDiscordMention(w util.BufWriter, source [
 	case *astDiscordCustomEmoji:
 		reactionMXC := node.portal.getEmojiMXCByDiscordID(strconv.FormatInt(node.id, 10), node.name, node.animated)
 		if !reactionMXC.IsEmpty() {
-			_, _ = fmt.Fprintf(w, `<img data-mx-emoticon src="%[1]s" alt="%[2]s" title="%[2]s" height="32"/>`, reactionMXC.String(), node.name)
+			attrs := "data-mx-emoticon"
+			if node.animated {
+				attrs += " data-mau-animated-emoji"
+			}
+			_, _ = fmt.Fprintf(w, `<img %[3]s src="%[1]s" alt="%[2]s" title="%[2]s" height="32"/>`, reactionMXC.String(), node.name, attrs)
 			return
 		}
 	case *astDiscordTimestamp:
@@ -305,7 +309,7 @@ func (r *discordTagHTMLRenderer) renderDiscordMention(w util.BufWriter, source [
 		const fullDatetimeFormat = "2006-01-02T15:04:05.000-0700"
 		fullRFC := ts.Format(fullDatetimeFormat)
 		fullHumanReadable := ts.Format(discordTimestampStyle('F').Format())
-		_, _ = fmt.Fprintf(w, `<time title="%s" datetime="%s"><strong>%s</strong></time>`, fullHumanReadable, fullRFC, formatted)
+		_, _ = fmt.Fprintf(w, `<time title="%s" datetime="%s" data-discord-style="%c"><strong>%s</strong></time>`, fullHumanReadable, fullRFC, node.style, formatted)
 	}
 	stringifiable, ok := n.(fmt.Stringer)
 	if ok {

go.mod

@@ -8,18 +8,18 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
-	github.com/lib/pq v1.10.7
+	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.16
-	github.com/rs/zerolog v1.29.0
+	github.com/rs/zerolog v1.29.1
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.2
 	github.com/yuin/goldmark v1.5.4
 	maunium.net/go/maulogger/v2 v2.4.1
-	maunium.net/go/mautrix v0.15.0
+	maunium.net/go/mautrix v0.15.2
 )
 
 require (
-	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
@@ -29,12 +29,13 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	go.mau.fi/zeroconfig v0.1.2 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	maunium.net/go/mauflag v1.0.0 // indirect
 )
 
-replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d
+replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20230426184739-79aea97f6660

go.sum

@@ -1,8 +1,8 @@
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d h1:xo6A9gSSu7mnxIXHBD1EPDyKEQFlI0N8r57Yf0gWiy8=
+github.com/beeper/discordgo v0.0.0-20230426184739-79aea97f6660 h1:5LFnUY/Aj/0k/UqeEmW2GS4ql1vxmivkrckPxUHf8oc=
-github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
+github.com/beeper/discordgo v0.0.0-20230426184739-79aea97f6660/go.mod h1:59+AOzzjmL6onAh62nuLXmn7dJCaC/owDLWbGtjTcFA=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 h1:rtAn27wIbmOGUs7RIbVgPEjb31ehTVniDwPGXyMxm5U=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -13,11 +13,10 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
-github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
@@ -28,8 +27,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
-github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -52,20 +51,16 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
 github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
@@ -77,5 +72,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
-maunium.net/go/mautrix v0.15.0 h1:gkK9HXc1SSPwY7qOAqchzj2xxYqiOYeee8lr28A2g/o=
+maunium.net/go/mautrix v0.15.2 h1:fUiVajeoOR92uJoSShHbCvh7uG6lDY4ZO4Mvt90LbjU=
-maunium.net/go/mautrix v0.15.0/go.mod h1:1v8QVDd7q/eJ+eg4sgeOSEafBAFhkt4ab2i97M3IkNQ=
+maunium.net/go/mautrix v0.15.2/go.mod h1:h4NwfKqE4YxGTLSgn/gawKzXAb2sF4qx8agL6QEFtGg=

guildportal.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	log "maunium.net/go/maulogger/v2"
+	"maunium.net/go/maulogger/v2/maulogadapt"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
@@ -219,7 +220,7 @@ func (guild *Guild) CreateMatrixRoom(user *User, meta *discordgo.Guild) error {
 	guild.bridge.guildsLock.Unlock()
 	guild.log.Infoln("Matrix room created:", guild.MXID)
 
-	user.ensureInvited(nil, guild.MXID, false)
+	user.ensureInvited(nil, guild.MXID, false, true)
 
 	return nil
 }
@@ -236,6 +237,7 @@ func (guild *Guild) UpdateInfo(source *User, meta *discordgo.Guild) *discordgo.G
 		guild.UpdateBridgeInfo()
 		guild.Update()
 	}
+	source.ensureInvited(nil, guild.MXID, false, false)
 	return meta
 }
 
@@ -295,12 +297,12 @@ func (guild *Guild) cleanup() {
 	intent := guild.bridge.Bot
 	if guild.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] {
 		err := intent.BeeperDeleteRoom(guild.MXID)
-		if err == nil || errors.Is(err, mautrix.MNotFound) {
+		if err != nil && !errors.Is(err, mautrix.MNotFound) {
-			return
+			guild.log.Errorfln("Failed to delete %s using hungryserv yeet endpoint: %v", guild.MXID, err)
 		}
-		guild.log.Warnfln("Failed to delete %s using hungryserv yeet endpoint, falling back to normal behavior: %v", guild.MXID, err)
+		return
 	}
-	guild.bridge.cleanupRoom(intent, guild.MXID, false, guild.log)
+	guild.bridge.cleanupRoom(intent, guild.MXID, false, *maulogadapt.MauAsZero(guild.log))
 }
 
 func (guild *Guild) RemoveMXID() {

main.go

@@ -18,13 +18,8 @@ package main
 
 import (
 	_ "embed"
-	"fmt"
-	"runtime"
-	"strings"
 	"sync"
 
-	"github.com/rs/zerolog"
-
 	"maunium.net/go/mautrix/bridge"
 	"maunium.net/go/mautrix/bridge/commands"
 	"maunium.net/go/mautrix/id"
@@ -101,22 +96,13 @@ func (br *DiscordBridge) Init() {
 
 	br.DB = database.New(br.Bridge.DB, br.Log.Sub("Database"))
 	discordLog = br.ZLog.With().Str("component", "discordgo").Logger()
-
-	// TODO move this to mautrix-go?
-	zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string {
-		files := strings.Split(file, "/")
-		file = files[len(files)-1]
-		name := runtime.FuncForPC(pc).Name()
-		fns := strings.Split(name, ".")
-		name = fns[len(fns)-1]
-		return fmt.Sprintf("%s:%d:%s()", file, line, name)
-	}
 }
 
 func (br *DiscordBridge) Start() {
 	if br.Config.Bridge.Provisioning.SharedSecret != "disable" {
 		br.provisioning = newProvisioningAPI(br)
 	}
+	go br.updatePuppetsContactInfo()
 	go br.startUsers()
 }
 
@@ -187,11 +173,13 @@ func main() {
 		attachmentTransfers: util.NewSyncMap[attachmentKey, *util.ReturnableOnce[*database.File]](),
 	}
 	br.Bridge = bridge.Bridge{
-		Name:         "mautrix-discord",
+		Name:              "mautrix-discord",
-		URL:          "https://github.com/mautrix/discord",
+		URL:               "https://github.com/mautrix/discord",
-		Description:  "A Matrix-Discord puppeting bridge.",
+		Description:       "A Matrix-Discord puppeting bridge.",
-		Version:      "0.2.0",
+		Version:           "0.4.0",
-		ProtocolName: "Discord",
+		ProtocolName:      "Discord",
+		BeeperServiceName: "discordgo",
+		BeeperNetworkName: "discord",
 
 		CryptoPickleKey: "maunium.net/go/mautrix-whatsapp",
 

portal_convert.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"html"
 	"strconv"
@@ -24,6 +25,7 @@ import (
 	"time"
 
 	"github.com/bwmarrin/discordgo"
+	"github.com/rs/zerolog"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
@@ -48,14 +50,14 @@ func (portal *Portal) createMediaFailedMessage(bridgeErr error) *event.MessageEv
 
 const DiscordStickerSize = 160
 
-func (portal *Portal) convertDiscordFile(typeName string, intent *appservice.IntentAPI, id, url string, content *event.MessageEventContent) *event.MessageEventContent {
+func (portal *Portal) convertDiscordFile(ctx context.Context, typeName string, intent *appservice.IntentAPI, id, url string, content *event.MessageEventContent) *event.MessageEventContent {
 	meta := AttachmentMeta{AttachmentID: id, MimeType: content.Info.MimeType}
 	if typeName == "sticker" && content.Info.MimeType == "application/json" {
 		meta.Converter = portal.bridge.convertLottie
 	}
 	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, meta)
 	if err != nil {
-		portal.log.Errorfln("Error copying attachment %s to Matrix: %v", id, err)
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to copy attachment to Matrix")
 		return portal.createMediaFailedMessage(err)
 	}
 	if typeName == "sticker" && content.Info.MimeType == "application/json" {
@@ -66,10 +68,6 @@ func (portal *Portal) convertDiscordFile(typeName string, intent *appservice.Int
 		content.Info.Width = dbFile.Width
 		content.Info.Height = dbFile.Height
 	}
-	if content.Info.Width == 0 && content.Info.Height == 0 && typeName == "sticker" {
-		content.Info.Width = DiscordStickerSize
-		content.Info.Height = DiscordStickerSize
-	}
 	if dbFile.DecryptionInfo != nil {
 		content.File = &event.EncryptedFileInfo{
 			EncryptedFile: *dbFile.DecryptionInfo,
@@ -78,8 +76,14 @@ func (portal *Portal) convertDiscordFile(typeName string, intent *appservice.Int
 	} else {
 		content.URL = dbFile.MXC.CUString()
 	}
+	return content
+}
 
-	if typeName == "sticker" && (content.Info.Width > DiscordStickerSize || content.Info.Height > DiscordStickerSize) {
+func (portal *Portal) cleanupConvertedStickerInfo(content *event.MessageEventContent) {
+	if content.Info.Width == 0 && content.Info.Height == 0 {
+		content.Info.Width = DiscordStickerSize
+		content.Info.Height = DiscordStickerSize
+	} else if content.Info.Width > DiscordStickerSize || content.Info.Height > DiscordStickerSize {
 		if content.Info.Width > content.Info.Height {
 			content.Info.Height /= content.Info.Width / DiscordStickerSize
 			content.Info.Width = DiscordStickerSize
@@ -91,36 +95,51 @@ func (portal *Portal) convertDiscordFile(typeName string, intent *appservice.Int
 			content.Info.Height = DiscordStickerSize
 		}
 	}
-	return content
 }
 
-func (portal *Portal) convertDiscordSticker(intent *appservice.IntentAPI, sticker *discordgo.Sticker) *ConvertedMessage {
+func (portal *Portal) convertDiscordSticker(ctx context.Context, intent *appservice.IntentAPI, sticker *discordgo.Sticker) *ConvertedMessage {
-	var mime string
+	var mime, ext string
 	switch sticker.FormatType {
 	case discordgo.StickerFormatTypePNG:
 		mime = "image/png"
+		ext = "png"
 	case discordgo.StickerFormatTypeAPNG:
 		mime = "image/apng"
+		ext = "png"
 	case discordgo.StickerFormatTypeLottie:
 		mime = "application/json"
+		ext = "json"
 	case discordgo.StickerFormatTypeGIF:
 		mime = "image/gif"
+		ext = "gif"
 	default:
-		portal.log.Warnfln("Unknown sticker format %d in %s", sticker.FormatType, sticker.ID)
+		zerolog.Ctx(ctx).Warn().
+			Int("sticker_format", int(sticker.FormatType)).
+			Str("sticker_id", sticker.ID).
+			Msg("Unknown sticker format")
 	}
+	content := &event.MessageEventContent{
+		Body: sticker.Name, // TODO find description from somewhere?
+		Info: &event.FileInfo{
+			MimeType: mime,
+		},
+	}
+
+	mxc := portal.bridge.Config.Bridge.MediaPatterns.Sticker(sticker.ID, ext)
+	if mxc.IsEmpty() {
+		content = portal.convertDiscordFile(ctx, "sticker", intent, sticker.ID, sticker.URL(), content)
+	} else {
+		content.URL = mxc.CUString()
+	}
+	portal.cleanupConvertedStickerInfo(content)
 	return &ConvertedMessage{
 		AttachmentID: sticker.ID,
 		Type:         event.EventSticker,
-		Content: portal.convertDiscordFile("sticker", intent, sticker.ID, sticker.URL(), &event.MessageEventContent{
+		Content:      content,
-			Body: sticker.Name, // TODO find description from somewhere?
-			Info: &event.FileInfo{
-				MimeType: mime,
-			},
-		}),
 	}
 }
 
-func (portal *Portal) convertDiscordAttachment(intent *appservice.IntentAPI, att *discordgo.MessageAttachment) *ConvertedMessage {
+func (portal *Portal) convertDiscordAttachment(ctx context.Context, intent *appservice.IntentAPI, att *discordgo.MessageAttachment) *ConvertedMessage {
 	content := &event.MessageEventContent{
 		Body: att.Filename,
 		Info: &event.FileInfo{
@@ -137,9 +156,20 @@ func (portal *Portal) convertDiscordAttachment(intent *appservice.IntentAPI, att
 		content.FileName = att.Filename
 	}
 
+	var extra map[string]any
+
 	switch strings.ToLower(strings.Split(att.ContentType, "/")[0]) {
 	case "audio":
 		content.MsgType = event.MsgAudio
+		if att.Waveform != nil {
+			// TODO convert waveform
+			extra = map[string]any{
+				"org.matrix.1767.audio": map[string]any{
+					"duration": int(att.DurationSeconds * 1000),
+				},
+				"org.matrix.msc3245.voice": map[string]any{},
+			}
+		}
 	case "image":
 		content.MsgType = event.MsgImage
 	case "video":
@@ -147,18 +177,25 @@ func (portal *Portal) convertDiscordAttachment(intent *appservice.IntentAPI, att
 	default:
 		content.MsgType = event.MsgFile
 	}
-	content = portal.convertDiscordFile("attachment", intent, att.ID, att.URL, content)
+	mxc := portal.bridge.Config.Bridge.MediaPatterns.Attachment(portal.Key.ChannelID, att.ID, att.Filename)
+	if mxc.IsEmpty() {
+		content = portal.convertDiscordFile(ctx, "attachment", intent, att.ID, att.URL, content)
+	} else {
+		content.URL = mxc.CUString()
+	}
 	return &ConvertedMessage{
 		AttachmentID: att.ID,
 		Type:         event.EventMessage,
 		Content:      content,
+		Extra:        extra,
 	}
 }
 
-func (portal *Portal) convertDiscordVideoEmbed(intent *appservice.IntentAPI, embed *discordgo.MessageEmbed) *ConvertedMessage {
+func (portal *Portal) convertDiscordVideoEmbed(ctx context.Context, intent *appservice.IntentAPI, embed *discordgo.MessageEmbed) *ConvertedMessage {
 	attachmentID := fmt.Sprintf("video_%s", embed.URL)
 	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, embed.Video.ProxyURL, portal.Encrypted, NoMeta)
 	if err != nil {
+		zerolog.Ctx(ctx).Err(err).Msg("Failed to copy video embed to Matrix")
 		return &ConvertedMessage{
 			AttachmentID: attachmentID,
 			Type:         event.EventMessage,
@@ -207,22 +244,24 @@ func (portal *Portal) convertDiscordVideoEmbed(intent *appservice.IntentAPI, emb
 	}
 }
 
-func (portal *Portal) convertDiscordMessage(intent *appservice.IntentAPI, msg *discordgo.Message) []*ConvertedMessage {
+func (portal *Portal) convertDiscordMessage(ctx context.Context, intent *appservice.IntentAPI, msg *discordgo.Message) []*ConvertedMessage {
 	predictedLength := len(msg.Attachments) + len(msg.StickerItems)
 	if msg.Content != "" {
 		predictedLength++
 	}
 	parts := make([]*ConvertedMessage, 0, predictedLength)
-	if textPart := portal.convertDiscordTextMessage(intent, msg); textPart != nil {
+	if textPart := portal.convertDiscordTextMessage(ctx, intent, msg); textPart != nil {
 		parts = append(parts, textPart)
 	}
+	log := zerolog.Ctx(ctx)
 	handledIDs := make(map[string]struct{})
 	for _, att := range msg.Attachments {
 		if _, handled := handledIDs[att.ID]; handled {
 			continue
 		}
 		handledIDs[att.ID] = struct{}{}
-		if part := portal.convertDiscordAttachment(intent, att); part != nil {
+		log := log.With().Str("attachment_id", att.ID).Logger()
+		if part := portal.convertDiscordAttachment(log.WithContext(ctx), intent, att); part != nil {
 			parts = append(parts, part)
 		}
 	}
@@ -231,11 +270,12 @@ func (portal *Portal) convertDiscordMessage(intent *appservice.IntentAPI, msg *d
 			continue
 		}
 		handledIDs[sticker.ID] = struct{}{}
-		if part := portal.convertDiscordSticker(intent, sticker); part != nil {
+		log := log.With().Str("sticker_id", sticker.ID).Logger()
+		if part := portal.convertDiscordSticker(log.WithContext(ctx), intent, sticker); part != nil {
 			parts = append(parts, part)
 		}
 	}
-	for _, embed := range msg.Embeds {
+	for i, embed := range msg.Embeds {
 		// Ignore non-video embeds, they're handled in convertDiscordTextMessage
 		if getEmbedType(embed) != EmbedVideo {
 			continue
@@ -245,7 +285,12 @@ func (portal *Portal) convertDiscordMessage(intent *appservice.IntentAPI, msg *d
 			continue
 		}
 		handledIDs[embed.URL] = struct{}{}
-		part := portal.convertDiscordVideoEmbed(intent, embed)
+		log := log.With().
+			Str("computed_embed_type", "video").
+			Str("embed_type", string(embed.Type)).
+			Int("embed_index", i).
+			Logger()
+		part := portal.convertDiscordVideoEmbed(log.WithContext(ctx), intent, embed)
 		if part != nil {
 			parts = append(parts, part)
 		}
@@ -274,7 +319,8 @@ const (
 	embedFooterDateSeparator = ` • `
 )
 
-func (portal *Portal) convertDiscordRichEmbed(intent *appservice.IntentAPI, embed *discordgo.MessageEmbed, msgID string, index int) string {
+func (portal *Portal) convertDiscordRichEmbed(ctx context.Context, intent *appservice.IntentAPI, embed *discordgo.MessageEmbed, msgID string, index int) string {
+	log := zerolog.Ctx(ctx)
 	var htmlParts []string
 	if embed.Author != nil {
 		var authorHTML string
@@ -286,7 +332,7 @@ func (portal *Portal) convertDiscordRichEmbed(intent *appservice.IntentAPI, embe
 		if embed.Author.ProxyIconURL != "" {
 			dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, embed.Author.ProxyIconURL, false, NoMeta)
 			if err != nil {
-				portal.log.Warnfln("Failed to reupload author icon in embed #%d of message %s: %v", index+1, msgID, err)
+				log.Warn().Err(err).Msg("Failed to reupload author icon in embed")
 			} else {
 				authorHTML = fmt.Sprintf(embedHTMLAuthorWithImage, dbFile.MXC, authorNameHTML)
 			}
@@ -336,7 +382,7 @@ func (portal *Portal) convertDiscordRichEmbed(intent *appservice.IntentAPI, embe
 	if embed.Image != nil {
 		dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, embed.Image.ProxyURL, false, NoMeta)
 		if err != nil {
-			portal.log.Warnfln("Failed to reupload image in embed #%d of message %s: %v", index+1, msgID, err)
+			log.Warn().Err(err).Msg("Failed to reupload image in embed")
 		} else {
 			htmlParts = append(htmlParts, fmt.Sprintf(embedHTMLImage, dbFile.MXC))
 		}
@@ -346,7 +392,7 @@ func (portal *Portal) convertDiscordRichEmbed(intent *appservice.IntentAPI, embe
 		formattedTime := embed.Timestamp
 		parsedTS, err := time.Parse(time.RFC3339, embed.Timestamp)
 		if err != nil {
-			portal.log.Warnfln("Failed to parse timestamp in embed #%d of message %s: %v", index+1, msgID, err)
+			log.Warn().Err(err).Msg("Failed to parse timestamp in embed")
 		} else {
 			formattedTime = parsedTS.Format(discordTimestampStyle('F').Format())
 		}
@@ -362,7 +408,7 @@ func (portal *Portal) convertDiscordRichEmbed(intent *appservice.IntentAPI, embe
 		if embed.Footer.ProxyIconURL != "" {
 			dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, embed.Footer.ProxyIconURL, false, NoMeta)
 			if err != nil {
-				portal.log.Warnfln("Failed to reupload footer icon in embed #%d of message %s: %v", index+1, msgID, err)
+				log.Warn().Err(err).Msg("Failed to reupload footer icon in embed")
 			} else {
 				footerHTML = fmt.Sprintf(embedHTMLFooterWithImage, dbFile.MXC, html.EscapeString(embed.Footer.Text), datePart)
 			}
@@ -391,40 +437,40 @@ type BeeperLinkPreview struct {
 	ImageEncryption *event.EncryptedFileInfo `json:"beeper:image:encryption,omitempty"`
 }
 
-func (portal *Portal) convertDiscordLinkEmbedImage(intent *appservice.IntentAPI, url string, width, height int, preview *BeeperLinkPreview) {
+func (portal *Portal) convertDiscordLinkEmbedImage(ctx context.Context, intent *appservice.IntentAPI, url string, width, height int, preview *BeeperLinkPreview) {
 	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, NoMeta)
 	if err != nil {
-		portal.log.Warnfln("Failed to copy image in URL preview: %v", err)
+		zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to reupload image in URL preview")
+		return
+	}
+	if width != 0 || height != 0 {
+		preview.ImageWidth = width
+		preview.ImageHeight = height
 	} else {
-		if width != 0 || height != 0 {
+		preview.ImageWidth = dbFile.Width
-			preview.ImageWidth = width
+		preview.ImageHeight = dbFile.Height
-			preview.ImageHeight = height
+	}
-		} else {
+	preview.ImageSize = dbFile.Size
-			preview.ImageWidth = dbFile.Width
+	preview.ImageType = dbFile.MimeType
-			preview.ImageHeight = dbFile.Height
+	if dbFile.Encrypted {
-		}
+		preview.ImageEncryption = &event.EncryptedFileInfo{
-		preview.ImageSize = dbFile.Size
+			EncryptedFile: *dbFile.DecryptionInfo,
-		preview.ImageType = dbFile.MimeType
+			URL:           dbFile.MXC.CUString(),
-		if dbFile.Encrypted {
-			preview.ImageEncryption = &event.EncryptedFileInfo{
-				EncryptedFile: *dbFile.DecryptionInfo,
-				URL:           dbFile.MXC.CUString(),
-			}
-		} else {
-			preview.ImageURL = dbFile.MXC.CUString()
 		}
+	} else {
+		preview.ImageURL = dbFile.MXC.CUString()
 	}
 }
 
-func (portal *Portal) convertDiscordLinkEmbedToBeeper(intent *appservice.IntentAPI, embed *discordgo.MessageEmbed) *BeeperLinkPreview {
+func (portal *Portal) convertDiscordLinkEmbedToBeeper(ctx context.Context, intent *appservice.IntentAPI, embed *discordgo.MessageEmbed) *BeeperLinkPreview {
 	var preview BeeperLinkPreview
 	preview.MatchedURL = embed.URL
 	preview.Title = embed.Title
 	preview.Description = embed.Description
 	if embed.Image != nil {
-		portal.convertDiscordLinkEmbedImage(intent, embed.Image.ProxyURL, embed.Image.Width, embed.Image.Height, &preview)
+		portal.convertDiscordLinkEmbedImage(ctx, intent, embed.Image.ProxyURL, embed.Image.Width, embed.Image.Height, &preview)
 	} else if embed.Thumbnail != nil {
-		portal.convertDiscordLinkEmbedImage(intent, embed.Thumbnail.ProxyURL, embed.Thumbnail.Width, embed.Thumbnail.Height, &preview)
+		portal.convertDiscordLinkEmbedImage(ctx, intent, embed.Thumbnail.ProxyURL, embed.Thumbnail.Width, embed.Thumbnail.Height, &preview)
 	}
 	return &preview
 }
@@ -472,7 +518,8 @@ func isPlainGifMessage(msg *discordgo.Message) bool {
 	return len(msg.Embeds) == 1 && msg.Embeds[0].Video != nil && msg.Embeds[0].URL == msg.Content && msg.Embeds[0].Type == discordgo.EmbedTypeGifv
 }
 
-func (portal *Portal) convertDiscordTextMessage(intent *appservice.IntentAPI, msg *discordgo.Message) *ConvertedMessage {
+func (portal *Portal) convertDiscordTextMessage(ctx context.Context, intent *appservice.IntentAPI, msg *discordgo.Message) *ConvertedMessage {
+	log := zerolog.Ctx(ctx)
 	if msg.Type == discordgo.MessageTypeCall {
 		return &ConvertedMessage{Type: event.EventMessage, Content: &event.MessageEventContent{
 			MsgType: event.MsgEmote,
@@ -495,15 +542,24 @@ func (portal *Portal) convertDiscordTextMessage(intent *appservice.IntentAPI, ms
 	}
 	previews := make([]*BeeperLinkPreview, 0)
 	for i, embed := range msg.Embeds {
+		if i == 0 && msg.MessageReference == nil && isReplyEmbed(embed) {
+			continue
+		}
+		with := log.With().
+			Str("embed_type", string(embed.Type)).
+			Int("embed_index", i)
 		switch getEmbedType(embed) {
 		case EmbedRich:
-			htmlParts = append(htmlParts, portal.convertDiscordRichEmbed(intent, embed, msg.ID, i))
+			log := with.Str("computed_embed_type", "rich").Logger()
+			htmlParts = append(htmlParts, portal.convertDiscordRichEmbed(log.WithContext(ctx), intent, embed, msg.ID, i))
 		case EmbedLinkPreview:
-			previews = append(previews, portal.convertDiscordLinkEmbedToBeeper(intent, embed))
+			log := with.Str("computed_embed_type", "link preview").Logger()
+			previews = append(previews, portal.convertDiscordLinkEmbedToBeeper(log.WithContext(ctx), intent, embed))
 		case EmbedVideo:
 			// Ignore video embeds, they're handled as separate messages
 		default:
-			portal.log.Warnfln("Unknown type %s in embed #%d of message %s", embed.Type, i+1, msg.ID)
+			log := with.Logger()
+			log.Warn().Msg("Unknown embed type in message")
 		}
 	}
 

puppet.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"regexp"
+	"strings"
 	"sync"
 
 	"github.com/bwmarrin/discordgo"
@@ -10,6 +11,7 @@ import (
 
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/bridge"
+	"maunium.net/go/mautrix/bridge/bridgeconfig"
 	"maunium.net/go/mautrix/id"
 
 	"go.mau.fi/mautrix-discord/database"
@@ -156,6 +158,18 @@ func (br *DiscordBridge) FormatPuppetMXID(did string) id.UserID {
 	)
 }
 
+func (br *DiscordBridge) updatePuppetsContactInfo() {
+	if br.Config.Homeserver.Software != bridgeconfig.SoftwareHungry {
+		return
+	}
+	for _, puppet := range br.GetAllPuppets() {
+		if !puppet.ContactInfoSet && puppet.NameSet {
+			puppet.ResendContactInfo()
+			puppet.Update()
+		}
+	}
+}
+
 func (puppet *Puppet) GetDisplayname() string {
 	return puppet.Name
 }
@@ -204,7 +218,7 @@ func (puppet *Puppet) UpdateName(info *discordgo.User) bool {
 		puppet.log.Warn().Err(err).Msg("Failed to update displayname")
 	} else {
 		go puppet.updatePortalMeta(func(portal *Portal) {
-			if portal.UpdateNameDirect(puppet.Name) {
+			if portal.UpdateNameDirect(puppet.Name, false) {
 				portal.Update()
 				portal.UpdateBridgeInfo()
 			}
@@ -223,12 +237,21 @@ func (puppet *Puppet) UpdateAvatar(info *discordgo.User) bool {
 	puppet.AvatarSet = false
 	puppet.AvatarURL = id.ContentURI{}
 
-	// TODO should we just use discord's default avatars for users with no avatar?
 	if puppet.Avatar != "" && (puppet.AvatarURL.IsEmpty() || avatarChanged) {
-		url, err := uploadAvatar(puppet.DefaultIntent(), info.AvatarURL(""))
+		downloadURL := discordgo.EndpointUserAvatar(info.ID, info.Avatar)
-		if err != nil {
+		ext := "png"
-			puppet.log.Warn().Err(err).Str("avatar_id", puppet.Avatar).Msg("Failed to reupload user avatar")
+		if strings.HasPrefix(info.Avatar, "a_") {
-			return true
+			downloadURL = discordgo.EndpointUserAvatarAnimated(info.ID, info.Avatar)
+			ext = "gif"
+		}
+		url := puppet.bridge.Config.Bridge.MediaPatterns.Avatar(info.ID, info.Avatar, ext)
+		if url.IsEmpty() {
+			var err error
+			url, err = uploadAvatar(puppet.DefaultIntent(), downloadURL)
+			if err != nil {
+				puppet.log.Warn().Err(err).Str("avatar_id", puppet.Avatar).Msg("Failed to reupload user avatar")
+				return true
+			}
 		}
 		puppet.AvatarURL = url
 	}
@@ -271,9 +294,53 @@ func (puppet *Puppet) UpdateInfo(source *User, info *discordgo.User) {
 	}
 
 	changed := false
+	changed = puppet.UpdateContactInfo(info) || changed
 	changed = puppet.UpdateName(info) || changed
 	changed = puppet.UpdateAvatar(info) || changed
 	if changed {
 		puppet.Update()
 	}
 }
+
+func (puppet *Puppet) UpdateContactInfo(info *discordgo.User) bool {
+	changed := false
+	if puppet.Username != info.Username {
+		puppet.Username = info.Username
+		changed = true
+	}
+	if puppet.Discriminator != info.Discriminator {
+		puppet.Discriminator = info.Discriminator
+		changed = true
+	}
+	if puppet.IsBot != info.Bot {
+		puppet.IsBot = info.Bot
+		changed = true
+	}
+	if changed {
+		puppet.ContactInfoSet = false
+		puppet.ResendContactInfo()
+		return true
+	}
+	return false
+}
+
+func (puppet *Puppet) ResendContactInfo() {
+	if puppet.bridge.Config.Homeserver.Software != bridgeconfig.SoftwareHungry || puppet.ContactInfoSet {
+		return
+	}
+	contactInfo := map[string]any{
+		"com.beeper.bridge.identifiers": []string{
+			fmt.Sprintf("discord:%s#%s", puppet.Username, puppet.Discriminator),
+		},
+		"com.beeper.bridge.remote_id":      puppet.ID,
+		"com.beeper.bridge.service":        puppet.bridge.BeeperServiceName,
+		"com.beeper.bridge.network":        puppet.bridge.BeeperNetworkName,
+		"com.beeper.bridge.is_network_bot": puppet.IsBot,
+	}
+	err := puppet.DefaultIntent().BeeperUpdateProfile(contactInfo)
+	if err != nil {
+		puppet.log.Warn().Err(err).Msg("Failed to store custom contact info in profile")
+	} else {
+		puppet.ContactInfoSet = true
+	}
+}

user.go

@@ -6,6 +6,7 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -15,6 +16,7 @@ import (
 	"github.com/bwmarrin/discordgo"
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
+	"maunium.net/go/mautrix/util/dbutil"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
@@ -61,6 +63,8 @@ type User struct {
 	pendingInteractionsLock sync.Mutex
 
 	nextDiscordUploadID atomic.Int32
+
+	relationships map[string]*discordgo.Relationship
 }
 
 func (user *User) GetRemoteID() string {
@@ -76,20 +80,24 @@ func (user *User) GetRemoteName() string {
 
 var discordLog zerolog.Logger
 
+func discordToZeroLevel(level int) zerolog.Level {
+	switch level {
+	case discordgo.LogError:
+		return zerolog.ErrorLevel
+	case discordgo.LogWarning:
+		return zerolog.WarnLevel
+	case discordgo.LogInformational:
+		return zerolog.InfoLevel
+	case discordgo.LogDebug:
+		fallthrough
+	default:
+		return zerolog.DebugLevel
+	}
+}
+
 func init() {
 	discordgo.Logger = func(msgL, caller int, format string, a ...interface{}) {
-		var level zerolog.Level
+		discordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...)
-		switch msgL {
-		case discordgo.LogError:
-			level = zerolog.ErrorLevel
-		case discordgo.LogWarning:
-			level = zerolog.WarnLevel
-		case discordgo.LogInformational:
-			level = zerolog.InfoLevel
-		case discordgo.LogDebug:
-			level = zerolog.DebugLevel
-		}
-		discordLog.WithLevel(level).Caller(caller+1).Msgf(strings.TrimSpace(format), a...)
 	}
 }
 
@@ -188,6 +196,8 @@ func (br *DiscordBridge) NewUser(dbUser *database.User) *User {
 		PermissionLevel: br.Config.Bridge.Permissions.Get(dbUser.MXID),
 
 		pendingInteractions: make(map[string]*WrappedCommandEvent),
+
+		relationships: make(map[string]*discordgo.Relationship),
 	}
 	user.nextDiscordUploadID.Store(rand.Int31n(100))
 	user.BridgeState = br.NewBridgeStateQueue(user)
@@ -241,7 +251,7 @@ func (user *User) startupTryConnect(retryCount int) {
 		user.log.Error().Err(err).Msg("Error connecting on startup")
 		closeErr := &websocket.CloseError{}
 		if errors.As(err, &closeErr) && closeErr.Code == 4004 {
-			user.invalidAuthHandler(nil, nil)
+			user.invalidAuthHandler(nil)
 		} else if retryCount < 6 {
 			user.BridgeState.Send(status.BridgeState{StateEvent: status.StateTransientDisconnect, Error: "dc-unknown-websocket-error", Message: err.Error()})
 			retryInSeconds := 2 << retryCount
@@ -323,7 +333,7 @@ func (user *User) getSpaceRoom(ptr *id.RoomID, name, topic string, parent id.Roo
 	} else {
 		*ptr = resp.RoomID
 		user.Update()
-		user.ensureInvited(nil, *ptr, false)
+		user.ensureInvited(nil, *ptr, false, true)
 
 		if parent != "" {
 			_, err = user.bridge.Bot.SendStateEvent(parent, event.StateSpaceChild, resp.RoomID.String(), &event.SpaceChildEventContent{
@@ -429,7 +439,7 @@ func (user *User) syncChatDoublePuppetDetails(portal *Portal, justCreated bool)
 	}
 
 	// TODO sync mute status properly
-	if portal.GuildID != "" && user.bridge.Config.Bridge.MuteChannelsOnCreate {
+	if portal.GuildID != "" && user.bridge.Config.Bridge.MuteChannelsOnCreate && justCreated {
 		user.mutePortal(doublePuppetIntent, portal, false)
 	}
 }
@@ -555,44 +565,90 @@ func (user *User) Connect() error {
 	// TODO move to config
 	if os.Getenv("DISCORD_DEBUG") == "1" {
 		session.LogLevel = discordgo.LogDebug
+	} else {
+		session.LogLevel = discordgo.LogInformational
+	}
+	userDiscordLog := user.log.With().Str("component", "discordgo").Logger()
+	session.Logger = func(msgL, caller int, format string, a ...interface{}) {
+		userDiscordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...)
 	}
 	if !session.IsUser {
 		session.Identify.Intents = BotIntents
 	}
+	session.EventHandler = user.eventHandlerSync
 
 	user.Session = session
 
-	user.Session.AddHandler(user.readyHandler)
-	user.Session.AddHandler(user.resumeHandler)
-	user.Session.AddHandler(user.connectedHandler)
-	user.Session.AddHandler(user.disconnectedHandler)
-	user.Session.AddHandler(user.invalidAuthHandler)
-
-	user.Session.AddHandler(user.guildCreateHandler)
-	user.Session.AddHandler(user.guildDeleteHandler)
-	user.Session.AddHandler(user.guildUpdateHandler)
-	user.Session.AddHandler(user.guildRoleCreateHandler)
-	user.Session.AddHandler(user.guildRoleUpdateHandler)
-	user.Session.AddHandler(user.guildRoleDeleteHandler)
-
-	user.Session.AddHandler(user.channelCreateHandler)
-	user.Session.AddHandler(user.channelDeleteHandler)
-	user.Session.AddHandler(user.channelPinsUpdateHandler)
-	user.Session.AddHandler(user.channelUpdateHandler)
-
-	user.Session.AddHandler(user.messageCreateHandler)
-	user.Session.AddHandler(user.messageDeleteHandler)
-	user.Session.AddHandler(user.messageUpdateHandler)
-	user.Session.AddHandler(user.reactionAddHandler)
-	user.Session.AddHandler(user.reactionRemoveHandler)
-	user.Session.AddHandler(user.messageAckHandler)
-	user.Session.AddHandler(user.typingStartHandler)
-
-	user.Session.AddHandler(user.interactionSuccessHandler)
-
 	return user.Session.Open()
 }
 
+func (user *User) eventHandlerSync(rawEvt any) {
+	go user.eventHandler(rawEvt)
+}
+
+func (user *User) eventHandler(rawEvt any) {
+	switch evt := rawEvt.(type) {
+	case *discordgo.Ready:
+		user.readyHandler(evt)
+	case *discordgo.Resumed:
+		user.resumeHandler(evt)
+	case *discordgo.Connect:
+		user.connectedHandler(evt)
+	case *discordgo.Disconnect:
+		user.disconnectedHandler(evt)
+	case *discordgo.InvalidAuth:
+		user.invalidAuthHandler(evt)
+	case *discordgo.GuildCreate:
+		user.guildCreateHandler(evt)
+	case *discordgo.GuildDelete:
+		user.guildDeleteHandler(evt)
+	case *discordgo.GuildUpdate:
+		user.guildUpdateHandler(evt)
+	case *discordgo.GuildRoleCreate:
+		user.discordRoleToDB(evt.GuildID, evt.Role, nil, nil)
+	case *discordgo.GuildRoleUpdate:
+		user.discordRoleToDB(evt.GuildID, evt.Role, nil, nil)
+	case *discordgo.GuildRoleDelete:
+		user.bridge.DB.Role.DeleteByID(evt.GuildID, evt.RoleID)
+	case *discordgo.ChannelCreate:
+		user.channelCreateHandler(evt)
+	case *discordgo.ChannelDelete:
+		user.channelDeleteHandler(evt)
+	case *discordgo.ChannelUpdate:
+		user.channelUpdateHandler(evt)
+	case *discordgo.ChannelRecipientAdd:
+		user.channelRecipientAdd(evt)
+	case *discordgo.ChannelRecipientRemove:
+		user.channelRecipientRemove(evt)
+	case *discordgo.RelationshipAdd:
+		user.relationshipAddHandler(evt)
+	case *discordgo.RelationshipRemove:
+		user.relationshipRemoveHandler(evt)
+	case *discordgo.RelationshipUpdate:
+		user.relationshipUpdateHandler(evt)
+	case *discordgo.MessageCreate:
+		user.pushPortalMessage(evt, "message create", evt.ChannelID, evt.GuildID)
+	case *discordgo.MessageDelete:
+		user.pushPortalMessage(evt, "message delete", evt.ChannelID, evt.GuildID)
+	case *discordgo.MessageUpdate:
+		user.pushPortalMessage(evt, "message update", evt.ChannelID, evt.GuildID)
+	case *discordgo.MessageReactionAdd:
+		user.pushPortalMessage(evt, "reaction add", evt.ChannelID, evt.GuildID)
+	case *discordgo.MessageReactionRemove:
+		user.pushPortalMessage(evt, "reaction remove", evt.ChannelID, evt.GuildID)
+	case *discordgo.MessageAck:
+		user.messageAckHandler(evt)
+	case *discordgo.TypingStart:
+		user.typingStartHandler(evt)
+	case *discordgo.InteractionSuccess:
+		user.interactionSuccessHandler(evt)
+	case *discordgo.Event:
+		// Ignore
+	default:
+		user.log.Debug().Type("event_type", evt).Msg("Unhandled event")
+	}
+}
+
 func (user *User) Disconnect() error {
 	user.Lock()
 	defer user.Unlock()
@@ -619,7 +675,24 @@ func (user *User) getGuildBridgingMode(guildID string) database.GuildBridgingMod
 	return guild.BridgingMode
 }
 
-func (user *User) readyHandler(_ *discordgo.Session, r *discordgo.Ready) {
+type ChannelSlice []*discordgo.Channel
+
+func (s ChannelSlice) Len() int {
+	return len(s)
+}
+
+func (s ChannelSlice) Less(i, j int) bool {
+	if s[i].Position != 0 || s[j].Position != 0 {
+		return s[i].Position < s[j].Position
+	}
+	return compareMessageIDs(s[i].LastMessageID, s[j].LastMessageID) == 1
+}
+
+func (s ChannelSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (user *User) readyHandler(r *discordgo.Ready) {
 	user.log.Debug().Msg("Discord connection ready")
 	user.bridgeStateLock.Lock()
 	user.wasLoggedOut = false
@@ -642,6 +715,10 @@ func (user *User) readyHandler(_ *discordgo.Session, r *discordgo.Ready) {
 	user.BridgeState.Send(status.BridgeState{StateEvent: status.StateBackfilling})
 	user.tryAutomaticDoublePuppeting()
 
+	for _, relationship := range r.Relationships {
+		user.relationships[relationship.ID] = relationship
+	}
+
 	updateTS := time.Now()
 	portalsInSpace := make(map[string]bool)
 	for _, guild := range user.GetPortals() {
@@ -650,6 +727,8 @@ func (user *User) readyHandler(_ *discordgo.Session, r *discordgo.Ready) {
 	for _, guild := range r.Guilds {
 		user.handleGuild(guild, updateTS, portalsInSpace[guild.ID])
 	}
+	// The private channel list doesn't seem to be sorted by default, so sort it by message IDs (highest=newest first)
+	sort.Sort(ChannelSlice(r.PrivateChannels))
 	for i, ch := range r.PrivateChannels {
 		portal := user.GetPortalByMeta(ch)
 		user.handlePrivateChannel(portal, ch, updateTS, i < user.bridge.Config.Bridge.PrivateChannelCreateLimit, portalsInSpace[portal.Key.ChannelID])
@@ -659,7 +738,7 @@ func (user *User) readyHandler(_ *discordgo.Session, r *discordgo.Ready) {
 	if r.ReadState != nil && r.ReadState.Version > user.ReadStateVersion {
 		// TODO can we figure out which read states are actually new?
 		for _, entry := range r.ReadState.Entries {
-			user.messageAckHandler(nil, &discordgo.MessageAck{
+			user.messageAckHandler(&discordgo.MessageAck{
 				MessageID: string(entry.LastMessageID),
 				ChannelID: entry.ID,
 			})
@@ -696,7 +775,7 @@ func (user *User) subscribeGuilds(delay time.Duration) {
 	}
 }
 
-func (user *User) resumeHandler(_ *discordgo.Session, r *discordgo.Resumed) {
+func (user *User) resumeHandler(_ *discordgo.Resumed) {
 	user.log.Debug().Msg("Discord connection resumed")
 	user.subscribeGuilds(0 * time.Second)
 }
@@ -718,6 +797,55 @@ func (user *User) addPrivateChannelToSpace(portal *Portal) bool {
 	}
 }
 
+func (user *User) relationshipAddHandler(r *discordgo.RelationshipAdd) {
+	user.log.Debug().Interface("relationship", r.Relationship).Msg("Relationship added")
+	user.relationships[r.ID] = r.Relationship
+	user.handleRelationshipChange(r.ID, r.Nickname)
+}
+
+func (user *User) relationshipUpdateHandler(r *discordgo.RelationshipUpdate) {
+	user.log.Debug().Interface("relationship", r.Relationship).Msg("Relationship update")
+	user.relationships[r.ID] = r.Relationship
+	user.handleRelationshipChange(r.ID, r.Nickname)
+}
+
+func (user *User) relationshipRemoveHandler(r *discordgo.RelationshipRemove) {
+	user.log.Debug().Str("other_user_id", r.ID).Msg("Relationship removed")
+	delete(user.relationships, r.ID)
+	user.handleRelationshipChange(r.ID, "")
+}
+
+func (user *User) handleRelationshipChange(userID, nickname string) {
+	puppet := user.bridge.GetPuppetByID(userID)
+	portal := user.FindPrivateChatWith(userID)
+	if portal == nil || puppet == nil {
+		return
+	}
+
+	updated := portal.FriendNick == (nickname != "")
+	portal.FriendNick = nickname != ""
+	if nickname != "" {
+		updated = portal.UpdateNameDirect(nickname, true)
+	} else if portal.Name != puppet.Name {
+		if portal.shouldSetDMRoomMetadata() {
+			updated = portal.UpdateNameDirect(puppet.Name, false)
+		} else if portal.NameSet {
+			_, err := portal.MainIntent().SendStateEvent(portal.MXID, event.StateRoomName, "", map[string]any{})
+			if err != nil {
+				portal.log.Warn().Err(err).Msg("Failed to clear room name after friend nickname was removed")
+			} else {
+				portal.log.Debug().Msg("Cleared room name after friend nickname was removed")
+				portal.NameSet = false
+				portal.Update()
+				updated = true
+			}
+		}
+	}
+	if !updated {
+		portal.Update()
+	}
+}
+
 func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel, timestamp time.Time, create, isInSpace bool) {
 	if create && portal.MXID == "" {
 		err := portal.CreateMatrixRoom(user, meta)
@@ -728,6 +856,7 @@ func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel,
 		}
 	} else {
 		portal.UpdateInfo(user, meta)
+		portal.ForwardBackfillMissed(user, meta)
 	}
 	user.MarkInPortal(database.UserPortal{
 		DiscordID: portal.Key.ChannelID,
@@ -759,7 +888,7 @@ func (user *User) addGuildToSpace(guild *Guild, isInSpace bool, timestamp time.T
 	return isInSpace
 }
 
-func (user *User) discordRoleToDB(guildID string, role *discordgo.Role, dbRole *database.Role) (*database.Role, bool) {
+func (user *User) discordRoleToDB(guildID string, role *discordgo.Role, dbRole *database.Role, txn dbutil.Execable) bool {
 	var changed bool
 	if dbRole == nil {
 		dbRole = user.bridge.DB.Role.New()
@@ -777,7 +906,10 @@ func (user *User) discordRoleToDB(guildID string, role *discordgo.Role, dbRole *
 			dbRole.Permissions != role.Permissions
 	}
 	dbRole.Role = *role
-	return dbRole, changed
+	if changed {
+		dbRole.Upsert(txn)
+	}
+	return changed
 }
 
 func (user *User) handleGuildRoles(guildID string, newRoles []*discordgo.Role) {
@@ -792,11 +924,8 @@ func (user *User) handleGuildRoles(guildID string, newRoles []*discordgo.Role) {
 		panic(err)
 	}
 	for _, role := range newRoles {
-		dbRole, changed := user.discordRoleToDB(guildID, role, existingRoleMap[role.ID])
+		user.discordRoleToDB(guildID, role, existingRoleMap[role.ID], txn)
 		delete(existingRoleMap, role.ID)
-		if changed {
-			dbRole.Upsert(txn)
-		}
 	}
 	for _, removeRole := range existingRoleMap {
 		removeRole.Delete(txn)
@@ -812,20 +941,6 @@ func (user *User) handleGuildRoles(guildID string, newRoles []*discordgo.Role) {
 	}
 }
 
-func (user *User) guildRoleCreateHandler(_ *discordgo.Session, r *discordgo.GuildRoleCreate) {
-	dbRole, _ := user.discordRoleToDB(r.GuildID, r.Role, nil)
-	dbRole.Upsert(nil)
-}
-
-func (user *User) guildRoleUpdateHandler(_ *discordgo.Session, r *discordgo.GuildRoleUpdate) {
-	dbRole, _ := user.discordRoleToDB(r.GuildID, r.Role, nil)
-	dbRole.Upsert(nil)
-}
-
-func (user *User) guildRoleDeleteHandler(_ *discordgo.Session, r *discordgo.GuildRoleDelete) {
-	user.bridge.DB.Role.DeleteByID(r.GuildID, r.RoleID)
-}
-
 func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSpace bool) {
 	guild := user.bridge.GetGuildByID(meta.ID, true)
 	guild.UpdateInfo(user, meta)
@@ -842,6 +957,9 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp
 				}
 			} else {
 				portal.UpdateInfo(user, ch)
+				if user.bridge.Config.Bridge.Backfill.MaxGuildMembers < 0 || meta.MemberCount < user.bridge.Config.Bridge.Backfill.MaxGuildMembers {
+					portal.ForwardBackfillMissed(user, ch)
+				}
 			}
 		}
 	}
@@ -851,7 +969,7 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp
 	user.addGuildToSpace(guild, isInSpace, timestamp)
 }
 
-func (user *User) connectedHandler(_ *discordgo.Session, _ *discordgo.Connect) {
+func (user *User) connectedHandler(_ *discordgo.Connect) {
 	user.bridgeStateLock.Lock()
 	defer user.bridgeStateLock.Unlock()
 	user.log.Debug().Msg("Connected to Discord")
@@ -861,7 +979,7 @@ func (user *User) connectedHandler(_ *discordgo.Session, _ *discordgo.Connect) {
 	}
 }
 
-func (user *User) disconnectedHandler(_ *discordgo.Session, _ *discordgo.Disconnect) {
+func (user *User) disconnectedHandler(_ *discordgo.Disconnect) {
 	user.bridgeStateLock.Lock()
 	defer user.bridgeStateLock.Unlock()
 	if user.wasLoggedOut {
@@ -873,7 +991,7 @@ func (user *User) disconnectedHandler(_ *discordgo.Session, _ *discordgo.Disconn
 	user.BridgeState.Send(status.BridgeState{StateEvent: status.StateTransientDisconnect, Error: "dc-transient-disconnect", Message: "Temporarily disconnected from Discord, trying to reconnect"})
 }
 
-func (user *User) invalidAuthHandler(_ *discordgo.Session, _ *discordgo.InvalidAuth) {
+func (user *User) invalidAuthHandler(_ *discordgo.InvalidAuth) {
 	user.bridgeStateLock.Lock()
 	defer user.bridgeStateLock.Unlock()
 	user.log.Info().Msg("Got logged out from Discord due to invalid token")
@@ -882,7 +1000,7 @@ func (user *User) invalidAuthHandler(_ *discordgo.Session, _ *discordgo.InvalidA
 	go user.Logout(false)
 }
 
-func (user *User) guildCreateHandler(_ *discordgo.Session, g *discordgo.GuildCreate) {
+func (user *User) guildCreateHandler(g *discordgo.GuildCreate) {
 	user.log.Info().
 		Str("guild_id", g.ID).
 		Str("name", g.Name).
@@ -891,7 +1009,7 @@ func (user *User) guildCreateHandler(_ *discordgo.Session, g *discordgo.GuildCre
 	user.handleGuild(g.Guild, time.Now(), false)
 }
 
-func (user *User) guildDeleteHandler(_ *discordgo.Session, g *discordgo.GuildDelete) {
+func (user *User) guildDeleteHandler(g *discordgo.GuildDelete) {
 	user.log.Info().Str("guild_id", g.ID).Msg("Got guild delete event")
 	user.MarkNotInPortal(g.ID)
 	guild := user.bridge.GetGuildByID(g.ID, false)
@@ -907,12 +1025,12 @@ func (user *User) guildDeleteHandler(_ *discordgo.Session, g *discordgo.GuildDel
 	}
 }
 
-func (user *User) guildUpdateHandler(_ *discordgo.Session, g *discordgo.GuildUpdate) {
+func (user *User) guildUpdateHandler(g *discordgo.GuildUpdate) {
 	user.log.Debug().Str("guild_id", g.ID).Msg("Got guild update event")
 	user.handleGuild(g.Guild, time.Now(), user.IsInSpace(g.ID))
 }
 
-func (user *User) channelCreateHandler(_ *discordgo.Session, c *discordgo.ChannelCreate) {
+func (user *User) channelCreateHandler(c *discordgo.ChannelCreate) {
 	if user.getGuildBridgingMode(c.GuildID) < database.GuildBridgeEverything {
 		user.log.Debug().
 			Str("guild_id", c.GuildID).Str("channel_id", c.ID).
@@ -942,7 +1060,7 @@ func (user *User) channelCreateHandler(_ *discordgo.Session, c *discordgo.Channe
 	}
 }
 
-func (user *User) channelDeleteHandler(_ *discordgo.Session, c *discordgo.ChannelDelete) {
+func (user *User) channelDeleteHandler(c *discordgo.ChannelDelete) {
 	portal := user.GetExistingPortalByID(c.ID)
 	if portal == nil {
 		user.log.Debug().
@@ -963,11 +1081,7 @@ func (user *User) channelDeleteHandler(_ *discordgo.Session, c *discordgo.Channe
 		Msg("Completed cleaning up channel")
 }
 
-func (user *User) channelPinsUpdateHandler(_ *discordgo.Session, c *discordgo.ChannelPinsUpdate) {
+func (user *User) channelUpdateHandler(c *discordgo.ChannelUpdate) {
-	user.log.Debug().Msg("channel pins update")
-}
-
-func (user *User) channelUpdateHandler(_ *discordgo.Session, c *discordgo.ChannelUpdate) {
 	portal := user.GetPortalByMeta(c.Channel)
 	if c.GuildID == "" {
 		user.handlePrivateChannel(portal, c.Channel, time.Now(), true, user.IsInSpace(portal.Key.String()))
@@ -976,6 +1090,20 @@ func (user *User) channelUpdateHandler(_ *discordgo.Session, c *discordgo.Channe
 	}
 }
 
+func (user *User) channelRecipientAdd(c *discordgo.ChannelRecipientAdd) {
+	portal := user.GetExistingPortalByID(c.ChannelID)
+	if portal != nil {
+		portal.syncParticipant(user, c.User, false)
+	}
+}
+
+func (user *User) channelRecipientRemove(c *discordgo.ChannelRecipientRemove) {
+	portal := user.GetExistingPortalByID(c.ChannelID)
+	if portal != nil {
+		portal.syncParticipant(user, c.User, true)
+	}
+}
+
 func (user *User) findPortal(channelID string) (*Portal, *Thread) {
 	portal := user.GetExistingPortalByID(channelID)
 	if portal != nil {
@@ -1040,26 +1168,6 @@ func (user *User) pushPortalMessage(msg interface{}, typeName, channelID, guildI
 	}
 }
 
-func (user *User) messageCreateHandler(_ *discordgo.Session, m *discordgo.MessageCreate) {
-	user.pushPortalMessage(m, "message create", m.ChannelID, m.GuildID)
-}
-
-func (user *User) messageDeleteHandler(_ *discordgo.Session, m *discordgo.MessageDelete) {
-	user.pushPortalMessage(m, "message delete", m.ChannelID, m.GuildID)
-}
-
-func (user *User) messageUpdateHandler(_ *discordgo.Session, m *discordgo.MessageUpdate) {
-	user.pushPortalMessage(m, "message update", m.ChannelID, m.GuildID)
-}
-
-func (user *User) reactionAddHandler(_ *discordgo.Session, m *discordgo.MessageReactionAdd) {
-	user.pushPortalMessage(m, "reaction add", m.ChannelID, m.GuildID)
-}
-
-func (user *User) reactionRemoveHandler(_ *discordgo.Session, m *discordgo.MessageReactionRemove) {
-	user.pushPortalMessage(m, "reaction remove", m.ChannelID, m.GuildID)
-}
-
 type CustomReadReceipt struct {
 	Timestamp          int64  `json:"ts,omitempty"`
 	DoublePuppetSource string `json:"fi.mau.double_puppet_source,omitempty"`
@@ -1084,7 +1192,7 @@ func (user *User) makeReadMarkerContent(eventID id.EventID) *CustomReadMarkers {
 	}
 }
 
-func (user *User) messageAckHandler(_ *discordgo.Session, m *discordgo.MessageAck) {
+func (user *User) messageAckHandler(m *discordgo.MessageAck) {
 	portal := user.GetExistingPortalByID(m.ChannelID)
 	if portal == nil || portal.MXID == "" {
 		return
@@ -1117,7 +1225,7 @@ func (user *User) messageAckHandler(_ *discordgo.Session, m *discordgo.MessageAc
 	}
 }
 
-func (user *User) typingStartHandler(_ *discordgo.Session, t *discordgo.TypingStart) {
+func (user *User) typingStartHandler(t *discordgo.TypingStart) {
 	portal := user.GetExistingPortalByID(t.ChannelID)
 	if portal == nil || portal.MXID == "" {
 		return
@@ -1125,7 +1233,7 @@ func (user *User) typingStartHandler(_ *discordgo.Session, t *discordgo.TypingSt
 	portal.handleDiscordTyping(t)
 }
 
-func (user *User) interactionSuccessHandler(_ *discordgo.Session, s *discordgo.InteractionSuccess) {
+func (user *User) interactionSuccessHandler(s *discordgo.InteractionSuccess) {
 	user.pendingInteractionsLock.Lock()
 	defer user.pendingInteractionsLock.Unlock()
 	ce, ok := user.pendingInteractions[s.Nonce]
@@ -1138,10 +1246,16 @@ func (user *User) interactionSuccessHandler(_ *discordgo.Session, s *discordgo.I
 	}
 }
 
-func (user *User) ensureInvited(intent *appservice.IntentAPI, roomID id.RoomID, isDirect bool) bool {
+func (user *User) ensureInvited(intent *appservice.IntentAPI, roomID id.RoomID, isDirect, ignoreCache bool) bool {
+	if roomID == "" {
+		return false
+	}
 	if intent == nil {
 		intent = user.bridge.Bot
 	}
+	if !ignoreCache && intent.StateStore.IsInvited(roomID, user.MXID) {
+		return true
+	}
 	ret := false
 
 	inviteContent := event.Content{