Bump version to v0.3.0

Signed-off-by: Sumner Evans <sumner@beeper.com>

CHANGELOG.md

@@ -1,3 +1,10 @@
+# v0.3.0 (2023-04-16)
+
+* Added support for backfilling on room creation and missed messages on startup.
+* Added options to automatically ratchet/delete megolm sessions to minimize
+  access to old messages.
+* Added basic support for incoming voice messages.
+
 # v0.2.0 (2023-03-16)
 
 * Switched to zerolog for logging.

backfill.go

@@ -0,0 +1,303 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"sort"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/rs/zerolog"
+	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/bridge/bridgeconfig"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+
+	"go.mau.fi/mautrix-discord/database"
+)
+
+func (portal *Portal) forwardBackfillInitial(source *User) {
+	defer portal.forwardBackfillLock.Unlock()
+	// This should only be called from CreateMatrixRoom which locks forwardBackfillLock before creating the room.
+	if portal.forwardBackfillLock.TryLock() {
+		panic("forwardBackfillInitial() called without locking forwardBackfillLock")
+	}
+
+	limit := portal.bridge.Config.Bridge.Backfill.Limits.Initial.Channel
+	if portal.GuildID == "" {
+		limit = portal.bridge.Config.Bridge.Backfill.Limits.Initial.DM
+	}
+	if limit == 0 {
+		return
+	}
+
+	log := portal.zlog.With().
+		Str("action", "initial backfill").
+		Str("room_id", portal.MXID.String()).
+		Int("limit", limit).
+		Logger()
+
+	portal.backfillLimited(log, source, limit, "")
+}
+
+func (portal *Portal) ForwardBackfillMissed(source *User, meta *discordgo.Channel) {
+	if portal.MXID == "" {
+		return
+	}
+
+	limit := portal.bridge.Config.Bridge.Backfill.Limits.Missed.Channel
+	if portal.GuildID == "" {
+		limit = portal.bridge.Config.Bridge.Backfill.Limits.Missed.DM
+	}
+	if limit == 0 {
+		return
+	}
+	log := portal.zlog.With().
+		Str("action", "missed event backfill").
+		Str("room_id", portal.MXID.String()).
+		Int("limit", limit).
+		Logger()
+
+	portal.forwardBackfillLock.Lock()
+	defer portal.forwardBackfillLock.Unlock()
+
+	lastMessage := portal.bridge.DB.Message.GetLast(portal.Key)
+	if lastMessage == nil || meta.LastMessageID == "" {
+		log.Debug().Msg("Not backfilling, no last message in database or no last message in metadata")
+		return
+	} else if !shouldBackfill(lastMessage.DiscordID, meta.LastMessageID) {
+		log.Debug().
+			Str("last_bridged_message", lastMessage.DiscordID).
+			Str("last_server_message", meta.LastMessageID).
+			Msg("Not backfilling, last message in database is newer than last message in metadata")
+		return
+	}
+	log.Debug().
+		Str("last_bridged_message", lastMessage.DiscordID).
+		Str("last_server_message", meta.LastMessageID).
+		Msg("Backfilling missed messages")
+	if limit < 0 {
+		portal.backfillUnlimitedMissed(log, source, lastMessage.DiscordID)
+	} else {
+		portal.backfillLimited(log, source, limit, lastMessage.DiscordID)
+	}
+}
+
+const messageFetchChunkSize = 50
+
+func (portal *Portal) collectBackfillMessages(log zerolog.Logger, source *User, limit int, until string) ([]*discordgo.Message, bool, error) {
+	var messages []*discordgo.Message
+	var before string
+	var foundAll bool
+	for {
+		log.Debug().Str("before_id", before).Msg("Fetching messages for backfill")
+		newMessages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, before, "", "")
+		if err != nil {
+			return nil, false, err
+		}
+		if until != "" {
+			for i, msg := range newMessages {
+				if compareMessageIDs(msg.ID, until) <= 0 {
+					log.Debug().
+						Str("message_id", msg.ID).
+						Str("until_id", until).
+						Msg("Found message that was already bridged")
+					newMessages = newMessages[:i]
+					foundAll = true
+					break
+				}
+			}
+		}
+		messages = append(messages, newMessages...)
+		log.Debug().Int("count", len(newMessages)).Msg("Added messages to backfill collection")
+		if len(newMessages) <= messageFetchChunkSize || len(messages) >= limit {
+			break
+		}
+		before = newMessages[len(newMessages)-1].ID
+	}
+	if len(messages) > limit {
+		foundAll = false
+		messages = messages[:limit]
+	}
+	return messages, foundAll, nil
+}
+
+func (portal *Portal) backfillLimited(log zerolog.Logger, source *User, limit int, after string) {
+	messages, foundAll, err := portal.collectBackfillMessages(log, source, limit, after)
+	if err != nil {
+		log.Err(err).Msg("Error collecting messages to forward backfill")
+		return
+	}
+	log.Info().
+		Int("count", len(messages)).
+		Bool("found_all", foundAll).
+		Msg("Collected messages to backfill")
+	sort.Sort(MessageSlice(messages))
+	if !foundAll {
+		_, err = portal.sendMatrixMessage(portal.MainIntent(), event.EventMessage, &event.MessageEventContent{
+			MsgType: event.MsgNotice,
+			Body:    "Some messages may have been missed here while the bridge was offline.",
+		}, nil, 0)
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to send missed message warning")
+		} else {
+			log.Debug().Msg("Sent warning about possibly missed messages")
+		}
+	}
+	portal.sendBackfillBatch(log, source, messages)
+}
+
+func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User, after string) {
+	for {
+		log.Debug().Str("after_id", after).Msg("Fetching chunk of messages to backfill")
+		messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, "", after, "")
+		if err != nil {
+			log.Err(err).Msg("Error fetching chunk of messages to forward backfill")
+			return
+		}
+		log.Debug().Int("count", len(messages)).Msg("Fetched chunk of messages to backfill")
+		sort.Sort(MessageSlice(messages))
+
+		portal.sendBackfillBatch(log, source, messages)
+
+		if len(messages) < messageFetchChunkSize {
+			// Assume that was all the missing messages
+			log.Debug().Msg("Chunk had less than 50 messages, stopping backfill")
+			return
+		}
+		after = messages[len(messages)-1].ID
+	}
+}
+
+func (portal *Portal) sendBackfillBatch(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+	if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry {
+		log.Debug().Msg("Using hungryserv, sending messages with batch send endpoint")
+		portal.forwardBatchSend(log, source, messages)
+	} else {
+		log.Debug().Msg("Not using hungryserv, sending messages one by one")
+		for _, msg := range messages {
+			portal.handleDiscordMessageCreate(source, msg, nil)
+		}
+	}
+}
+
+func (portal *Portal) forwardBatchSend(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+	evts := make([]*event.Event, 0, len(messages))
+	dbMessages := make([]database.Message, 0, len(messages))
+	for _, msg := range messages {
+		for _, mention := range msg.Mentions {
+			puppet := portal.bridge.GetPuppetByID(mention.ID)
+			puppet.UpdateInfo(nil, mention)
+		}
+
+		puppet := portal.bridge.GetPuppetByID(msg.Author.ID)
+		puppet.UpdateInfo(source, msg.Author)
+		intent := puppet.IntentFor(portal)
+		replyTo := portal.getReplyTarget(source, msg.MessageReference, true)
+
+		ts, _ := discordgo.SnowflakeTimestamp(msg.ID)
+		parts := portal.convertDiscordMessage(intent, msg)
+		for i, part := range parts {
+			if replyTo != nil {
+				part.Content.RelatesTo = &event.RelatesTo{InReplyTo: replyTo}
+				// Only set reply for first event
+				replyTo = nil
+			}
+			partName := part.AttachmentID
+			// Always use blank part name for first part so that replies and other things
+			// can reference it without knowing about attachments.
+			if i == 0 {
+				partName = ""
+			}
+			evt := &event.Event{
+				ID:        portal.deterministicEventID(msg.ID, partName),
+				Type:      part.Type,
+				Sender:    intent.UserID,
+				Timestamp: ts.UnixMilli(),
+				Content: event.Content{
+					Parsed: part.Content,
+					Raw:    part.Extra,
+				},
+			}
+			var err error
+			evt.Type, err = portal.encrypt(intent, &evt.Content, evt.Type)
+			if err != nil {
+				log.Err(err).Msg("Failed to encrypt event")
+				continue
+			}
+			intent.AddDoublePuppetValue(&evt.Content)
+			evts = append(evts, evt)
+			dbMessages = append(dbMessages, database.Message{
+				Channel:      portal.Key,
+				DiscordID:    msg.ID,
+				SenderID:     msg.Author.ID,
+				Timestamp:    ts,
+				AttachmentID: part.AttachmentID,
+			})
+		}
+	}
+	if len(evts) == 0 {
+		log.Warn().Msg("Didn't get any events to backfill")
+		return
+	}
+	log.Info().Int("events", len(evts)).Msg("Converted messages to backfill")
+	resp, err := portal.MainIntent().BatchSend(portal.MXID, &mautrix.ReqBatchSend{
+		BeeperNewMessages: true,
+		Events:            evts,
+	})
+	if err != nil {
+		log.Err(err).Msg("Error sending backfill batch")
+		return
+	}
+	for i, evtID := range resp.EventIDs {
+		dbMessages[i].MXID = evtID
+	}
+	portal.bridge.DB.Message.MassInsert(portal.Key, dbMessages)
+	log.Info().Msg("Inserted backfilled batch to database")
+}
+
+func (portal *Portal) deterministicEventID(messageID, partName string) id.EventID {
+	data := fmt.Sprintf("%s/discord/%s/%s", portal.MXID, messageID, partName)
+	sum := sha256.Sum256([]byte(data))
+	return id.EventID(fmt.Sprintf("$%s:discord.com", base64.RawURLEncoding.EncodeToString(sum[:])))
+}
+
+// compareMessageIDs compares two Discord message IDs.
+//
+// If the first ID is lower, -1 is returned.
+// If the second ID is lower, 1 is returned.
+// If the IDs are equal, 0 is returned.
+func compareMessageIDs(id1, id2 string) int {
+	if id1 == id2 {
+		return 0
+	}
+	if len(id1) < len(id2) {
+		return -1
+	} else if len(id2) < len(id1) {
+		return 1
+	}
+	if id1 < id2 {
+		return -1
+	}
+	return 1
+}
+
+func shouldBackfill(latestBridgedIDStr, latestIDFromServerStr string) bool {
+	return compareMessageIDs(latestBridgedIDStr, latestIDFromServerStr) == -1
+}
+
+type MessageSlice []*discordgo.Message
+
+var _ sort.Interface = (MessageSlice)(nil)
+
+func (a MessageSlice) Len() int {
+	return len(a)
+}
+
+func (a MessageSlice) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a MessageSlice) Less(i, j int) bool {
+	return compareMessageIDs(a[i].ID, a[j].ID) == -1
+}

config/bridge.go

@@ -32,7 +32,7 @@ type BridgeConfig struct {
 	DisplaynameTemplate       string `yaml:"displayname_template"`
 	ChannelNameTemplate       string `yaml:"channel_name_template"`
 	GuildNameTemplate         string `yaml:"guild_name_template"`
-	PrivateChatPortalMeta     bool   `yaml:"private_chat_portal_meta"`
+	PrivateChatPortalMeta     string `yaml:"private_chat_portal_meta"`
 	PrivateChannelCreateLimit int    `yaml:"startup_private_channel_create_limit"`
 
 	PortalMessageBuffer int `yaml:"portal_message_buffer"`
@@ -66,6 +66,14 @@ type BridgeConfig struct {
 	CommandPrefix      string                           `yaml:"command_prefix"`
 	ManagementRoomText bridgeconfig.ManagementRoomTexts `yaml:"management_room_text"`
 
+	Backfill struct {
+		Limits struct {
+			Initial BackfillLimitPart `yaml:"initial"`
+			Missed  BackfillLimitPart `yaml:"missed"`
+		} `yaml:"forward_limits"`
+		MaxGuildMembers int `yaml:"max_guild_members"`
+	} `yaml:"backfill"`
+
 	Encryption bridgeconfig.EncryptionConfig `yaml:"encryption"`
 
 	Provisioning struct {
@@ -81,6 +89,11 @@ type BridgeConfig struct {
 	guildNameTemplate   *template.Template `yaml:"-"`
 }
 
+type BackfillLimitPart struct {
+	DM      int `yaml:"dm"`
+	Channel int `yaml:"channel"`
+}
+
 func (bc *BridgeConfig) GetResendBridgeInfo() bool {
 	return bc.ResendBridgeInfo
 }

config/upgrade.go

@@ -1,5 +1,5 @@
 // mautrix-discord - A Matrix-Discord puppeting bridge.
-// Copyright (C) 2022 Tulir Asokan
+// Copyright (C) 2023 Tulir Asokan
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -31,7 +31,15 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Str, "bridge", "displayname_template")
 	helper.Copy(up.Str, "bridge", "channel_name_template")
 	helper.Copy(up.Str, "bridge", "guild_name_template")
-	helper.Copy(up.Bool, "bridge", "private_chat_portal_meta")
+	if legacyPrivateChatPortalMeta, ok := helper.Get(up.Bool, "bridge", "private_chat_portal_meta"); ok {
+		updatedPrivateChatPortalMeta := "default"
+		if legacyPrivateChatPortalMeta == "true" {
+			updatedPrivateChatPortalMeta = "always"
+		}
+		helper.Set(up.Str, updatedPrivateChatPortalMeta, "bridge", "private_chat_portal_meta")
+	} else {
+		helper.Copy(up.Str, "bridge", "private_chat_portal_meta")
+	}
 	helper.Copy(up.Int, "bridge", "startup_private_channel_create_limit")
 	helper.Copy(up.Int, "bridge", "portal_message_buffer")
 	helper.Copy(up.Bool, "bridge", "delivery_receipts")
@@ -59,11 +67,24 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Str, "bridge", "management_room_text", "welcome_connected")
 	helper.Copy(up.Str, "bridge", "management_room_text", "welcome_unconnected")
 	helper.Copy(up.Str|up.Null, "bridge", "management_room_text", "additional_help")
+	helper.Copy(up.Bool, "bridge", "backfill", "enabled")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "dm")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "dm")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "max_guild_members")
 	helper.Copy(up.Bool, "bridge", "encryption", "allow")
 	helper.Copy(up.Bool, "bridge", "encryption", "default")
 	helper.Copy(up.Bool, "bridge", "encryption", "require")
 	helper.Copy(up.Bool, "bridge", "encryption", "appservice")
 	helper.Copy(up.Bool, "bridge", "encryption", "allow_key_sharing")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_outbound_on_ack")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "dont_store_outbound")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "ratchet_on_decrypt")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_fully_used_on_decrypt")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_prev_on_new_session")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_on_device_delete")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "periodically_delete_expired")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "receive")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "send")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "share")

database/message.go

@@ -70,6 +70,11 @@ func (mq *MessageQuery) GetLastInThread(key PortalKey, threadID string) *Message
 	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver, threadID))
 }
 
+func (mq *MessageQuery) GetLast(key PortalKey) *Message {
+	query := messageSelect + " WHERE dc_chan_id=$1 AND dc_chan_receiver=$2 AND dc_edit_index=0 ORDER BY timestamp DESC LIMIT 1"
+	return mq.New().Scan(mq.db.QueryRow(query, key.ChannelID, key.Receiver))
+}
+
 func (mq *MessageQuery) DeleteAll(key PortalKey) {
 	query := "DELETE FROM message WHERE dc_chan_id=$1 AND dc_chan_receiver=$2"
 	_, err := mq.db.Exec(query, key.ChannelID, key.Receiver)
@@ -90,6 +95,36 @@ func (mq *MessageQuery) GetByMXID(key PortalKey, mxid id.EventID) *Message {
 	return mq.New().Scan(row)
 }
 
+func (mq *MessageQuery) MassInsert(key PortalKey, msgs []Message) {
+	if len(msgs) == 0 {
+		return
+	}
+	valueStringFormat := "($%d, $%d, $%d, $1, $2, $%d, $%d, $%d, $%d)"
+	if mq.db.Dialect == dbutil.SQLite {
+		valueStringFormat = strings.ReplaceAll(valueStringFormat, "$", "?")
+	}
+	params := make([]interface{}, 2+len(msgs)*7)
+	placeholders := make([]string, len(msgs))
+	params[0] = key.ChannelID
+	params[1] = key.Receiver
+	for i, msg := range msgs {
+		baseIndex := 2 + i*7
+		params[baseIndex] = msg.DiscordID
+		params[baseIndex+1] = msg.AttachmentID
+		params[baseIndex+2] = msg.EditIndex
+		params[baseIndex+3] = msg.SenderID
+		params[baseIndex+4] = msg.Timestamp.UnixMilli()
+		params[baseIndex+5] = msg.ThreadID
+		params[baseIndex+6] = msg.MXID
+		placeholders[i] = fmt.Sprintf(valueStringFormat, baseIndex+1, baseIndex+2, baseIndex+3, baseIndex+4, baseIndex+5, baseIndex+6, baseIndex+7)
+	}
+	_, err := mq.db.Exec(fmt.Sprintf(messageMassInsertTemplate, strings.Join(placeholders, ", ")), params...)
+	if err != nil {
+		mq.log.Warnfln("Failed to insert %d messages: %v", len(msgs), err)
+		panic(err)
+	}
+}
+
 type Message struct {
 	db  *Database
 	log log.Logger
@@ -147,7 +182,7 @@ type MessagePart struct {
 	MXID         id.EventID
 }
 
-func (m *Message) MassInsert(msgs []MessagePart) {
+func (m *Message) MassInsertParts(msgs []MessagePart) {
 	if len(msgs) == 0 {
 		return
 	}

example-config.yaml

@@ -97,9 +97,11 @@ bridge:
     # Available variables:
     #   .Name - Guild name
     guild_name_template: '{{.Name}}'
-    # Should the bridge explicitly set the avatar and room name for DM portal rooms?
+    # Whether to explicitly set the avatar and room name for private chat portal rooms.
-    # This is implicitly enabled in encrypted rooms.
+    # If set to `default`, this will be enabled in encrypted rooms and disabled in unencrypted rooms.
-    private_chat_portal_meta: false
+    # If set to `always`, all DM rooms will have explicit names and avatars set.
+    # If set to `never`, DM rooms will never have names and avatars set.
+    private_chat_portal_meta: default
 
     portal_message_buffer: 128
 
@@ -184,6 +186,28 @@ bridge:
         # Optional extra text sent when joining a management room.
         additional_help: ""
 
+    # Settings for backfilling messages.
+    backfill:
+        # Limits for forward backfilling.
+        forward_limits:
+            # Initial backfill (when creating portal). 0 means backfill is disabled.
+            # A special unlimited value is not supported, you must set a limit. Initial backfill will
+            # fetch all messages first before backfilling anything, so high limits can take a lot of time.
+            initial:
+                dm: 0
+                channel: 0
+            # Missed message backfill (on startup).
+            # 0 means backfill is disabled, -1 means fetch all messages since last bridged message.
+            # When using unlimited backfill (-1), messages are backfilled as they are fetched.
+            # With limits, all messages up to the limit are fetched first and backfilled afterwards.
+            missed:
+                dm: 0
+                channel: 0
+        # Maximum members in a guild to enable backfilling. Set to -1 to disable limit.
+        # This can be used as a rough heuristic to disable backfilling in channels that are too active.
+        # Currently only applies to missed message backfill.
+        max_guild_members: -1
+
     # End-to-bridge encryption support options.
     #
     # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
@@ -200,6 +224,23 @@ bridge:
         # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
         # You must use a client that supports requesting keys from other users to use this feature.
         allow_key_sharing: false
+        # Options for deleting megolm sessions from the bridge.
+        delete_keys:
+            # Beeper-specific: delete outbound sessions when hungryserv confirms
+            # that the user has uploaded the key to key backup.
+            delete_outbound_on_ack: false
+            # Don't store outbound sessions in the inbound table.
+            dont_store_outbound: false
+            # Ratchet megolm sessions forward after decrypting messages.
+            ratchet_on_decrypt: false
+            # Delete fully used keys (index >= max_messages) after decrypting messages.
+            delete_fully_used_on_decrypt: false
+            # Delete previous megolm sessions from same device when receiving a new one.
+            delete_prev_on_new_session: false
+            # Delete megolm sessions received from a device when the device is deleted.
+            delete_on_device_delete: false
+            # Periodically delete megolm sessions when 2x max_age has passed since receiving the session.
+            periodically_delete_expired: false
         # What level of device verification should be required from users?
         #
         # Valid levels:

go.mod

@@ -8,18 +8,18 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
-	github.com/lib/pq v1.10.7
+	github.com/lib/pq v1.10.8
 	github.com/mattn/go-sqlite3 v1.14.16
-	github.com/rs/zerolog v1.29.0
+	github.com/rs/zerolog v1.29.1
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.2
 	github.com/yuin/goldmark v1.5.4
 	maunium.net/go/maulogger/v2 v2.4.1
-	maunium.net/go/mautrix v0.15.0
+	maunium.net/go/mautrix v0.15.1
 )
 
 require (
-	github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
@@ -29,12 +29,12 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	go.mau.fi/zeroconfig v0.1.2 // indirect
-	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/crypto v0.8.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/net v0.9.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	maunium.net/go/mauflag v1.0.0 // indirect
 )
 
-replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d
+replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20230416132336-325ee6a8c961

go.sum

@@ -1,8 +1,8 @@
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d h1:xo6A9gSSu7mnxIXHBD1EPDyKEQFlI0N8r57Yf0gWiy8=
+github.com/beeper/discordgo v0.0.0-20230416132336-325ee6a8c961 h1:eSGaliexlehYBeP4YQW8dQpV9XWWgfR1qH8kfHgrDcY=
-github.com/beeper/discordgo v0.0.0-20230301201402-cf4c62e5f53d/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
+github.com/beeper/discordgo v0.0.0-20230416132336-325ee6a8c961/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 h1:rtAn27wIbmOGUs7RIbVgPEjb31ehTVniDwPGXyMxm5U=
+github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
-github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -16,8 +16,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE=
-github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
@@ -28,8 +28,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
+github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
-github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -53,16 +53,16 @@ github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5ta
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -77,5 +77,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
-maunium.net/go/mautrix v0.15.0 h1:gkK9HXc1SSPwY7qOAqchzj2xxYqiOYeee8lr28A2g/o=
+maunium.net/go/mautrix v0.15.1 h1:pmCtMjYRpd83+2UL+KTRFYQo5to0373yulimvLK+1k0=
-maunium.net/go/mautrix v0.15.0/go.mod h1:1v8QVDd7q/eJ+eg4sgeOSEafBAFhkt4ab2i97M3IkNQ=
+maunium.net/go/mautrix v0.15.1/go.mod h1:icQIrvz2NldkRLTuzSGzmaeuMUmw+fzO7UVycPeauN8=

main.go

@@ -187,11 +187,13 @@ func main() {
 		attachmentTransfers: util.NewSyncMap[attachmentKey, *util.ReturnableOnce[*database.File]](),
 	}
 	br.Bridge = bridge.Bridge{
-		Name:         "mautrix-discord",
+		Name:              "mautrix-discord",
-		URL:          "https://github.com/mautrix/discord",
+		URL:               "https://github.com/mautrix/discord",
-		Description:  "A Matrix-Discord puppeting bridge.",
+		Description:       "A Matrix-Discord puppeting bridge.",
-		Version:      "0.2.0",
+		Version:           "0.3.0",
-		ProtocolName: "Discord",
+		ProtocolName:      "Discord",
+		BeeperServiceName: "discordgo",
+		BeeperNetworkName: "discord",
 
 		CryptoPickleKey: "maunium.net/go/mautrix-whatsapp",
 

portal.go

@@ -12,8 +12,10 @@ import (
 
 	"github.com/bwmarrin/discordgo"
 	"github.com/gabriel-vasile/mimetype"
+	"github.com/rs/zerolog"
+	"maunium.net/go/maulogger/v2/maulogadapt"
 
-	log "maunium.net/go/maulogger/v2"
+	"maunium.net/go/maulogger/v2"
 
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
@@ -51,7 +53,9 @@ type Portal struct {
 	Guild  *Guild
 
 	bridge *DiscordBridge
-	log    log.Logger
+	// Deprecated
+	log  maulogger.Logger
+	zlog zerolog.Logger
 
 	roomCreateLock sync.Mutex
 	encryptLock    sync.Mutex
@@ -64,6 +68,8 @@ type Portal struct {
 	commands     map[string]*discordgo.ApplicationCommand
 	commandsLock sync.RWMutex
 
+	forwardBackfillLock sync.Mutex
+
 	currentlyTyping     []id.UserID
 	currentlyTypingLock sync.Mutex
 }
@@ -232,7 +238,10 @@ func (br *DiscordBridge) NewPortal(dbPortal *database.Portal) *Portal {
 	portal := &Portal{
 		Portal: dbPortal,
 		bridge: br,
-		log:    br.Log.Sub(fmt.Sprintf("Portal/%s", dbPortal.Key)),
+		zlog: br.ZLog.With().
+			Str("channel_id", dbPortal.Key.ChannelID).
+			Str("channel_receiver", dbPortal.Key.Receiver).
+			Logger(),
 
 		discordMessages: make(chan portalDiscordMessage, br.Config.Bridge.PortalMessageBuffer),
 		matrixMessages:  make(chan portalMatrixMessage, br.Config.Bridge.PortalMessageBuffer),
@@ -241,6 +250,7 @@ func (br *DiscordBridge) NewPortal(dbPortal *database.Portal) *Portal {
 
 		commands: make(map[string]*discordgo.ApplicationCommand),
 	}
+	portal.log = maulogadapt.ZeroAsMau(&portal.zlog)
 
 	go portal.messageLoop()
 
@@ -336,6 +346,12 @@ func (portal *Portal) UpdateBridgeInfo() {
 	}
 }
 
+func (portal *Portal) shouldSetDMRoomMetadata() bool {
+	return !portal.IsPrivateChat() ||
+		portal.bridge.Config.Bridge.PrivateChatPortalMeta == "always" ||
+		(portal.IsEncrypted() && portal.bridge.Config.Bridge.PrivateChatPortalMeta != "never")
+}
+
 func (portal *Portal) GetEncryptionEventContent() (evt *event.EncryptionEventContent) {
 	evt = &event.EncryptionEventContent{Algorithm: id.AlgorithmMegolmV1}
 	if rot := portal.bridge.Config.Bridge.Encryption.Rotation; rot.EnableCustom {
@@ -375,13 +391,32 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 		StateKey: &bridgeInfoStateKey,
 	}}
 
-	if !portal.AvatarURL.IsEmpty() {
+	var invite []id.UserID
+
+	if portal.bridge.Config.Bridge.Encryption.Default {
+		initialState = append(initialState, &event.Event{
+			Type: event.StateEncryption,
+			Content: event.Content{
+				Parsed: portal.GetEncryptionEventContent(),
+			},
+		})
+		portal.Encrypted = true
+
+		if portal.IsPrivateChat() {
+			invite = append(invite, portal.bridge.Bot.UserID)
+		}
+	}
+
+	if !portal.AvatarURL.IsEmpty() && portal.shouldSetDMRoomMetadata() {
 		initialState = append(initialState, &event.Event{
 			Type: event.StateRoomAvatar,
 			Content: event.Content{Parsed: &event.RoomAvatarEventContent{
 				URL: portal.AvatarURL,
 			}},
 		})
+		portal.AvatarSet = true
+	} else {
+		portal.AvatarSet = false
 	}
 
 	creationContent := make(map[string]interface{})
@@ -417,23 +452,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 		})
 	}
 
-	var invite []id.UserID
+	req := &mautrix.ReqCreateRoom{
-
-	if portal.bridge.Config.Bridge.Encryption.Default {
-		initialState = append(initialState, &event.Event{
-			Type: event.StateEncryption,
-			Content: event.Content{
-				Parsed: portal.GetEncryptionEventContent(),
-			},
-		})
-		portal.Encrypted = true
-
-		if portal.IsPrivateChat() {
-			invite = append(invite, portal.bridge.Bot.UserID)
-		}
-	}
-
-	resp, err := intent.CreateRoom(&mautrix.ReqCreateRoom{
 		Visibility:      "private",
 		Name:            portal.Name,
 		Topic:           portal.Topic,
@@ -442,15 +461,28 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 		IsDirect:        portal.IsPrivateChat(),
 		InitialState:    initialState,
 		CreationContent: creationContent,
-	})
+	}
+	if !portal.shouldSetDMRoomMetadata() {
+		req.Name = ""
+	}
+
+	var backfillStarted bool
+	portal.forwardBackfillLock.Lock()
+	defer func() {
+		if !backfillStarted {
+			portal.log.Debugln("Backfill wasn't started, unlocking forward backfill lock")
+			portal.forwardBackfillLock.Unlock()
+		}
+	}()
+
+	resp, err := intent.CreateRoom(req)
 	if err != nil {
 		portal.log.Warnln("Failed to create room:", err)
 		return err
 	}
 
-	portal.NameSet = true
+	portal.NameSet = len(req.Name) > 0
-	portal.TopicSet = true
+	portal.TopicSet = len(req.Topic) > 0
-	portal.AvatarSet = !portal.AvatarURL.IsEmpty()
 	portal.MXID = resp.RoomID
 	portal.bridge.portalsLock.Lock()
 	portal.bridge.portalsByMXID[portal.MXID] = portal
@@ -490,6 +522,9 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 		portal.Update()
 	}
 
+	go portal.forwardBackfillInitial(user)
+	backfillStarted = true
+
 	return nil
 }
 
@@ -507,6 +542,8 @@ func (portal *Portal) handleDiscordMessages(msg portalDiscordMessage) {
 			return
 		}
 	}
+	portal.forwardBackfillLock.Lock()
+	defer portal.forwardBackfillLock.Unlock()
 
 	switch convertedMsg := msg.msg.(type) {
 	case *discordgo.MessageCreate:
@@ -536,7 +573,7 @@ func (portal *Portal) markMessageHandled(discordID string, editIndex int, author
 	msg.SenderID = authorID
 	msg.Timestamp = timestamp
 	msg.ThreadID = threadID
-	msg.MassInsert(parts)
+	msg.MassInsertParts(parts)
 }
 
 func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Message, thread *Thread) {
@@ -565,7 +602,7 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 	intent := puppet.IntentFor(portal)
 
 	var discordThreadID string
-	var threadRootEvent, lastThreadEvent, replyToEvent id.EventID
+	var threadRootEvent, lastThreadEvent id.EventID
 	if thread != nil {
 		discordThreadID = thread.ID
 		threadRootEvent = thread.RootMXID
@@ -575,30 +612,25 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 			lastThreadEvent = lastInThread.MXID
 		}
 	}
-
+	replyTo := portal.getReplyTarget(user, msg.MessageReference, false)
-	if msg.MessageReference != nil {
-		// This could be used to find cross-channel replies, but Matrix doesn't support those currently.
-		//key := database.PortalKey{msg.MessageReference.ChannelID, user.ID}
-		replyToMsg := portal.bridge.DB.Message.GetByDiscordID(portal.Key, msg.MessageReference.MessageID)
-		if len(replyToMsg) > 0 {
-			replyToEvent = replyToMsg[0].MXID
-		}
-	}
 
 	ts, _ := discordgo.SnowflakeTimestamp(msg.ID)
 	parts := portal.convertDiscordMessage(intent, msg)
 	dbParts := make([]database.MessagePart, 0, len(parts))
 	for i, part := range parts {
-		if (replyToEvent != "" || threadRootEvent != "") && part.Content.RelatesTo == nil {
+		if (replyTo != nil || threadRootEvent != "") && part.Content.RelatesTo == nil {
 			part.Content.RelatesTo = &event.RelatesTo{}
 		}
 		if threadRootEvent != "" {
 			part.Content.RelatesTo.SetThread(threadRootEvent, lastThreadEvent)
 		}
-		if replyToEvent != "" {
+		if replyTo != nil {
-			part.Content.RelatesTo.SetReplyTo(replyToEvent)
+			part.Content.RelatesTo.SetReplyTo(replyTo.EventID)
+			if replyTo.UnstableRoomID != "" {
+				part.Content.RelatesTo.InReplyTo.UnstableRoomID = replyTo.UnstableRoomID
+			}
 			// Only set reply for first event
-			replyToEvent = ""
+			replyTo = nil
 		}
 		resp, err := portal.sendMatrixMessage(intent, part.Type, part.Content, part.Extra, ts.UnixMilli())
 		if err != nil {
@@ -617,6 +649,42 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 	}
 }
 
+func (portal *Portal) getReplyTarget(source *User, ref *discordgo.MessageReference, allowNonExistent bool) *event.InReplyTo {
+	if ref == nil {
+		return nil
+	}
+	isHungry := portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry
+	if !isHungry {
+		allowNonExistent = false
+	}
+	// TODO add config option for cross-room replies
+	crossRoomReplies := isHungry
+
+	targetPortal := portal
+	if ref.ChannelID != portal.Key.ChannelID && crossRoomReplies {
+		targetPortal = portal.bridge.GetExistingPortalByID(database.PortalKey{ChannelID: ref.ChannelID, Receiver: source.DiscordID})
+		if targetPortal == nil {
+			return nil
+		}
+	}
+	replyToMsg := portal.bridge.DB.Message.GetByDiscordID(targetPortal.Key, ref.MessageID)
+	if len(replyToMsg) > 0 {
+		if !crossRoomReplies {
+			return &event.InReplyTo{EventID: replyToMsg[0].MXID}
+		}
+		return &event.InReplyTo{
+			EventID:        replyToMsg[0].MXID,
+			UnstableRoomID: targetPortal.MXID,
+		}
+	} else if allowNonExistent {
+		return &event.InReplyTo{
+			EventID:        targetPortal.deterministicEventID(ref.MessageID, ""),
+			UnstableRoomID: targetPortal.MXID,
+		}
+	}
+	return nil
+}
+
 const JoinThreadReaction = "join thread"
 
 func (portal *Portal) sendThreadCreationNotice(thread *Thread) {
@@ -862,6 +930,8 @@ func (portal *Portal) sendMatrixMessage(intent *appservice.IntentAPI, eventType
 }
 
 func (portal *Portal) handleMatrixMessages(msg portalMatrixMessage) {
+	portal.forwardBackfillLock.Lock()
+	defer portal.forwardBackfillLock.Unlock()
 	switch msg.evt.Type {
 	case event.EventMessage, event.EventSticker:
 		portal.handleMatrixMessage(msg.user, msg.evt)
@@ -1053,9 +1123,9 @@ func (portal *Portal) sendMessageMetrics(evt *event.Event, err error, part strin
 		evtDescription += fmt.Sprintf(" of %s", evt.Redacts)
 	}
 	if err != nil {
-		level := log.LevelError
+		level := maulogger.LevelError
 		if part == "Ignoring" {
-			level = log.LevelDebug
+			level = maulogger.LevelDebug
 		}
 		portal.log.Logfln(level, "%s %s %s from %s: %v", part, msgType, evtDescription, evt.Sender, err)
 		reason, statusCode, isCertain, sendNotice, _ := errorToStatusReason(err)
@@ -1346,10 +1416,10 @@ func (portal *Portal) cleanup(puppetsOnly bool) {
 	intent := portal.MainIntent()
 	if portal.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] {
 		err := intent.BeeperDeleteRoom(portal.MXID)
-		if err == nil || errors.Is(err, mautrix.MNotFound) {
+		if err != nil && !errors.Is(err, mautrix.MNotFound) {
-			return
+			portal.log.Errorfln("Failed to delete %s using hungryserv yeet endpoint: %v", portal.MXID, err)
 		}
-		portal.log.Warnfln("Failed to delete %s using hungryserv yeet endpoint, falling back to normal behavior: %v", portal.MXID, err)
+		return
 	}
 
 	if portal.IsPrivateChat() {
@@ -1363,7 +1433,7 @@ func (portal *Portal) cleanup(puppetsOnly bool) {
 	portal.bridge.cleanupRoom(intent, portal.MXID, puppetsOnly, portal.log)
 }
 
-func (br *DiscordBridge) cleanupRoom(intent *appservice.IntentAPI, mxid id.RoomID, puppetsOnly bool, log log.Logger) {
+func (br *DiscordBridge) cleanupRoom(intent *appservice.IntentAPI, mxid id.RoomID, puppetsOnly bool, log maulogger.Logger) {
 	members, err := intent.JoinedMembers(mxid)
 	if err != nil {
 		log.Errorln("Failed to get portal members for cleanup:", err)
@@ -1727,9 +1797,7 @@ func (portal *Portal) UpdateName(meta *discordgo.Channel) bool {
 }
 
 func (portal *Portal) UpdateNameDirect(name string) bool {
-	if portal.Name == name && (portal.NameSet || portal.MXID == "") {
+	if portal.Name == name && (portal.NameSet || portal.MXID == "" || !portal.shouldSetDMRoomMetadata()) {
-		return false
-	} else if !portal.Encrypted && !portal.bridge.Config.Bridge.PrivateChatPortalMeta && portal.IsPrivateChat() {
 		return false
 	}
 	portal.log.Debugfln("Updating name %q -> %q", portal.Name, name)
@@ -1740,7 +1808,7 @@ func (portal *Portal) UpdateNameDirect(name string) bool {
 }
 
 func (portal *Portal) updateRoomName() {
-	if portal.MXID != "" {
+	if portal.MXID != "" && portal.shouldSetDMRoomMetadata() {
 		_, err := portal.MainIntent().SetRoomName(portal.MXID, portal.Name)
 		if err != nil {
 			portal.log.Warnln("Failed to update room name:", err)
@@ -1751,9 +1819,7 @@ func (portal *Portal) updateRoomName() {
 }
 
 func (portal *Portal) UpdateAvatarFromPuppet(puppet *Puppet) bool {
-	if portal.Avatar == puppet.Avatar && portal.AvatarURL == puppet.AvatarURL && (portal.AvatarSet || portal.MXID == "") {
+	if portal.Avatar == puppet.Avatar && portal.AvatarURL == puppet.AvatarURL && (portal.AvatarSet || portal.MXID == "" || !portal.shouldSetDMRoomMetadata()) {
-		return false
-	} else if !portal.Encrypted && !portal.bridge.Config.Bridge.PrivateChatPortalMeta && portal.IsPrivateChat() {
 		return false
 	}
 	portal.log.Debugfln("Updating avatar from puppet %q -> %q", portal.Avatar, puppet.Avatar)
@@ -1786,7 +1852,7 @@ func (portal *Portal) UpdateGroupDMAvatar(iconID string) bool {
 }
 
 func (portal *Portal) updateRoomAvatar() {
-	if portal.MXID == "" || portal.AvatarURL.IsEmpty() {
+	if portal.MXID == "" || portal.AvatarURL.IsEmpty() || !portal.shouldSetDMRoomMetadata() {
 		return
 	}
 	_, err := portal.MainIntent().SetRoomAvatar(portal.MXID, portal.AvatarURL)

portal_convert.go

@@ -137,9 +137,20 @@ func (portal *Portal) convertDiscordAttachment(intent *appservice.IntentAPI, att
 		content.FileName = att.Filename
 	}
 
+	var extra map[string]any
+
 	switch strings.ToLower(strings.Split(att.ContentType, "/")[0]) {
 	case "audio":
 		content.MsgType = event.MsgAudio
+		if att.Waveform != nil {
+			// TODO convert waveform
+			extra = map[string]any{
+				"org.matrix.1767.audio": map[string]any{
+					"duration": int(att.DurationSeconds * 1000),
+				},
+				"org.matrix.msc3245.voice": map[string]any{},
+			}
+		}
 	case "image":
 		content.MsgType = event.MsgImage
 	case "video":
@@ -152,6 +163,7 @@ func (portal *Portal) convertDiscordAttachment(intent *appservice.IntentAPI, att
 		AttachmentID: att.ID,
 		Type:         event.EventMessage,
 		Content:      content,
+		Extra:        extra,
 	}
 }
 

user.go

@@ -728,6 +728,7 @@ func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel,
 		}
 	} else {
 		portal.UpdateInfo(user, meta)
+		portal.ForwardBackfillMissed(user, meta)
 	}
 	user.MarkInPortal(database.UserPortal{
 		DiscordID: portal.Key.ChannelID,
@@ -842,6 +843,9 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp
 				}
 			} else {
 				portal.UpdateInfo(user, ch)
+				if user.bridge.Config.Bridge.Backfill.MaxGuildMembers < 0 || meta.MemberCount < user.bridge.Config.Bridge.Backfill.MaxGuildMembers {
+					portal.ForwardBackfillMissed(user, ch)
+				}
 			}
 		}
 	}