Bump version to v0.6.5

Obviously won't help if the encryption hardening options are enabled,
because the point of those is to prevent the bridge from decrypting old
messages.

Fixes #131

.github/workflows/go.yml

@@ -5,13 +5,19 @@ on: [push, pull_request]
 jobs:
   lint:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version: ["1.20", "1.21"]
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.20"
+          go-version: ${{ matrix.go-version }}
+          cache: true
 
       - name: Install libolm
         run: sudo apt-get install libolm-dev libolm3

.gitignore

@@ -4,3 +4,6 @@
 
 *.db*
 *.log*
+
+/mautrix-discord
+/start

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
@@ -13,3 +13,8 @@ repos:
     hooks:
       - id: go-imports-repo
       - id: go-vet-repo-mod
+
+  - repo: https://github.com/beeper/pre-commit-go
+    rev: v0.2.2
+    hooks:
+      - id: zerolog-ban-msgf

CHANGELOG.md

@@ -1,3 +1,53 @@
+# v0.6.5 (2024-01-16)
+
+* Fixed adding reply embed to webhook sends if the Matrix room is encrypted.
+
+# v0.6.4 (2023-11-16)
+
+* Changed error messages to be sent in a thread if the errored message was in
+  a thread.
+
+# v0.6.3 (2023-10-16)
+
+* Fixed op7 reconnects during connection causing the bridge to get stuck
+  disconnected.
+* Fixed double puppet of recipient joining DM portals when both ends of a DM
+  are using the same bridge.
+
+# v0.6.2 (2023-09-16)
+
+* Added support for double puppeting with arbitrary `as_token`s.
+  See [docs](https://docs.mau.fi/bridges/general/double-puppeting.html#appservice-method-new) for more info.
+* Adjusted markdown parsing rules to allow inline links in normal messages.
+* Fixed panic if redacting an attachment fails.
+* Fixed panic when handling video embeds with no URLs
+  (thanks to [@odrling] in [#110]).
+
+[@odrling]: https://github.com/odrling
+[#110]: https://github.com/mautrix/discord/pull/110
+
+# v0.6.1 (2023-08-16)
+
+* Bumped minimum Go version to 1.20.
+* Fixed all logged-in users being invited to existing portal rooms even if they
+  don't have permission to view the channel on Discord.
+* Fixed gif links not being treated as embeds if the canonical URL is different
+  than the URL in the message body.
+
+# v0.6.0 (2023-07-16)
+
+* Added initial support for backfilling threads.
+* Exposed `Application` flag to displayname template.
+* Changed `m.emote` bridging to use italics on Discord.
+* Updated Docker image to Alpine 3.18.
+* Added limit to parallel media transfers to avoid high memory usage if lots
+  of messages are received at the same time.
+* Fixed guilds being unbridged if Discord has server issues and temporarily
+  marks a guild as unavailable.
+* Fixed using `guilds bridge` command without `--entire` flag.
+* Fixed panic if lottieconverter isn't installed.
+* Fixed relay webhook secret being leaked in network error messages.
+
 # v0.5.0 (2023-06-16)
 
 * Added support for intentional mentions in Matrix (MSC3952).

Dockerfile

@@ -1,6 +1,6 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17 AS lottie
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.18 AS lottie
 
-FROM golang:1-alpine3.17 AS builder
+FROM golang:1-alpine3.18 AS builder
 
 RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev
 
@@ -8,7 +8,7 @@ COPY . /build
 WORKDIR /build
 RUN go build -o /usr/bin/mautrix-discord
 
-FROM alpine:3.17
+FROM alpine:3.18
 
 ENV UID=1337 \
     GID=1337

Dockerfile.ci

@@ -1,6 +1,6 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17 AS lottie
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.18 AS lottie
 
-FROM alpine:3.17
+FROM alpine:3.18
 
 ENV UID=1337 \
     GID=1337

Dockerfile.dev

@@ -1,6 +1,6 @@
-FROM dock.mau.dev/tulir/lottieconverter:alpine-3.17 AS lottie
+FROM dock.mau.dev/tulir/lottieconverter:alpine-3.18 AS lottie
 
-FROM golang:1-alpine3.17 AS builder
+FROM golang:1-alpine3.18 AS builder
 
 RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl \
     zlib libpng giflib libstdc++ libgcc

ROADMAP.md

@@ -1,11 +1,12 @@
 # Features & roadmap
 * Matrix → Discord
-  * [x] Message content
+  * [ ] Message content
     * [x] Plain text
     * [x] Formatted messages
     * [x] Media/files
     * [x] Replies
     * [x] Threads
+    * [ ] Custom emojis
   * [x] Message redactions
   * [x] Reactions
     * [x] Unicode emojis
@@ -45,7 +46,7 @@
   * [x] Message deletions
   * [x] Reactions
     * [x] Unicode emojis
-    * [x] Custom emojis (not yet supported on Matrix)
+    * [x] Custom emojis ([MSC4027](https://github.com/matrix-org/matrix-spec-proposals/pull/4027))
   * [x] Avatars
   * [ ] Presence
   * [ ] Typing notifications (currently partial support: DMs work after you type in them)

attachments.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"image"
 	"io"
@@ -12,23 +13,23 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/bwmarrin/discordgo"
 	"github.com/gabriel-vasile/mimetype"
-
+	"go.mau.fi/util/exsync"
+	"go.mau.fi/util/ffmpeg"
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/crypto/attachment"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util"
-	"maunium.net/go/mautrix/util/ffmpeg"
 
 	"go.mau.fi/mautrix-discord/database"
 )
 
-func downloadDiscordAttachment(url string) ([]byte, error) {
+func downloadDiscordAttachment(url string, maxSize int64) ([]byte, error) {
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
@@ -44,9 +45,24 @@ func downloadDiscordAttachment(url string) ([]byte, error) {
 	defer resp.Body.Close()
 	if resp.StatusCode > 300 {
 		data, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, data)
+		return nil, fmt.Errorf("unexpected status %d downloading %s: %s", resp.StatusCode, url, data)
+	}
+	if resp.Header.Get("Content-Length") != "" {
+		length, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse content length: %w", err)
+		} else if length > maxSize {
+			return nil, fmt.Errorf("attachment too large (%d > %d)", length, maxSize)
+		}
+		return io.ReadAll(resp.Body)
+	} else {
+		var mbe *http.MaxBytesError
+		data, err := io.ReadAll(http.MaxBytesReader(nil, resp.Body, maxSize))
+		if err != nil && errors.As(err, &mbe) {
+			return nil, fmt.Errorf("attachment too large (over %d)", maxSize)
+		}
+		return data, err
 	}
-	return io.ReadAll(resp.Body)
 }
 
 func uploadDiscordAttachment(url string, data []byte) error {
@@ -99,7 +115,7 @@ func downloadMatrixAttachment(intent *appservice.IntentAPI, content *event.Messa
 	return data, nil
 }
 
-func (br *DiscordBridge) uploadMatrixAttachment(intent *appservice.IntentAPI, data []byte, url string, encrypt bool, meta AttachmentMeta) (*database.File, error) {
+func (br *DiscordBridge) uploadMatrixAttachment(intent *appservice.IntentAPI, data []byte, url string, encrypt bool, meta AttachmentMeta, semaWg *sync.WaitGroup) (*database.File, error) {
 	dbFile := br.DB.File.New()
 	dbFile.Timestamp = time.Now()
 	dbFile.URL = url
@@ -135,7 +151,9 @@ func (br *DiscordBridge) uploadMatrixAttachment(intent *appservice.IntentAPI, da
 		dbFile.MXC = resp.ContentURI
 		req.MXC = resp.ContentURI
 		req.UnstableUploadURL = resp.UnstableUploadURL
+		semaWg.Add(1)
 		go func() {
+			defer semaWg.Done()
 			_, err = intent.UploadMedia(req)
 			if err != nil {
 				br.Log.Errorfln("Failed to upload %s: %v", req.MXC, err)
@@ -250,7 +268,7 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur
 	returnDBFile = br.DB.File.Get(url, encrypt)
 	if returnDBFile == nil {
 		transferKey := attachmentKey{url, encrypt}
-		once, _ := br.attachmentTransfers.GetOrSet(transferKey, &util.ReturnableOnce[*database.File]{})
+		once, _ := br.attachmentTransfers.GetOrSet(transferKey, &exsync.ReturnableOnce[*database.File]{})
 		returnDBFile, returnErr = once.Do(func() (onceDBFile *database.File, onceErr error) {
 			if isCacheable {
 				onceDBFile = br.DB.File.Get(url, encrypt)
@@ -259,8 +277,25 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur
 				}
 			}
 
+			const attachmentSizeVal = 1
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			onceErr = br.parallelAttachmentSemaphore.Acquire(ctx, attachmentSizeVal)
+			cancel()
+			if onceErr != nil {
+				br.ZLog.Warn().Err(onceErr).Msg("Failed to acquire semaphore")
+				onceErr = fmt.Errorf("reuploading timed out")
+				return
+			}
+			var semaWg sync.WaitGroup
+			semaWg.Add(1)
+			defer semaWg.Done()
+			go func() {
+				semaWg.Wait()
+				br.parallelAttachmentSemaphore.Release(attachmentSizeVal)
+			}()
+
 			var data []byte
-			data, onceErr = downloadDiscordAttachment(url)
+			data, onceErr = downloadDiscordAttachment(url, br.MediaConfig.UploadSize)
 			if onceErr != nil {
 				return
 			}
@@ -273,7 +308,7 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur
 				}
 			}
 
-			onceDBFile, onceErr = br.uploadMatrixAttachment(intent, data, url, encrypt, meta)
+			onceDBFile, onceErr = br.uploadMatrixAttachment(intent, data, url, encrypt, meta, &semaWg)
 			if onceErr != nil {
 				return
 			}

backfill.go

@@ -10,15 +10,18 @@ import (
 	"github.com/bwmarrin/discordgo"
 	"github.com/rs/zerolog"
 	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 
 	"go.mau.fi/mautrix-discord/database"
 )
 
-func (portal *Portal) forwardBackfillInitial(source *User) {
+func (portal *Portal) forwardBackfillInitial(source *User, thread *Thread) {
-	defer portal.forwardBackfillLock.Unlock()
+	log := portal.log
+	defer func() {
+		log.Debug().Msg("Forward backfill finished, unlocking lock")
+		portal.forwardBackfillLock.Unlock()
+	}()
 	// This should only be called from CreateMatrixRoom which locks forwardBackfillLock before creating the room.
 	if portal.forwardBackfillLock.TryLock() {
 		panic("forwardBackfillInitial() called without locking forwardBackfillLock")
@@ -27,21 +30,28 @@ func (portal *Portal) forwardBackfillInitial(source *User) {
 	limit := portal.bridge.Config.Bridge.Backfill.Limits.Initial.Channel
 	if portal.GuildID == "" {
 		limit = portal.bridge.Config.Bridge.Backfill.Limits.Initial.DM
+		if thread != nil {
+			limit = portal.bridge.Config.Bridge.Backfill.Limits.Initial.Thread
+			thread.initialBackfillAttempted = true
+		}
 	}
 	if limit == 0 {
 		return
 	}
 
-	log := portal.log.With().
+	with := log.With().
 		Str("action", "initial backfill").
 		Str("room_id", portal.MXID.String()).
-		Int("limit", limit).
+		Int("limit", limit)
-		Logger()
+	if thread != nil {
+		with = with.Str("thread_id", thread.ID)
+	}
+	log = with.Logger()
 
-	portal.backfillLimited(log, source, limit, "")
+	portal.backfillLimited(log, source, limit, "", thread)
 }
 
-func (portal *Portal) ForwardBackfillMissed(source *User, meta *discordgo.Channel) {
+func (portal *Portal) ForwardBackfillMissed(source *User, serverLastMessageID string, thread *Thread) {
 	if portal.MXID == "" {
 		return
 	}
@@ -49,50 +59,65 @@ func (portal *Portal) ForwardBackfillMissed(source *User, meta *discordgo.Channe
 	limit := portal.bridge.Config.Bridge.Backfill.Limits.Missed.Channel
 	if portal.GuildID == "" {
 		limit = portal.bridge.Config.Bridge.Backfill.Limits.Missed.DM
+		if thread != nil {
+			limit = portal.bridge.Config.Bridge.Backfill.Limits.Missed.Thread
+		}
 	}
 	if limit == 0 {
 		return
 	}
-	log := portal.log.With().
+	with := portal.log.With().
 		Str("action", "missed event backfill").
 		Str("room_id", portal.MXID.String()).
-		Int("limit", limit).
+		Int("limit", limit)
-		Logger()
+	if thread != nil {
+		with = with.Str("thread_id", thread.ID)
+	}
+	log := with.Logger()
 
 	portal.forwardBackfillLock.Lock()
 	defer portal.forwardBackfillLock.Unlock()
 
-	lastMessage := portal.bridge.DB.Message.GetLast(portal.Key)
+	var lastMessage *database.Message
-	if lastMessage == nil || meta.LastMessageID == "" {
+	if thread != nil {
+		lastMessage = portal.bridge.DB.Message.GetLastInThread(portal.Key, thread.ID)
+	} else {
+		lastMessage = portal.bridge.DB.Message.GetLast(portal.Key)
+	}
+	if lastMessage == nil || serverLastMessageID == "" {
 		log.Debug().Msg("Not backfilling, no last message in database or no last message in metadata")
 		return
-	} else if !shouldBackfill(lastMessage.DiscordID, meta.LastMessageID) {
+	} else if !shouldBackfill(lastMessage.DiscordID, serverLastMessageID) {
 		log.Debug().
 			Str("last_bridged_message", lastMessage.DiscordID).
-			Str("last_server_message", meta.LastMessageID).
+			Str("last_server_message", serverLastMessageID).
 			Msg("Not backfilling, last message in database is newer than last message in metadata")
 		return
 	}
 	log.Debug().
 		Str("last_bridged_message", lastMessage.DiscordID).
-		Str("last_server_message", meta.LastMessageID).
+		Str("last_server_message", serverLastMessageID).
 		Msg("Backfilling missed messages")
 	if limit < 0 {
-		portal.backfillUnlimitedMissed(log, source, lastMessage.DiscordID)
+		portal.backfillUnlimitedMissed(log, source, lastMessage.DiscordID, thread)
 	} else {
-		portal.backfillLimited(log, source, limit, lastMessage.DiscordID)
+		portal.backfillLimited(log, source, limit, lastMessage.DiscordID, thread)
 	}
 }
 
 const messageFetchChunkSize = 50
 
-func (portal *Portal) collectBackfillMessages(log zerolog.Logger, source *User, limit int, until string) ([]*discordgo.Message, bool, error) {
+func (portal *Portal) collectBackfillMessages(log zerolog.Logger, source *User, limit int, until string, thread *Thread) ([]*discordgo.Message, bool, error) {
 	var messages []*discordgo.Message
 	var before string
 	var foundAll bool
+	protoChannelID := portal.Key.ChannelID
+	if thread != nil {
+		protoChannelID = thread.ID
+	}
 	for {
 		log.Debug().Str("before_id", before).Msg("Fetching messages for backfill")
-		newMessages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, before, "", "")
+		newMessages, err := source.Session.ChannelMessages(protoChannelID, messageFetchChunkSize, before, "", "")
 		if err != nil {
 			return nil, false, err
 		}
@@ -123,8 +148,8 @@ func (portal *Portal) collectBackfillMessages(log zerolog.Logger, source *User,
 	return messages, foundAll, nil
 }
 
-func (portal *Portal) backfillLimited(log zerolog.Logger, source *User, limit int, after string) {
+func (portal *Portal) backfillLimited(log zerolog.Logger, source *User, limit int, after string, thread *Thread) {
-	messages, foundAll, err := portal.collectBackfillMessages(log, source, limit, after)
+	messages, foundAll, err := portal.collectBackfillMessages(log, source, limit, after, thread)
 	if err != nil {
 		log.Err(err).Msg("Error collecting messages to forward backfill")
 		return
@@ -145,13 +170,17 @@ func (portal *Portal) backfillLimited(log zerolog.Logger, source *User, limit in
 			log.Debug().Msg("Sent warning about possibly missed messages")
 		}
 	}
-	portal.sendBackfillBatch(log, source, messages)
+	portal.sendBackfillBatch(log, source, messages, thread)
 }
 
-func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User, after string) {
+func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User, after string, thread *Thread) {
+	protoChannelID := portal.Key.ChannelID
+	if thread != nil {
+		protoChannelID = thread.ID
+	}
 	for {
 		log.Debug().Str("after_id", after).Msg("Fetching chunk of messages to backfill")
-		messages, err := source.Session.ChannelMessages(portal.Key.ChannelID, messageFetchChunkSize, "", after, "")
+		messages, err := source.Session.ChannelMessages(protoChannelID, messageFetchChunkSize, "", after, "")
 		if err != nil {
 			log.Err(err).Msg("Error fetching chunk of messages to forward backfill")
 			return
@@ -159,7 +188,7 @@ func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User,
 		log.Debug().Int("count", len(messages)).Msg("Fetched chunk of messages to backfill")
 		sort.Sort(MessageSlice(messages))
 
-		portal.sendBackfillBatch(log, source, messages)
+		portal.sendBackfillBatch(log, source, messages, thread)
 
 		if len(messages) < messageFetchChunkSize {
 			// Assume that was all the missing messages
@@ -170,28 +199,28 @@ func (portal *Portal) backfillUnlimitedMissed(log zerolog.Logger, source *User,
 	}
 }
 
-func (portal *Portal) sendBackfillBatch(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+func (portal *Portal) sendBackfillBatch(log zerolog.Logger, source *User, messages []*discordgo.Message, thread *Thread) {
-	if portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry {
+	if portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureBatchSending) {
 		log.Debug().Msg("Using hungryserv, sending messages with batch send endpoint")
-		portal.forwardBatchSend(log, source, messages)
+		portal.forwardBatchSend(log, source, messages, thread)
 	} else {
 		log.Debug().Msg("Not using hungryserv, sending messages one by one")
 		for _, msg := range messages {
-			portal.handleDiscordMessageCreate(source, msg, nil)
+			portal.handleDiscordMessageCreate(source, msg, thread)
 		}
 	}
 }
 
-func (portal *Portal) forwardBatchSend(log zerolog.Logger, source *User, messages []*discordgo.Message) {
+func (portal *Portal) forwardBatchSend(log zerolog.Logger, source *User, messages []*discordgo.Message, thread *Thread) {
-	evts, dbMessages := portal.convertMessageBatch(log, source, messages)
+	evts, metas, dbMessages := portal.convertMessageBatch(log, source, messages, thread)
 	if len(evts) == 0 {
 		log.Warn().Msg("Didn't get any events to backfill")
 		return
 	}
 	log.Info().Int("events", len(evts)).Msg("Converted messages to backfill")
-	resp, err := portal.MainIntent().BatchSend(portal.MXID, &mautrix.ReqBatchSend{
+	resp, err := portal.MainIntent().BeeperBatchSend(portal.MXID, &mautrix.ReqBeeperBatchSend{
-		BeeperNewMessages: true,
+		Forward: true,
-		Events:            evts,
+		Events:  evts,
 	})
 	if err != nil {
 		log.Err(err).Msg("Error sending backfill batch")
@@ -199,25 +228,42 @@ func (portal *Portal) forwardBatchSend(log zerolog.Logger, source *User, message
 	}
 	for i, evtID := range resp.EventIDs {
 		dbMessages[i].MXID = evtID
+		if metas[i] != nil && metas[i].Flags == discordgo.MessageFlagsHasThread {
+			// TODO proper context
+			ctx := log.WithContext(context.Background())
+			portal.bridge.threadFound(ctx, source, &dbMessages[i], metas[i].ID, metas[i].Thread)
+		}
 	}
 	portal.bridge.DB.Message.MassInsert(portal.Key, dbMessages)
-	log.Info().Msg("Inserted backfilled batch to database")
 }
 
-func (portal *Portal) convertMessageBatch(log zerolog.Logger, source *User, messages []*discordgo.Message) ([]*event.Event, []database.Message) {
+func (portal *Portal) convertMessageBatch(log zerolog.Logger, source *User, messages []*discordgo.Message, thread *Thread) ([]*event.Event, []*discordgo.Message, []database.Message) {
+	var discordThreadID string
+	var threadRootEvent, lastThreadEvent id.EventID
+	if thread != nil {
+		discordThreadID = thread.ID
+		threadRootEvent = thread.RootMXID
+		lastThreadEvent = threadRootEvent
+		lastInThread := portal.bridge.DB.Message.GetLastInThread(portal.Key, thread.ID)
+		if lastInThread != nil {
+			lastThreadEvent = lastInThread.MXID
+		}
+	}
+
 	evts := make([]*event.Event, 0, len(messages))
 	dbMessages := make([]database.Message, 0, len(messages))
+	metas := make([]*discordgo.Message, 0, len(messages))
 	ctx := context.Background()
 	for _, msg := range messages {
 		for _, mention := range msg.Mentions {
 			puppet := portal.bridge.GetPuppetByID(mention.ID)
-			puppet.UpdateInfo(nil, mention, "")
+			puppet.UpdateInfo(nil, mention, nil)
 		}
 
 		puppet := portal.bridge.GetPuppetByID(msg.Author.ID)
-		puppet.UpdateInfo(source, msg.Author, msg.WebhookID)
+		puppet.UpdateInfo(source, msg.Author, msg)
 		intent := puppet.IntentFor(portal)
-		replyTo := portal.getReplyTarget(source, "", msg.MessageReference, msg.Embeds, true)
+		replyTo := portal.getReplyTarget(source, discordThreadID, msg.MessageReference, msg.Embeds, true)
 		mentions := portal.convertDiscordMentions(msg, false)
 
 		ts, _ := discordgo.SnowflakeTimestamp(msg.ID)
@@ -228,8 +274,14 @@ func (portal *Portal) convertMessageBatch(log zerolog.Logger, source *User, mess
 			Logger()
 		parts := portal.convertDiscordMessage(log.WithContext(ctx), puppet, intent, msg)
 		for i, part := range parts {
+			if (replyTo != nil || threadRootEvent != "") && part.Content.RelatesTo == nil {
+				part.Content.RelatesTo = &event.RelatesTo{}
+			}
+			if threadRootEvent != "" {
+				part.Content.RelatesTo.SetThread(threadRootEvent, lastThreadEvent)
+			}
 			if replyTo != nil {
-				part.Content.RelatesTo = &event.RelatesTo{InReplyTo: replyTo}
+				part.Content.RelatesTo.SetReplyTo(replyTo.EventID)
 				// Only set reply for first event
 				replyTo = nil
 			}
@@ -270,9 +322,15 @@ func (portal *Portal) convertMessageBatch(log zerolog.Logger, source *User, mess
 				AttachmentID: part.AttachmentID,
 				SenderMXID:   intent.UserID,
 			})
+			if i == 0 {
+				metas = append(metas, msg)
+			} else {
+				metas = append(metas, nil)
+			}
+			lastThreadEvent = evt.ID
 		}
 	}
-	return evts, dbMessages
+	return evts, metas, dbMessages
 }
 
 func (portal *Portal) deterministicEventID(messageID, partName string) id.EventID {

commands.go

@@ -159,7 +159,7 @@ func fnLoginToken(ce *WrappedCommandEvent) {
 		ce.Reply("Error connecting to Discord: %v", err)
 		return
 	}
-	ce.Reply("Successfully logged in as %s#%s", ce.User.Session.State.User.Username, ce.User.Session.State.User.Discriminator)
+	ce.Reply("Successfully logged in as @%s", ce.User.Session.State.User.Username)
 }
 
 var cmdLoginQR = &commands.FullHandler{
@@ -228,7 +228,7 @@ func fnLoginQR(ce *WrappedCommandEvent) {
 	ce.User.DiscordID = user.UserID
 	ce.User.Update()
 	ce.User.Unlock()
-	ce.Reply("Successfully logged in as %s#%s", user.Username, user.Discriminator)
+	ce.Reply("Successfully logged in as @%s", user.Username)
 }
 
 func sendQRCode(ce *WrappedCommandEvent, code string) id.EventID {
@@ -308,7 +308,7 @@ func fnPing(ce *WrappedCommandEvent) {
 	} else if ce.User.wasDisconnected {
 		ce.Reply("You're logged in, but the Discord connection seems to be dead 💥")
 	} else {
-		ce.Reply("You're logged in as %s#%s (`%s`)", ce.User.Session.State.User.Username, ce.User.Session.State.User.Discriminator, ce.User.DiscordID)
+		ce.Reply("You're logged in as @%s (`%s`)", ce.User.Session.State.User.Username, ce.User.DiscordID)
 	}
 }
 

config/bridge.go

@@ -67,9 +67,7 @@ type BridgeConfig struct {
 		} `yaml:"args"`
 	} `yaml:"animated_sticker"`
 
-	DoublePuppetServerMap      map[string]string `yaml:"double_puppet_server_map"`
+	DoublePuppetConfig bridgeconfig.DoublePuppetConfig `yaml:",inline"`
-	DoublePuppetAllowDiscovery bool              `yaml:"double_puppet_allow_discovery"`
-	LoginSharedSecretMap       map[string]string `yaml:"login_shared_secret_map"`
 
 	CommandPrefix      string                           `yaml:"command_prefix"`
 	ManagementRoomText bridgeconfig.ManagementRoomTexts `yaml:"management_room_text"`
@@ -85,8 +83,9 @@ type BridgeConfig struct {
 	Encryption bridgeconfig.EncryptionConfig `yaml:"encryption"`
 
 	Provisioning struct {
-		Prefix       string `yaml:"prefix"`
+		Prefix         string `yaml:"prefix"`
-		SharedSecret string `yaml:"shared_secret"`
+		SharedSecret   string `yaml:"shared_secret"`
+		DebugEndpoints bool   `yaml:"debug_endpoints"`
 	} `yaml:"provisioning"`
 
 	Permissions bridgeconfig.PermissionConfig `yaml:"permissions"`
@@ -207,6 +206,7 @@ func (mp *MediaPatterns) Avatar(userID, avatarID, ext string) id.ContentURI {
 type BackfillLimitPart struct {
 	DM      int `yaml:"dm"`
 	Channel int `yaml:"channel"`
+	Thread  int `yaml:"thread"`
 }
 
 func (bc *BridgeConfig) GetResendBridgeInfo() bool {
@@ -271,6 +271,10 @@ func (bc *BridgeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 var _ bridgeconfig.BridgeConfig = (*BridgeConfig)(nil)
 
+func (bc BridgeConfig) GetDoublePuppetConfig() bridgeconfig.DoublePuppetConfig {
+	return bc.DoublePuppetConfig
+}
+
 func (bc BridgeConfig) GetEncryptionConfig() bridgeconfig.EncryptionConfig {
 	return bc.Encryption
 }
@@ -291,12 +295,17 @@ func (bc BridgeConfig) FormatUsername(userID string) string {
 
 type DisplaynameParams struct {
 	*discordgo.User
-	Webhook bool
+	Webhook     bool
+	Application bool
 }
 
-func (bc BridgeConfig) FormatDisplayname(user *discordgo.User, webhook bool) string {
+func (bc BridgeConfig) FormatDisplayname(user *discordgo.User, webhook, application bool) string {
 	var buffer strings.Builder
-	_ = bc.displaynameTemplate.Execute(&buffer, &DisplaynameParams{user, webhook})
+	_ = bc.displaynameTemplate.Execute(&buffer, &DisplaynameParams{
+		User:        user,
+		Webhook:     webhook,
+		Application: application,
+	})
 	return buffer.String()
 }
 

config/config.go

@@ -29,7 +29,7 @@ type Config struct {
 
 func (config *Config) CanAutoDoublePuppet(userID id.UserID) bool {
 	_, homeserver, _ := userID.Parse()
-	_, hasSecret := config.Bridge.LoginSharedSecretMap[homeserver]
+	_, hasSecret := config.Bridge.DoublePuppetConfig.SharedSecretMap[homeserver]
 
 	return hasSecret
 }

config/upgrade.go

@@ -17,9 +17,9 @@
 package config
 
 import (
+	up "go.mau.fi/util/configupgrade"
+	"go.mau.fi/util/random"
 	"maunium.net/go/mautrix/bridge/bridgeconfig"
-	"maunium.net/go/mautrix/util"
-	up "maunium.net/go/mautrix/util/configupgrade"
 )
 
 func DoUpgrade(helper *up.Helper) {
@@ -79,8 +79,10 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Bool, "bridge", "backfill", "enabled")
 	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "dm")
 	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "initial", "thread")
 	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "dm")
 	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "channel")
+	helper.Copy(up.Int, "bridge", "backfill", "forward_limits", "missed", "thread")
 	helper.Copy(up.Int, "bridge", "backfill", "max_guild_members")
 	helper.Copy(up.Bool, "bridge", "encryption", "allow")
 	helper.Copy(up.Bool, "bridge", "encryption", "default")
@@ -95,6 +97,7 @@ func DoUpgrade(helper *up.Helper) {
 	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_prev_on_new_session")
 	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_on_device_delete")
 	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "periodically_delete_expired")
+	helper.Copy(up.Bool, "bridge", "encryption", "delete_keys", "delete_outdated_inbound")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "receive")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "send")
 	helper.Copy(up.Str, "bridge", "encryption", "verification_levels", "share")
@@ -105,11 +108,12 @@ func DoUpgrade(helper *up.Helper) {
 
 	helper.Copy(up.Str, "bridge", "provisioning", "prefix")
 	if secret, ok := helper.Get(up.Str, "bridge", "provisioning", "shared_secret"); !ok || secret == "generate" {
-		sharedSecret := util.RandomString(64)
+		sharedSecret := random.String(64)
 		helper.Set(up.Str, sharedSecret, "bridge", "provisioning", "shared_secret")
 	} else {
 		helper.Copy(up.Str, "bridge", "provisioning", "shared_secret")
 	}
+	helper.Copy(up.Bool, "bridge", "provisioning", "debug_endpoints")
 
 	helper.Copy(up.Map, "bridge", "permissions")
 	//helper.Copy(up.Bool, "bridge", "relay", "enabled")

custompuppet.go

@@ -1,170 +1,72 @@
 package main
 
 import (
-	"crypto/hmac"
-	"crypto/sha512"
-	"encoding/hex"
-	"errors"
-	"fmt"
-
-	"maunium.net/go/mautrix"
-	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/id"
 )
 
-var (
-	ErrNoCustomMXID    = errors.New("no custom mxid set")
-	ErrMismatchingMXID = errors.New("whoami result does not match custom mxid")
-)
-
-func (br *DiscordBridge) newDoublePuppetClient(mxid id.UserID, accessToken string) (*mautrix.Client, error) {
-	_, homeserver, err := mxid.Parse()
-	if err != nil {
-		return nil, err
-	}
-
-	homeserverURL, found := br.Config.Bridge.DoublePuppetServerMap[homeserver]
-	if !found {
-		if homeserver == br.AS.HomeserverDomain {
-			homeserverURL = ""
-		} else if br.Config.Bridge.DoublePuppetAllowDiscovery {
-			resp, err := mautrix.DiscoverClientAPI(homeserver)
-			if err != nil {
-				return nil, fmt.Errorf("failed to find homeserver URL for %s: %v", homeserver, err)
-			}
-
-			homeserverURL = resp.Homeserver.BaseURL
-			br.Log.Debugfln("Discovered URL %s for %s to enable double puppeting for %s", homeserverURL, homeserver, mxid)
-		} else {
-			return nil, fmt.Errorf("double puppeting from %s is not allowed", homeserver)
-		}
-	}
-
-	return br.AS.NewExternalMautrixClient(mxid, accessToken, homeserverURL)
-}
-
-func (puppet *Puppet) clearCustomMXID() {
-	puppet.CustomMXID = ""
-	puppet.AccessToken = ""
-	puppet.customIntent = nil
-	puppet.customUser = nil
-}
-
-func (puppet *Puppet) newCustomIntent() (*appservice.IntentAPI, error) {
-	if puppet.CustomMXID == "" {
-		return nil, ErrNoCustomMXID
-	}
-
-	client, err := puppet.bridge.newDoublePuppetClient(puppet.CustomMXID, puppet.AccessToken)
-	if err != nil {
-		return nil, err
-	}
-
-	ia := puppet.bridge.AS.NewIntentAPI("custom")
-	ia.Client = client
-	ia.Localpart, _, _ = puppet.CustomMXID.Parse()
-	ia.UserID = puppet.CustomMXID
-	ia.IsCustomPuppet = true
-	return ia, nil
-}
-
-func (puppet *Puppet) StartCustomMXID(reloginOnFail bool) error {
-	if puppet.CustomMXID == "" {
-		puppet.clearCustomMXID()
-		return nil
-	}
-
-	intent, err := puppet.newCustomIntent()
-	if err != nil {
-		puppet.clearCustomMXID()
-		return err
-	}
-
-	resp, err := intent.Whoami()
-	if err != nil {
-		if !reloginOnFail || (errors.Is(err, mautrix.MUnknownToken) && !puppet.tryRelogin(err, "initializing double puppeting")) {
-			puppet.clearCustomMXID()
-			return err
-		}
-
-		intent.AccessToken = puppet.AccessToken
-	} else if resp.UserID != puppet.CustomMXID {
-		puppet.clearCustomMXID()
-		return ErrMismatchingMXID
-	}
-
-	puppet.customIntent = intent
-	puppet.customUser = puppet.bridge.GetUserByMXID(puppet.CustomMXID)
-	return nil
-}
-
-func (puppet *Puppet) tryRelogin(cause error, action string) bool {
-	if !puppet.bridge.Config.CanAutoDoublePuppet(puppet.CustomMXID) {
-		return false
-	}
-	log := puppet.log.With().
-		AnErr("cause_error", cause).
-		Str("while_action", action).
-		Logger()
-	log.Debug().Msg("Trying to relogin")
-	accessToken, err := puppet.loginWithSharedSecret(puppet.CustomMXID)
-	if err != nil {
-		log.Error().Err(err).Msg("Failed to relogin")
-		return false
-	}
-	log.Info().Msg("Successfully relogined")
-	puppet.AccessToken = accessToken
-	puppet.Update()
-	return true
-}
-
-func (puppet *Puppet) loginWithSharedSecret(mxid id.UserID) (string, error) {
-	_, homeserver, _ := mxid.Parse()
-	puppet.log.Debug().Str("user_id", mxid.String()).Msg("Logging into double puppet target with shared secret")
-	loginSecret := puppet.bridge.Config.Bridge.LoginSharedSecretMap[homeserver]
-	client, err := puppet.bridge.newDoublePuppetClient(mxid, "")
-	if err != nil {
-		return "", fmt.Errorf("failed to create mautrix client to log in: %v", err)
-	}
-	req := mautrix.ReqLogin{
-		Identifier:               mautrix.UserIdentifier{Type: mautrix.IdentifierTypeUser, User: string(mxid)},
-		DeviceID:                 "Discord Bridge",
-		InitialDeviceDisplayName: "Discord Bridge",
-	}
-	if loginSecret == "appservice" {
-		client.AccessToken = puppet.bridge.AS.Registration.AppToken
-		req.Type = mautrix.AuthTypeAppservice
-	} else {
-		mac := hmac.New(sha512.New, []byte(loginSecret))
-		mac.Write([]byte(mxid))
-		req.Password = hex.EncodeToString(mac.Sum(nil))
-		req.Type = mautrix.AuthTypePassword
-	}
-	resp, err := client.Login(&req)
-	if err != nil {
-		return "", err
-	}
-	return resp.AccessToken, nil
-}
-
 func (puppet *Puppet) SwitchCustomMXID(accessToken string, mxid id.UserID) error {
-	prevCustomMXID := puppet.CustomMXID
 	puppet.CustomMXID = mxid
 	puppet.AccessToken = accessToken
-
+	puppet.Update()
 	err := puppet.StartCustomMXID(false)
 	if err != nil {
 		return err
 	}
-
-	if prevCustomMXID != "" {
-		delete(puppet.bridge.puppetsByCustomMXID, prevCustomMXID)
-	}
-	if puppet.CustomMXID != "" {
-		puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet
-	}
-	puppet.bridge.AS.StateStore.MarkRegistered(puppet.CustomMXID)
-	puppet.Update()
 	// TODO leave rooms with default puppet
 	return nil
 }
+
+func (puppet *Puppet) ClearCustomMXID() {
+	save := puppet.CustomMXID != "" || puppet.AccessToken != ""
+	puppet.bridge.puppetsLock.Lock()
+	if puppet.CustomMXID != "" && puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] == puppet {
+		delete(puppet.bridge.puppetsByCustomMXID, puppet.CustomMXID)
+	}
+	puppet.bridge.puppetsLock.Unlock()
+	puppet.CustomMXID = ""
+	puppet.AccessToken = ""
+	puppet.customIntent = nil
+	puppet.customUser = nil
+	if save {
+		puppet.Update()
+	}
+}
+
+func (puppet *Puppet) StartCustomMXID(reloginOnFail bool) error {
+	newIntent, newAccessToken, err := puppet.bridge.DoublePuppet.Setup(puppet.CustomMXID, puppet.AccessToken, reloginOnFail)
+	if err != nil {
+		puppet.ClearCustomMXID()
+		return err
+	}
+	puppet.bridge.puppetsLock.Lock()
+	puppet.bridge.puppetsByCustomMXID[puppet.CustomMXID] = puppet
+	puppet.bridge.puppetsLock.Unlock()
+	if puppet.AccessToken != newAccessToken {
+		puppet.AccessToken = newAccessToken
+		puppet.Update()
+	}
+	puppet.customIntent = newIntent
+	puppet.customUser = puppet.bridge.GetUserByMXID(puppet.CustomMXID)
+	return nil
+}
+
+func (user *User) tryAutomaticDoublePuppeting() {
+	if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) {
+		return
+	}
+	user.log.Debug().Msg("Checking if double puppeting needs to be enabled")
+	puppet := user.bridge.GetPuppetByID(user.DiscordID)
+	if len(puppet.CustomMXID) > 0 {
+		user.log.Debug().Msg("User already has double-puppeting enabled")
+		// Custom puppet already enabled
+		return
+	}
+	puppet.CustomMXID = user.MXID
+	err := puppet.StartCustomMXID(true)
+	if err != nil {
+		user.log.Warn().Err(err).Msg("Failed to login with shared secret")
+	} else {
+		// TODO leave rooms with default puppet
+		user.log.Debug().Msg("Successfully automatically enabled custom puppet")
+	}
+}

database/database.go

@@ -5,10 +5,9 @@ import (
 
 	_ "github.com/lib/pq"
 	_ "github.com/mattn/go-sqlite3"
+	"go.mau.fi/util/dbutil"
 	"maunium.net/go/maulogger/v2"
 
-	"maunium.net/go/mautrix/util/dbutil"
-
 	"go.mau.fi/mautrix-discord/database/upgrades"
 )
 

database/file.go

@@ -6,11 +6,10 @@ import (
 	"errors"
 	"time"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/crypto/attachment"
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type FileQuery struct {

database/guild.go

@@ -6,10 +6,9 @@ import (
 	"fmt"
 	"strings"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
 	"maunium.net/go/mautrix/id"
-
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type GuildBridgingMode int

database/message.go

@@ -7,10 +7,9 @@ import (
 	"strings"
 	"time"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type MessageQuery struct {

database/portal.go

@@ -4,11 +4,9 @@ import (
 	"database/sql"
 
 	"github.com/bwmarrin/discordgo"
-
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 // language=postgresql

database/puppet.go

@@ -3,15 +3,14 @@ package database
 import (
 	"database/sql"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 const (
 	puppetSelect = "SELECT id, name, name_set, avatar, avatar_url, avatar_set," +
-		" contact_info_set, global_name, username, discriminator, is_bot, is_webhook, custom_mxid, access_token, next_batch" +
+		" contact_info_set, global_name, username, discriminator, is_bot, is_webhook, is_application, custom_mxid, access_token, next_batch" +
 		" FROM puppet "
 )
 
@@ -80,6 +79,7 @@ type Puppet struct {
 	Discriminator string
 	IsBot         bool
 	IsWebhook     bool
+	IsApplication bool
 
 	CustomMXID  id.UserID
 	AccessToken string
@@ -91,7 +91,7 @@ func (p *Puppet) Scan(row dbutil.Scannable) *Puppet {
 	var customMXID, accessToken, nextBatch sql.NullString
 
 	err := row.Scan(&p.ID, &p.Name, &p.NameSet, &p.Avatar, &avatarURL, &p.AvatarSet, &p.ContactInfoSet,
-		&p.GlobalName, &p.Username, &p.Discriminator, &p.IsBot, &p.IsWebhook, &customMXID, &accessToken, &nextBatch)
+		&p.GlobalName, &p.Username, &p.Discriminator, &p.IsBot, &p.IsWebhook, &p.IsApplication, &customMXID, &accessToken, &nextBatch)
 
 	if err != nil {
 		if err != sql.ErrNoRows {
@@ -114,13 +114,13 @@ func (p *Puppet) Insert() {
 	query := `
 		INSERT INTO puppet (
 			id, name, name_set, avatar, avatar_url, avatar_set, contact_info_set,
-			global_name, username, discriminator, is_bot, is_webhook,
+			global_name, username, discriminator, is_bot, is_webhook, is_application,
 			custom_mxid, access_token, next_batch
 		)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
 	`
 	_, err := p.db.Exec(query, p.ID, p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet, p.ContactInfoSet,
-		p.GlobalName, p.Username, p.Discriminator, p.IsBot, p.IsWebhook,
+		p.GlobalName, p.Username, p.Discriminator, p.IsBot, p.IsWebhook, p.IsApplication,
 		strPtr(p.CustomMXID), strPtr(p.AccessToken), strPtr(p.NextBatch))
 
 	if err != nil {
@@ -132,14 +132,14 @@ func (p *Puppet) Insert() {
 func (p *Puppet) Update() {
 	query := `
 		UPDATE puppet SET name=$1, name_set=$2, avatar=$3, avatar_url=$4, avatar_set=$5, contact_info_set=$6,
-		                  global_name=$7, username=$8, discriminator=$9, is_bot=$10, is_webhook=$11,
+		                  global_name=$7, username=$8, discriminator=$9, is_bot=$10, is_webhook=$11, is_application=$12,
-		                  custom_mxid=$12, access_token=$13, next_batch=$14
+		                  custom_mxid=$13, access_token=$14, next_batch=$15
-		WHERE id=$15
+		WHERE id=$16
 	`
 	_, err := p.db.Exec(
 		query,
 		p.Name, p.NameSet, p.Avatar, p.AvatarURL.String(), p.AvatarSet, p.ContactInfoSet,
-		p.GlobalName, p.Username, p.Discriminator, p.IsBot, p.IsWebhook,
+		p.GlobalName, p.Username, p.Discriminator, p.IsBot, p.IsWebhook, p.IsApplication,
 		strPtr(p.CustomMXID), strPtr(p.AccessToken), strPtr(p.NextBatch),
 		p.ID,
 	)

database/reaction.go

@@ -4,10 +4,9 @@ import (
 	"database/sql"
 	"errors"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type ReactionQuery struct {

database/role.go

@@ -4,11 +4,9 @@ import (
 	"database/sql"
 	"errors"
 
-	log "maunium.net/go/maulogger/v2"
-
-	"maunium.net/go/mautrix/util/dbutil"
-
 	"github.com/bwmarrin/discordgo"
+	"go.mau.fi/util/dbutil"
+	log "maunium.net/go/maulogger/v2"
 )
 
 type RoleQuery struct {

database/thread.go

@@ -4,10 +4,9 @@ import (
 	"database/sql"
 	"errors"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type ThreadQuery struct {

database/upgrades/00-latest-revision.sql

@@ -1,4 +1,4 @@
--- v0 -> v22 (compatible with v19+): Latest revision
+-- v0 -> v23 (compatible with v19+): Latest revision
 
 CREATE TABLE guild (
     dcid       TEXT PRIMARY KEY,
@@ -71,11 +71,12 @@ CREATE TABLE puppet (
 
     contact_info_set BOOLEAN NOT NULL DEFAULT false,
 
-    global_name   TEXT NOT NULL DEFAULT '',
+    global_name    TEXT NOT NULL DEFAULT '',
-    username      TEXT NOT NULL DEFAULT '',
+    username       TEXT NOT NULL DEFAULT '',
-    discriminator TEXT NOT NULL DEFAULT '',
+    discriminator  TEXT NOT NULL DEFAULT '',
-    is_bot        BOOLEAN NOT NULL DEFAULT false,
+    is_bot         BOOLEAN NOT NULL DEFAULT false,
-    is_webhook    BOOLEAN NOT NULL DEFAULT false,
+    is_webhook     BOOLEAN NOT NULL DEFAULT false,
+    is_application BOOLEAN NOT NULL DEFAULT false,
 
     custom_mxid  TEXT,
     access_token TEXT,

database/upgrades/23-puppet-is-application.sql

@@ -0,0 +1,2 @@
+-- v23 (compatible with v19+): Store is application status for puppets
+ALTER TABLE puppet ADD COLUMN is_application BOOLEAN NOT NULL DEFAULT false;

database/upgrades/upgrades.go

@@ -19,7 +19,7 @@ package upgrades
 import (
 	"embed"
 
-	"maunium.net/go/mautrix/util/dbutil"
+	"go.mau.fi/util/dbutil"
 )
 
 var Table dbutil.UpgradeTable

database/user.go

@@ -3,10 +3,9 @@ package database
 import (
 	"database/sql"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 type UserQuery struct {

database/userportal.go

@@ -5,8 +5,8 @@ import (
 	"errors"
 	"time"
 
+	"go.mau.fi/util/dbutil"
 	log "maunium.net/go/maulogger/v2"
-	"maunium.net/go/mautrix/util/dbutil"
 )
 
 const (

example-config.yaml

@@ -92,7 +92,8 @@ bridge:
     #   .Discriminator - The 4 numbers after the name on Discord
     #   .Bot - Whether the user is a bot
     #   .System - Whether the user is an official system user
-    #   .Webhook - Whether the user is a webhook
+    #   .Webhook - Whether the user is a webhook and is not an application
+    #   .Application - Whether the user is an application
     displayname_template: '{{or .GlobalName .Username}}{{if .Bot}} (bot){{end}}'
     # Displayname template for Discord channels (bridged as rooms, or spaces when type=4).
     # Available variables:
@@ -232,6 +233,7 @@ bridge:
             initial:
                 dm: 0
                 channel: 0
+                thread: 0
             # Missed message backfill (on startup).
             # 0 means backfill is disabled, -1 means fetch all messages since last bridged message.
             # When using unlimited backfill (-1), messages are backfilled as they are fetched.
@@ -239,6 +241,7 @@ bridge:
             missed:
                 dm: 0
                 channel: 0
+                thread: 0
         # Maximum members in a guild to enable backfilling. Set to -1 to disable limit.
         # This can be used as a rough heuristic to disable backfilling in channels that are too active.
         # Currently only applies to missed message backfill.
@@ -279,6 +282,10 @@ bridge:
             delete_on_device_delete: false
             # Periodically delete megolm sessions when 2x max_age has passed since receiving the session.
             periodically_delete_expired: false
+            # Delete inbound megolm sessions that don't have the received_at field used for
+            # automatic ratcheting and expired session deletion. This is meant as a migration
+            # to delete old keys prior to the bridge update.
+            delete_outdated_inbound: false
         # What level of device verification should be required from users?
         #
         # Valid levels:
@@ -325,6 +332,8 @@ bridge:
         # Shared secret for authentication. If set to "generate", a random secret will be generated,
         # or if set to "disable", the provisioning API will be disabled.
         shared_secret: generate
+        # Enable debug API at /debug with provisioning authentication.
+        debug_endpoints: false
 
     # Permissions for using the bridge.
     # Permitted values:

formatter.go

@@ -26,13 +26,12 @@ import (
 	"github.com/yuin/goldmark/extension"
 	"github.com/yuin/goldmark/parser"
 	"github.com/yuin/goldmark/util"
+	"go.mau.fi/util/variationselector"
 	"golang.org/x/exp/slices"
-
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/format"
 	"maunium.net/go/mautrix/format/mdext"
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util/variationselector"
 )
 
 // escapeFixer is a hacky partial fix for the difference in escaping markdown, used with escapeReplacement
@@ -156,11 +155,14 @@ func (br *DiscordBridge) pillConverter(displayname, mxid, eventID string, ctx fo
 	return displayname
 }
 
+const discordLinkPattern = `https?://[^<\p{Zs}\x{feff}]*[^"'),.:;\]\p{Zs}\x{feff}]`
+
 // Discord links start with http:// or https://, contain at least two characters afterwards,
 // don't contain < or whitespace anywhere, and don't end with "'),.:;]
 //
 // Zero-width whitespace is mostly in the Format category and is allowed, except \uFEFF isn't for some reason
-var discordLinkRegex = regexp.MustCompile(`https?://[^<\p{Zs}\x{feff}]*[^"'),.:;\]\p{Zs}\x{feff}]`)
+var discordLinkRegex = regexp.MustCompile(discordLinkPattern)
+var discordLinkRegexFull = regexp.MustCompile("^" + discordLinkPattern + "$")
 
 var discordMarkdownEscaper = strings.NewReplacer(
 	`\`, `\\`,
@@ -214,6 +216,14 @@ var matrixHTMLParser = &format.HTMLParser{
 		}
 		return fmt.Sprintf("||%s||", text)
 	},
+	LinkConverter: func(text, href string, ctx format.Context) string {
+		if text == href {
+			return text
+		} else if !discordLinkRegexFull.MatchString(href) {
+			return fmt.Sprintf("%s (%s)", escapeDiscordMarkdown(text), escapeDiscordMarkdown(href))
+		}
+		return fmt.Sprintf("[%s](%s)", escapeDiscordMarkdown(text), href)
+	},
 }
 
 func (portal *Portal) parseMatrixHTML(content *event.MessageEventContent) (string, *discordgo.MessageAllowedMentions) {

go.mod

@@ -1,41 +1,43 @@
 module go.mau.fi/mautrix-discord
 
-go 1.19
+go 1.20
 
 require (
 	github.com/bwmarrin/discordgo v0.27.0
-	github.com/gabriel-vasile/mimetype v1.4.2
+	github.com/gabriel-vasile/mimetype v1.4.3
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/lib/pq v1.10.9
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.19
-	github.com/rs/zerolog v1.29.1
+	github.com/rs/zerolog v1.31.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.4
-	github.com/yuin/goldmark v1.5.4
+	github.com/yuin/goldmark v1.6.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
+	go.mau.fi/util v0.2.1
+	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848
+	golang.org/x/sync v0.5.0
 	maunium.net/go/maulogger/v2 v2.4.1
-	maunium.net/go/mautrix v0.15.3
+	maunium.net/go/mautrix v0.16.2
 )
 
 require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/tidwall/gjson v1.14.4 // indirect
+	github.com/tidwall/gjson v1.17.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	go.mau.fi/zeroconfig v0.1.2 // indirect
-	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
-	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	maunium.net/go/mauflag v1.0.0 // indirect
 )
 
-replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20230512133900-5b12693331c0
+replace github.com/bwmarrin/discordgo => github.com/beeper/discordgo v0.0.0-20231013182643-f333f2578a3c

go.sum

@@ -1,12 +1,12 @@
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
-github.com/beeper/discordgo v0.0.0-20230512133900-5b12693331c0 h1:ECBEbC4ruaXzcVJJ4UurkGpT/Xlm9ZnwsHiHn9gjPZw=
+github.com/beeper/discordgo v0.0.0-20231013182643-f333f2578a3c h1:WaJ9eX8eyOBHD8te5t7xzm27uwhfaN94o8vUVFXliyA=
-github.com/beeper/discordgo v0.0.0-20230512133900-5b12693331c0/go.mod h1:59+AOzzjmL6onAh62nuLXmn7dJCaC/owDLWbGtjTcFA=
+github.com/beeper/discordgo v0.0.0-20231013182643-f333f2578a3c/go.mod h1:59+AOzzjmL6onAh62nuLXmn7dJCaC/owDLWbGtjTcFA=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -16,45 +16,51 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
-github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
-github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
+github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68=
-github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw=
+go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c=
 go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
 go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
-golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
-golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
@@ -65,5 +71,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
-maunium.net/go/mautrix v0.15.3 h1:C9BHSUM0gYbuZmAtopuLjIcH5XHLb/ZjTEz7nN+0jN0=
+maunium.net/go/mautrix v0.16.2 h1:a6GUJXNWsTEOO8VE4dROBfCIfPp50mqaqzv7KPzChvg=
-maunium.net/go/mautrix v0.15.3/go.mod h1:zLrQqdxJlLkurRCozTc9CL6FySkgZlO/kpCYxBILSLE=
+maunium.net/go/mautrix v0.16.2/go.mod h1:YL4l4rZB46/vj/ifRMEjcibbvHjgxHftOF1SgmruLu4=

guildportal.go

@@ -298,7 +298,7 @@ func (guild *Guild) cleanup() {
 		return
 	}
 	intent := guild.bridge.Bot
-	if guild.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] {
+	if guild.bridge.SpecVersions.Supports(mautrix.BeeperFeatureRoomYeeting) {
 		err := intent.BeeperDeleteRoom(guild.MXID)
 		if err != nil && !errors.Is(err, mautrix.MNotFound) {
 			guild.log.Errorfln("Failed to delete %s using hungryserv yeet endpoint: %v", guild.MXID, err)

main.go

@@ -20,11 +20,12 @@ import (
 	_ "embed"
 	"sync"
 
+	"go.mau.fi/util/configupgrade"
+	"go.mau.fi/util/exsync"
+	"golang.org/x/sync/semaphore"
 	"maunium.net/go/mautrix/bridge"
 	"maunium.net/go/mautrix/bridge/commands"
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util"
-	"maunium.net/go/mautrix/util/configupgrade"
 
 	"go.mau.fi/mautrix-discord/config"
 	"go.mau.fi/mautrix-discord/database"
@@ -73,7 +74,8 @@ type DiscordBridge struct {
 	puppetsByCustomMXID map[id.UserID]*Puppet
 	puppetsLock         sync.Mutex
 
-	attachmentTransfers *util.SyncMap[attachmentKey, *util.ReturnableOnce[*database.File]]
+	attachmentTransfers         *exsync.Map[attachmentKey, *exsync.ReturnableOnce[*database.File]]
+	parallelAttachmentSemaphore *semaphore.Weighted
 }
 
 func (br *DiscordBridge) GetExampleConfig() string {
@@ -170,13 +172,14 @@ func main() {
 		puppets:             make(map[string]*Puppet),
 		puppetsByCustomMXID: make(map[id.UserID]*Puppet),
 
-		attachmentTransfers: util.NewSyncMap[attachmentKey, *util.ReturnableOnce[*database.File]](),
+		attachmentTransfers:         exsync.NewMap[attachmentKey, *exsync.ReturnableOnce[*database.File]](),
+		parallelAttachmentSemaphore: semaphore.NewWeighted(3),
 	}
 	br.Bridge = bridge.Bridge{
 		Name:              "mautrix-discord",
 		URL:               "https://github.com/mautrix/discord",
 		Description:       "A Matrix-Discord puppeting bridge.",
-		Version:           "0.5.0",
+		Version:           "0.6.5",
 		ProtocolName:      "Discord",
 		BeeperServiceName: "discordgo",
 		BeeperNetworkName: "discord",

portal.go

@@ -18,6 +18,8 @@ import (
 	"github.com/bwmarrin/discordgo"
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/rs/zerolog"
+	"go.mau.fi/util/exsync"
+	"go.mau.fi/util/variationselector"
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/bridge"
@@ -26,8 +28,6 @@ import (
 	"maunium.net/go/mautrix/crypto/attachment"
 	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
-	"maunium.net/go/mautrix/util"
-	"maunium.net/go/mautrix/util/variationselector"
 
 	"go.mau.fi/mautrix-discord/config"
 	"go.mau.fi/mautrix-discord/database"
@@ -62,7 +62,7 @@ type Portal struct {
 	discordMessages chan portalDiscordMessage
 	matrixMessages  chan portalMatrixMessage
 
-	recentMessages *util.RingBuffer[string, *discordgo.Message]
+	recentMessages *exsync.RingBuffer[string, *discordgo.Message]
 
 	commands     map[string]*discordgo.ApplicationCommand
 	commandsLock sync.RWMutex
@@ -260,7 +260,7 @@ func (br *DiscordBridge) NewPortal(dbPortal *database.Portal) *Portal {
 		discordMessages: make(chan portalDiscordMessage, br.Config.Bridge.PortalMessageBuffer),
 		matrixMessages:  make(chan portalMatrixMessage, br.Config.Bridge.PortalMessageBuffer),
 
-		recentMessages: util.NewRingBuffer[string, *discordgo.Message](recentMessageBufferSize),
+		recentMessages: exsync.NewRingBuffer[string, *discordgo.Message](recentMessageBufferSize),
 
 		commands: make(map[string]*discordgo.ApplicationCommand),
 	}
@@ -541,7 +541,7 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 		portal.Update()
 	}
 
-	go portal.forwardBackfillInitial(user)
+	go portal.forwardBackfillInitial(user, nil)
 	backfillStarted = true
 
 	return nil
@@ -549,13 +549,15 @@ func (portal *Portal) CreateMatrixRoom(user *User, channel *discordgo.Channel) e
 
 func (portal *Portal) handleDiscordMessages(msg portalDiscordMessage) {
 	if portal.MXID == "" {
-		_, ok := msg.msg.(*discordgo.MessageCreate)
+		msgCreate, ok := msg.msg.(*discordgo.MessageCreate)
 		if !ok {
 			portal.log.Warn().Msg("Can't create Matrix room from non new message event")
 			return
 		}
 
-		portal.log.Debug().Msg("Creating Matrix room from incoming message")
+		portal.log.Debug().
+			Str("message_id", msgCreate.ID).
+			Msg("Creating Matrix room from incoming message")
 		if err := portal.CreateMatrixRoom(msg.user, nil); err != nil {
 			portal.log.Err(err).Msg("Failed to create portal room")
 			return
@@ -586,7 +588,7 @@ func (portal *Portal) ensureUserInvited(user *User, ignoreCache bool) bool {
 	return user.ensureInvited(portal.MainIntent(), portal.MXID, portal.IsPrivateChat(), ignoreCache)
 }
 
-func (portal *Portal) markMessageHandled(discordID string, authorID string, timestamp time.Time, threadID string, senderMXID id.UserID, parts []database.MessagePart) {
+func (portal *Portal) markMessageHandled(discordID string, authorID string, timestamp time.Time, threadID string, senderMXID id.UserID, parts []database.MessagePart) *database.Message {
 	msg := portal.bridge.DB.Message.New()
 	msg.Channel = portal.Key
 	msg.DiscordID = discordID
@@ -595,6 +597,9 @@ func (portal *Portal) markMessageHandled(discordID string, authorID string, time
 	msg.ThreadID = threadID
 	msg.SenderMXID = senderMXID
 	msg.MassInsertParts(parts)
+	msg.MXID = parts[0].MXID
+	msg.AttachmentID = parts[0].AttachmentID
+	return msg
 }
 
 func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Message, thread *Thread) {
@@ -619,10 +624,10 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 		log.Debug().Msg("Dropping duplicate message")
 		return
 	}
-	log.Debug().Msg("Starting handling of Discord message")
 
+	handlingStartTime := time.Now()
 	puppet := portal.bridge.GetPuppetByID(msg.Author.ID)
-	puppet.UpdateInfo(user, msg.Author, msg.WebhookID)
+	puppet.UpdateInfo(user, msg.Author, msg)
 	intent := puppet.IntentFor(portal)
 
 	var discordThreadID string
@@ -642,6 +647,7 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 	ts, _ := discordgo.SnowflakeTimestamp(msg.ID)
 	parts := portal.convertDiscordMessage(ctx, puppet, intent, msg)
 	dbParts := make([]database.MessagePart, 0, len(parts))
+	eventIDs := zerolog.Dict()
 	for i, part := range parts {
 		if (replyTo != nil || threadRootEvent != "") && part.Content.RelatesTo == nil {
 			part.Content.RelatesTo = &event.RelatesTo{}
@@ -672,13 +678,20 @@ func (portal *Portal) handleDiscordMessageCreate(user *User, msg *discordgo.Mess
 		}
 		lastThreadEvent = resp.EventID
 		dbParts = append(dbParts, database.MessagePart{AttachmentID: part.AttachmentID, MXID: resp.EventID})
+		eventIDs.Str(part.AttachmentID, resp.EventID.String())
 	}
+
+	log = log.With().Dur("handling_time", time.Since(handlingStartTime)).Logger()
 	if len(parts) == 0 {
 		log.Warn().Msg("Unhandled message")
 	} else if len(dbParts) == 0 {
 		log.Warn().Msg("All parts of message failed to send to Matrix")
 	} else {
-		portal.markMessageHandled(msg.ID, msg.Author.ID, ts, discordThreadID, intent.UserID, dbParts)
+		log.Debug().Dict("event_ids", eventIDs).Msg("Finished handling Discord message")
+		firstDBMessage := portal.markMessageHandled(msg.ID, msg.Author.ID, ts, discordThreadID, intent.UserID, dbParts)
+		if msg.Flags == discordgo.MessageFlagsHasThread {
+			portal.bridge.threadFound(ctx, user, firstDBMessage, msg.ID, msg.Thread)
+		}
 	}
 }
 
@@ -702,12 +715,8 @@ func (portal *Portal) getReplyTarget(source *User, threadID string, ref *discord
 	if ref == nil {
 		return nil
 	}
-	isHungry := portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry
-	if !isHungry {
-		allowNonExistent = false
-	}
 	// TODO add config option for cross-room replies
-	crossRoomReplies := isHungry
+	crossRoomReplies := portal.bridge.Config.Homeserver.Software == bridgeconfig.SoftwareHungry
 
 	targetPortal := portal
 	if ref.ChannelID != portal.Key.ChannelID && ref.ChannelID != threadID && crossRoomReplies {
@@ -807,11 +816,7 @@ func (portal *Portal) handleDiscordMessageUpdate(user *User, msg *discordgo.Mess
 	}
 
 	if msg.Flags == discordgo.MessageFlagsHasThread {
-		thread := portal.bridge.GetThreadByID(msg.ID, existing[0])
+		portal.bridge.threadFound(ctx, user, existing[0], msg.ID, msg.Thread)
-		log.Debug().Msg("Marked message as thread root")
-		if thread.CreationNoticeMXID == "" {
-			portal.sendThreadCreationNotice(ctx, thread)
-		}
 	}
 
 	if msg.Author == nil {
@@ -849,6 +854,7 @@ func (portal *Portal) handleDiscordMessageUpdate(user *User, msg *discordgo.Mess
 	puppet := portal.bridge.GetPuppetByID(msg.Author.ID)
 	intent := puppet.IntentFor(portal)
 
+	redactions := zerolog.Dict()
 	attachmentMap := map[string]*database.Message{}
 	for _, existingPart := range existing {
 		if existingPart.AttachmentID != "" {
@@ -876,11 +882,13 @@ func (portal *Portal) handleDiscordMessageUpdate(user *User, msg *discordgo.Mess
 		}
 	}
 	for _, deletedAttachment := range attachmentMap {
-		_, err := intent.RedactEvent(portal.MXID, deletedAttachment.MXID)
+		resp, err := intent.RedactEvent(portal.MXID, deletedAttachment.MXID)
 		if err != nil {
-			log.Warn().Err(err).
+			log.Err(err).
 				Str("event_id", deletedAttachment.MXID.String()).
 				Msg("Failed to redact attachment")
+		} else {
+			redactions.Str(deletedAttachment.AttachmentID, resp.EventID.String())
 		}
 		deletedAttachment.Delete()
 	}
@@ -928,6 +936,10 @@ func (portal *Portal) handleDiscordMessageUpdate(user *User, msg *discordgo.Mess
 	if msg.EditedTimestamp != nil {
 		existing[0].UpdateEditTimestamp(*msg.EditedTimestamp)
 	}
+	log.Debug().
+		Str("event_id", resp.EventID.String()).
+		Dict("redacted_attachments", redactions).
+		Msg("Finished handling Discord edit")
 }
 
 func (portal *Portal) handleDiscordMessageDelete(user *User, msg *discordgo.Message) {
@@ -992,7 +1004,7 @@ func (portal *Portal) handleDiscordTyping(evt *discordgo.TypingStart) {
 
 func (portal *Portal) syncParticipant(source *User, participant *discordgo.User, remove bool) {
 	puppet := portal.bridge.GetPuppetByID(participant.ID)
-	puppet.UpdateInfo(source, participant, "")
+	puppet.UpdateInfo(source, participant, nil)
 	log := portal.log.With().
 		Str("participant_id", participant.ID).
 		Str("ghost_mxid", puppet.MXID.String()).
@@ -1019,11 +1031,14 @@ func (portal *Portal) syncParticipant(source *User, participant *discordgo.User,
 func (portal *Portal) syncParticipants(source *User, participants []*discordgo.User) {
 	for _, participant := range participants {
 		puppet := portal.bridge.GetPuppetByID(participant.ID)
-		puppet.UpdateInfo(source, participant, "")
+		puppet.UpdateInfo(source, participant, nil)
 
-		user := portal.bridge.GetUserByID(participant.ID)
+		var user *User
-		if user != nil {
+		if participant.ID != portal.OtherUserID {
-			portal.ensureUserInvited(user, false)
+			user = portal.bridge.GetUserByID(participant.ID)
+			if user != nil {
+				portal.ensureUserInvited(user, false)
+			}
 		}
 
 		if user == nil || !puppet.IntentFor(portal).IsCustomPuppet {
@@ -1099,7 +1114,7 @@ func (portal *Portal) getEvent(mxid id.EventID) (*event.Event, error) {
 	if evt.Type == event.EventEncrypted {
 		decryptedEvt, err := portal.bridge.Crypto.Decrypt(evt)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to decrypt event: %w", err)
 		} else {
 			evt = decryptedEvt
 		}
@@ -1160,7 +1175,7 @@ func (portal *Portal) startThreadFromMatrix(sender *User, threadRoot id.EventID)
 	}
 }
 
-func (portal *Portal) sendErrorMessage(msgType, message string, confirmed bool) id.EventID {
+func (portal *Portal) sendErrorMessage(evt *event.Event, msgType, message string, confirmed bool) id.EventID {
 	if !portal.bridge.Config.Bridge.MessageErrorNotices {
 		return ""
 	}
@@ -1168,10 +1183,18 @@ func (portal *Portal) sendErrorMessage(msgType, message string, confirmed bool)
 	if confirmed {
 		certainty = "was not"
 	}
-	resp, err := portal.sendMatrixMessage(portal.MainIntent(), event.EventMessage, &event.MessageEventContent{
+	if portal.RelayWebhookSecret != "" {
+		message = strings.ReplaceAll(message, portal.RelayWebhookSecret, "<redacted>")
+	}
+	content := &event.MessageEventContent{
 		MsgType: event.MsgNotice,
 		Body:    fmt.Sprintf("\u26a0 Your %s %s bridged: %v", msgType, certainty, message),
-	}, nil, 0)
+	}
+	relatable, ok := evt.Content.Parsed.(event.Relatable)
+	if ok && relatable.OptionalGetRelatesTo().GetThreadParent() != "" {
+		content.GetRelatesTo().SetThread(relatable.OptionalGetRelatesTo().GetThreadParent(), evt.ID)
+	}
+	resp, err := portal.sendMatrixMessage(portal.MainIntent(), event.EventMessage, content, nil, 0)
 	if err != nil {
 		portal.log.Warn().Err(err).Msg("Failed to send bridging error message")
 		return ""
@@ -1336,7 +1359,7 @@ func (portal *Portal) sendMessageMetrics(evt *event.Event, err error, part strin
 			if humanMessage == "" {
 				humanMessage = err.Error()
 			}
-			portal.sendErrorMessage(msgType, humanMessage, isCertain)
+			portal.sendErrorMessage(evt, msgType, humanMessage, isCertain)
 		}
 		portal.sendStatusEvent(evt.ID, err)
 	} else {
@@ -1387,13 +1410,9 @@ func cutBody(body string) string {
 }
 
 func (portal *Portal) convertReplyMessageToEmbed(eventID id.EventID, url string) (*discordgo.MessageEmbed, error) {
-	evt, err := portal.MainIntent().GetEvent(portal.MXID, eventID)
+	evt, err := portal.getEvent(eventID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to fetch event: %w", err)
+		return nil, fmt.Errorf("failed to get reply target event: %w", err)
-	}
-	err = evt.Content.ParseRaw(evt.Type)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse event content: %w", err)
 	}
 	content, ok := evt.Content.Parsed.(*event.MessageEventContent)
 	if !ok {
@@ -1463,9 +1482,10 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
 		}
 		return
 	} else if threadRoot := content.GetRelatesTo().GetThreadParent(); threadRoot != "" {
-		existingThread := portal.bridge.DB.Thread.GetByMatrixRootMsg(threadRoot)
+		existingThread := portal.bridge.GetThreadByRootMXID(threadRoot)
 		if existingThread != nil {
 			threadID = existingThread.ID
+			existingThread.initialBackfillAttempted = true
 		} else {
 			if isWebhookSend {
 				// TODO start thread with bot?
@@ -1518,6 +1538,9 @@ func (portal *Portal) handleMatrixMessage(sender *User, evt *event.Event) {
 	switch content.MsgType {
 	case event.MsgText, event.MsgEmote, event.MsgNotice:
 		sendReq.Content, sendReq.AllowedMentions = portal.parseMatrixHTML(content)
+		if content.MsgType == event.MsgEmote {
+			sendReq.Content = fmt.Sprintf("_%s_", sendReq.Content)
+		}
 	case event.MsgAudio, event.MsgFile, event.MsgImage, event.MsgVideo:
 		data, err := downloadMatrixAttachment(portal.MainIntent(), content)
 		if err != nil {
@@ -1700,7 +1723,7 @@ func (portal *Portal) cleanup(puppetsOnly bool) {
 		return
 	}
 	intent := portal.MainIntent()
-	if portal.bridge.SpecVersions.UnstableFeatures["com.beeper.room_yeeting"] {
+	if portal.bridge.SpecVersions.Supports(mautrix.BeeperFeatureRoomYeeting) {
 		err := intent.BeeperDeleteRoom(portal.MXID)
 		if err != nil && !errors.Is(err, mautrix.MNotFound) {
 			portal.log.Err(err).Msg("Failed to delete room using hungryserv yeet endpoint")
@@ -1848,7 +1871,7 @@ func (portal *Portal) handleMatrixReaction(sender *User, evt *event.Event) {
 func (portal *Portal) handleDiscordReaction(user *User, reaction *discordgo.MessageReaction, add bool, thread *Thread, member *discordgo.Member) {
 	puppet := portal.bridge.GetPuppetByID(reaction.UserID)
 	if member != nil {
-		puppet.UpdateInfo(user, member.User, "")
+		puppet.UpdateInfo(user, member.User, nil)
 	}
 	intent := puppet.IntentFor(portal)
 

portal_convert.go

@@ -82,6 +82,9 @@ func (portal *Portal) convertDiscordFile(ctx context.Context, typeName string, i
 }
 
 func (portal *Portal) cleanupConvertedStickerInfo(content *event.MessageEventContent) {
+	if content.Info == nil {
+		return
+	}
 	if content.Info.Width == 0 && content.Info.Height == 0 {
 		content.Info.Width = DiscordStickerSize
 		content.Info.Height = DiscordStickerSize
@@ -198,8 +201,18 @@ func (portal *Portal) convertDiscordVideoEmbed(ctx context.Context, intent *apps
 	var proxyURL string
 	if embed.Video != nil {
 		proxyURL = embed.Video.ProxyURL
-	} else {
+	} else if embed.Thumbnail != nil {
 		proxyURL = embed.Thumbnail.ProxyURL
+	} else {
+		zerolog.Ctx(ctx).Warn().Str("embed_url", embed.URL).Msg("No video or thumbnail proxy URL found in embed")
+		return &ConvertedMessage{
+			AttachmentID: attachmentID,
+			Type:         event.EventMessage,
+			Content: &event.MessageEventContent{
+				Body:    "Failed to bridge media: no video or thumbnail proxy URL found in embed",
+				MsgType: event.MsgNotice,
+			},
+		}
 	}
 	dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, proxyURL, portal.Encrypted, NoMeta)
 	if err != nil {
@@ -308,6 +321,12 @@ func (portal *Portal) convertDiscordMessage(ctx context.Context, puppet *Puppet,
 			parts = append(parts, part)
 		}
 	}
+	if len(parts) == 0 && msg.Thread != nil {
+		parts = append(parts, &ConvertedMessage{Type: event.EventMessage, Content: &event.MessageEventContent{
+			MsgType: event.MsgText,
+			Body:    fmt.Sprintf("Created a thread: %s", msg.Thread.Name),
+		}})
+	}
 	for _, part := range parts {
 		puppet.addWebhookMeta(part, msg)
 		puppet.addMemberMeta(part, msg)
@@ -323,24 +342,20 @@ func (puppet *Puppet) addMemberMeta(part *ConvertedMessage, msg *discordgo.Messa
 		part.Extra = make(map[string]any)
 	}
 	var avatarURL id.ContentURI
+	var discordAvatarURL string
 	if msg.Member.Avatar != "" {
 		var err error
-		avatarURL, err = puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), msg.GuildID, msg.Author.ID, msg.Author.Avatar)
+		avatarURL, discordAvatarURL, err = puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), msg.GuildID, msg.Author.ID, msg.Author.Avatar)
 		if err != nil {
 			puppet.log.Warn().Err(err).
 				Str("avatar_id", msg.Author.Avatar).
 				Msg("Failed to reupload guild user avatar")
 		}
 	}
-	var discordAvararURL string
-	if msg.Member.Avatar != "" {
-		msg.Member.User = msg.Author
-		discordAvararURL = msg.Member.AvatarURL("")
-	}
 	part.Extra["fi.mau.discord.guild_member_metadata"] = map[string]any{
 		"nick":       msg.Member.Nick,
 		"avatar_id":  msg.Member.Avatar,
-		"avatar_url": discordAvararURL,
+		"avatar_url": discordAvatarURL,
 		"avatar_mxc": avatarURL.String(),
 	}
 	if msg.Member.Nick != "" || !avatarURL.IsEmpty() {
@@ -370,7 +385,7 @@ func (puppet *Puppet) addWebhookMeta(part *ConvertedMessage, msg *discordgo.Mess
 	var avatarURL id.ContentURI
 	if msg.Author.Avatar != "" {
 		var err error
-		avatarURL, err = puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), "", msg.Author.ID, msg.Author.Avatar)
+		avatarURL, _, err = puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), "", msg.Author.ID, msg.Author.Avatar)
 		if err != nil {
 			puppet.log.Warn().Err(err).
 				Str("avatar_id", msg.Author.Avatar).
@@ -616,9 +631,14 @@ func getEmbedType(msg *discordgo.Message, embed *discordgo.MessageEmbed) BridgeE
 }
 
 func isPlainGifMessage(msg *discordgo.Message) bool {
-	return len(msg.Embeds) == 1 && msg.Embeds[0].URL == msg.Content &&
+	if len(msg.Embeds) != 1 {
-		((msg.Embeds[0].Type == discordgo.EmbedTypeGifv && msg.Embeds[0].Video != nil) ||
+		return false
-			(msg.Embeds[0].Type == discordgo.EmbedTypeImage && msg.Embeds[0].Image == nil && msg.Embeds[0].Thumbnail != nil))
+	}
+	embed := msg.Embeds[0]
+	isGifVideo := embed.Type == discordgo.EmbedTypeGifv && embed.Video != nil
+	isGifImage := embed.Type == discordgo.EmbedTypeImage && embed.Image == nil && embed.Thumbnail != nil
+	contentIsOnlyURL := msg.Content == embed.URL || discordLinkRegexFull.MatchString(msg.Content)
+	return contentIsOnlyURL && (isGifVideo || isGifImage)
 }
 
 func (portal *Portal) convertDiscordMentions(msg *discordgo.Message, syncGhosts bool) *event.Mentions {
@@ -626,7 +646,7 @@ func (portal *Portal) convertDiscordMentions(msg *discordgo.Message, syncGhosts
 	for _, mention := range msg.Mentions {
 		puppet := portal.bridge.GetPuppetByID(mention.ID)
 		if syncGhosts {
-			puppet.UpdateInfo(nil, mention, "")
+			puppet.UpdateInfo(nil, mention, nil)
 		}
 		user := portal.bridge.GetUserByID(mention.ID)
 		if user != nil {
@@ -659,11 +679,11 @@ func (portal *Portal) convertDiscordTextMessage(ctx context.Context, intent *app
 	var htmlParts []string
 	if msg.Interaction != nil {
 		puppet := portal.bridge.GetPuppetByID(msg.Interaction.User.ID)
-		puppet.UpdateInfo(nil, msg.Interaction.User, "")
+		puppet.UpdateInfo(nil, msg.Interaction.User, nil)
 		htmlParts = append(htmlParts, fmt.Sprintf(msgInteractionTemplateHTML, puppet.MXID, puppet.Name, msg.Interaction.Name))
 	}
 	if msg.Content != "" && !isPlainGifMessage(msg) {
-		htmlParts = append(htmlParts, portal.renderDiscordMarkdownOnlyHTML(msg.Content, false))
+		htmlParts = append(htmlParts, portal.renderDiscordMarkdownOnlyHTML(msg.Content, true))
 	}
 	previews := make([]*BeeperLinkPreview, 0)
 	for i, embed := range msg.Embeds {
@@ -706,7 +726,7 @@ func (portal *Portal) convertDiscordTextMessage(ctx context.Context, intent *app
 		"com.beeper.linkpreviews": previews,
 	}
 
-	if msg.WebhookID != "" && portal.bridge.Config.Bridge.PrefixWebhookMessages {
+	if msg.WebhookID != "" && msg.ApplicationID == "" && portal.bridge.Config.Bridge.PrefixWebhookMessages {
 		content.EnsureHasHTML()
 		content.Body = fmt.Sprintf("%s: %s", msg.Author.Username, content.Body)
 		content.FormattedBody = fmt.Sprintf("<strong>%s</strong>: %s", html.EscapeString(msg.Author.Username), content.FormattedBody)

provisioning.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"net"
 	"net/http"
+	_ "net/http/pprof"
 	"strings"
 	"time"
 
@@ -71,6 +72,13 @@ func newProvisioningAPI(br *DiscordBridge) *ProvisioningAPI {
 	r.HandleFunc("/v1/guilds/{guildID}", p.guildsBridge).Methods(http.MethodPost)
 	r.HandleFunc("/v1/guilds/{guildID}", p.guildsUnbridge).Methods(http.MethodDelete)
 
+	if p.bridge.Config.Bridge.Provisioning.DebugEndpoints {
+		p.log.Debugln("Enabling debug API at /debug")
+		r := p.bridge.AS.Router.PathPrefix("/debug").Subrouter()
+		r.Use(p.authMiddleware)
+		r.PathPrefix("/pprof").Handler(http.DefaultServeMux)
+	}
+
 	return p
 }
 

puppet.go

@@ -9,9 +9,9 @@ import (
 	"github.com/bwmarrin/discordgo"
 	"github.com/rs/zerolog"
 
+	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/bridge"
-	"maunium.net/go/mautrix/bridge/bridgeconfig"
 	"maunium.net/go/mautrix/id"
 
 	"go.mau.fi/mautrix-discord/database"
@@ -195,7 +195,7 @@ func (puppet *Puppet) updatePortalMeta(meta func(portal *Portal)) {
 }
 
 func (puppet *Puppet) UpdateName(info *discordgo.User) bool {
-	newName := puppet.bridge.Config.Bridge.FormatDisplayname(info, puppet.IsWebhook)
+	newName := puppet.bridge.Config.Bridge.FormatDisplayname(info, puppet.IsWebhook, puppet.IsApplication)
 	if puppet.Name == newName && puppet.NameSet {
 		return false
 	}
@@ -216,7 +216,7 @@ func (puppet *Puppet) UpdateName(info *discordgo.User) bool {
 	return true
 }
 
-func (br *DiscordBridge) reuploadUserAvatar(intent *appservice.IntentAPI, guildID, userID, avatarID string) (id.ContentURI, error) {
+func (br *DiscordBridge) reuploadUserAvatar(intent *appservice.IntentAPI, guildID, userID, avatarID string) (id.ContentURI, string, error) {
 	var downloadURL, ext string
 	if guildID == "" {
 		downloadURL = discordgo.EndpointUserAvatar(userID, avatarID)
@@ -233,17 +233,19 @@ func (br *DiscordBridge) reuploadUserAvatar(intent *appservice.IntentAPI, guildI
 			ext = "gif"
 		}
 	}
-	url := br.Config.Bridge.MediaPatterns.Avatar(userID, avatarID, ext)
+	if guildID == "" {
-	if !url.IsEmpty() {
+		url := br.Config.Bridge.MediaPatterns.Avatar(userID, avatarID, ext)
-		return url, nil
+		if !url.IsEmpty() {
+			return url, downloadURL, nil
+		}
 	}
 	copied, err := br.copyAttachmentToMatrix(intent, downloadURL, false, AttachmentMeta{
 		AttachmentID: fmt.Sprintf("avatar/%s/%s/%s", guildID, userID, avatarID),
 	})
 	if err != nil {
-		return url, err
+		return id.ContentURI{}, downloadURL, err
 	}
-	return copied.MXC, nil
+	return copied.MXC, downloadURL, nil
 }
 
 func (puppet *Puppet) UpdateAvatar(info *discordgo.User) bool {
@@ -260,7 +262,7 @@ func (puppet *Puppet) UpdateAvatar(info *discordgo.User) bool {
 	puppet.AvatarURL = id.ContentURI{}
 
 	if puppet.Avatar != "" && (puppet.AvatarURL.IsEmpty() || avatarChanged) {
-		url, err := puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), "", info.ID, puppet.Avatar)
+		url, _, err := puppet.bridge.reuploadUserAvatar(puppet.DefaultIntent(), "", info.ID, puppet.Avatar)
 		if err != nil {
 			puppet.log.Warn().Err(err).Str("avatar_id", puppet.Avatar).Msg("Failed to reupload user avatar")
 			return true
@@ -283,7 +285,7 @@ func (puppet *Puppet) UpdateAvatar(info *discordgo.User) bool {
 	return true
 }
 
-func (puppet *Puppet) UpdateInfo(source *User, info *discordgo.User, webhookID string) {
+func (puppet *Puppet) UpdateInfo(source *User, info *discordgo.User, message *discordgo.Message) {
 	puppet.syncLock.Lock()
 	defer puppet.syncLock.Unlock()
 
@@ -306,9 +308,24 @@ func (puppet *Puppet) UpdateInfo(source *User, info *discordgo.User, webhookID s
 	}
 
 	changed := false
-	if webhookID != "" && webhookID == info.ID && !puppet.IsWebhook {
+	if message != nil {
-		puppet.IsWebhook = true
+		if message.WebhookID != "" && message.ApplicationID == "" && !puppet.IsWebhook {
-		changed = true
+			puppet.log.Debug().
+				Str("message_id", message.ID).
+				Str("webhook_id", message.WebhookID).
+				Msg("Found webhook ID in message, marking ghost as a webhook")
+			puppet.IsWebhook = true
+			changed = true
+		}
+		if message.ApplicationID != "" && !puppet.IsApplication {
+			puppet.log.Debug().
+				Str("message_id", message.ID).
+				Str("application_id", message.ApplicationID).
+				Msg("Found application ID in message, marking ghost as an application")
+			puppet.IsApplication = true
+			puppet.IsWebhook = false
+			changed = true
+		}
 	}
 	changed = puppet.UpdateContactInfo(info) || changed
 	changed = puppet.UpdateName(info) || changed
@@ -345,12 +362,16 @@ func (puppet *Puppet) UpdateContactInfo(info *discordgo.User) bool {
 }
 
 func (puppet *Puppet) ResendContactInfo() {
-	if puppet.bridge.Config.Homeserver.Software != bridgeconfig.SoftwareHungry || puppet.ContactInfoSet {
+	if !puppet.bridge.SpecVersions.Supports(mautrix.BeeperFeatureArbitraryProfileMeta) || puppet.ContactInfoSet {
 		return
 	}
+	discordUsername := puppet.Username
+	if puppet.Discriminator != "0" {
+		discordUsername += "#" + puppet.Discriminator
+	}
 	contactInfo := map[string]any{
 		"com.beeper.bridge.identifiers": []string{
-			fmt.Sprintf("discord:%s#%s", puppet.Username, puppet.Discriminator),
+			fmt.Sprintf("discord:%s", discordUsername),
 		},
 		"com.beeper.bridge.remote_id":      puppet.ID,
 		"com.beeper.bridge.service":        puppet.bridge.BeeperServiceName,

thread.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"context"
 	"sync"
 	"time"
 
 	"github.com/bwmarrin/discordgo"
+	"github.com/rs/zerolog"
+	"golang.org/x/exp/slices"
 	"maunium.net/go/mautrix/id"
 
 	"go.mau.fi/mautrix-discord/database"
@@ -14,7 +17,8 @@ type Thread struct {
 	*database.Thread
 	Parent *Portal
 
-	creationNoticeLock sync.Mutex
+	creationNoticeLock       sync.Mutex
+	initialBackfillAttempted bool
 }
 
 func (br *DiscordBridge) GetThreadByID(id string, root *database.Message) *Thread {
@@ -74,12 +78,63 @@ func (br *DiscordBridge) loadThread(dbThread *database.Thread, id string, root *
 	return thread
 }
 
+func (br *DiscordBridge) threadFound(ctx context.Context, source *User, rootMessage *database.Message, id string, metadata *discordgo.Channel) {
+	thread := br.GetThreadByID(id, rootMessage)
+	log := zerolog.Ctx(ctx)
+	log.Debug().Msg("Marked message as thread root")
+	if thread.CreationNoticeMXID == "" {
+		thread.Parent.sendThreadCreationNotice(ctx, thread)
+	}
+	// TODO member_ids_preview is probably not guaranteed to contain the source user
+	if source != nil && metadata != nil && slices.Contains(metadata.MemberIDsPreview, source.DiscordID) && !source.IsInPortal(thread.ID) {
+		source.MarkInPortal(database.UserPortal{
+			DiscordID: thread.ID,
+			Type:      database.UserPortalTypeThread,
+			Timestamp: time.Now(),
+		})
+		if metadata.MessageCount > 0 {
+			go thread.maybeInitialBackfill(source)
+		} else {
+			thread.initialBackfillAttempted = true
+		}
+	}
+}
+
+func (thread *Thread) maybeInitialBackfill(source *User) {
+	if thread.initialBackfillAttempted || thread.Parent.bridge.Config.Bridge.Backfill.Limits.Initial.Thread == 0 {
+		return
+	}
+	thread.Parent.forwardBackfillLock.Lock()
+	if thread.Parent.bridge.DB.Message.GetLastInThread(thread.Parent.Key, thread.ID) != nil {
+		thread.Parent.forwardBackfillLock.Unlock()
+		return
+	}
+	thread.Parent.forwardBackfillInitial(source, thread)
+}
+
 func (thread *Thread) Join(user *User) {
 	if user.IsInPortal(thread.ID) {
 		return
 	}
 	log := user.log.With().Str("thread_id", thread.ID).Str("channel_id", thread.ParentID).Logger()
 	log.Debug().Msg("Joining thread")
+
+	var doBackfill, backfillStarted bool
+	if !thread.initialBackfillAttempted && thread.Parent.bridge.Config.Bridge.Backfill.Limits.Initial.Thread > 0 {
+		thread.Parent.forwardBackfillLock.Lock()
+		lastMessage := thread.Parent.bridge.DB.Message.GetLastInThread(thread.Parent.Key, thread.ID)
+		if lastMessage != nil {
+			thread.Parent.forwardBackfillLock.Unlock()
+		} else {
+			doBackfill = true
+			defer func() {
+				if !backfillStarted {
+					thread.Parent.forwardBackfillLock.Unlock()
+				}
+			}()
+		}
+	}
+
 	var err error
 	if user.Session.IsUser {
 		err = user.Session.ThreadJoinWithLocation(thread.ID, discordgo.ThreadJoinLocationContextMenu)
@@ -94,5 +149,9 @@ func (thread *Thread) Join(user *User) {
 			Type:      database.UserPortalTypeThread,
 			Timestamp: time.Now(),
 		})
+		if doBackfill {
+			go thread.Parent.forwardBackfillInitial(user, thread)
+			backfillStarted = true
+		}
 	}
 }

user.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/rand"
@@ -16,8 +17,7 @@ import (
 	"github.com/bwmarrin/discordgo"
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
-	"maunium.net/go/mautrix/util/dbutil"
+	"go.mau.fi/util/dbutil"
-
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/appservice"
 	"maunium.net/go/mautrix/bridge"
@@ -73,6 +73,9 @@ func (user *User) GetRemoteID() string {
 
 func (user *User) GetRemoteName() string {
 	if user.Session != nil && user.Session.State != nil && user.Session.State.User != nil {
+		if user.Session.State.User.Discriminator == "0" {
+			return fmt.Sprintf("@%s", user.Session.State.User.Username)
+		}
 		return fmt.Sprintf("%s#%s", user.Session.State.User.Username, user.Session.State.User.Discriminator)
 	}
 	return user.DiscordID
@@ -97,7 +100,7 @@ func discordToZeroLevel(level int) zerolog.Level {
 
 func init() {
 	discordgo.Logger = func(msgL, caller int, format string, a ...interface{}) {
-		discordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...)
+		discordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...) // zerolog-allow-msgf
 	}
 }
 
@@ -365,37 +368,6 @@ func (user *User) GetDMSpaceRoom() id.RoomID {
 	return user.getSpaceRoom(&user.DMSpaceRoom, "Direct Messages", "Your Discord direct messages", user.GetSpaceRoom())
 }
 
-func (user *User) tryAutomaticDoublePuppeting() {
-	user.Lock()
-	defer user.Unlock()
-
-	if !user.bridge.Config.CanAutoDoublePuppet(user.MXID) {
-		return
-	}
-
-	user.log.Debug().Msg("Checking if double puppeting needs to be enabled")
-
-	puppet := user.bridge.GetPuppetByID(user.DiscordID)
-	if puppet.CustomMXID != "" {
-		user.log.Debug().Msg("User already has double-puppeting enabled")
-		return
-	}
-
-	accessToken, err := puppet.loginWithSharedSecret(user.MXID)
-	if err != nil {
-		user.log.Warn().Err(err).Msg("Failed to login with shared secret")
-		return
-	}
-
-	err = puppet.SwitchCustomMXID(accessToken, user.MXID)
-	if err != nil {
-		puppet.log.Warn().Err(err).Msg("Failed to switch to auto-logined custom puppet")
-		return
-	}
-
-	user.log.Info().Msg("Successfully automatically enabled custom puppet")
-}
-
 func (user *User) ViewingChannel(portal *Portal) bool {
 	if portal.GuildID != "" || !user.Session.IsUser {
 		return false
@@ -576,7 +548,7 @@ func (user *User) Connect() error {
 	}
 	userDiscordLog := user.log.With().Str("component", "discordgo").Logger()
 	session.Logger = func(msgL, caller int, format string, a ...interface{}) {
-		userDiscordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...)
+		userDiscordLog.WithLevel(discordToZeroLevel(msgL)).Caller(caller+1).Msgf(strings.TrimSpace(format), a...) // zerolog-allow-msgf
 	}
 	if !session.IsUser {
 		session.Identify.Intents = BotIntents
@@ -585,7 +557,15 @@ func (user *User) Connect() error {
 
 	user.Session = session
 
-	return user.Session.Open()
+	for {
+		err = user.Session.Open()
+		if errors.Is(err, discordgo.ErrImmediateDisconnect) {
+			user.log.Warn().Err(err).Msg("Retrying initial connection in 5 seconds")
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		return err
+	}
 }
 
 func (user *User) eventHandlerSync(rawEvt any) {
@@ -650,6 +630,8 @@ func (user *User) eventHandler(rawEvt any) {
 		user.typingStartHandler(evt)
 	case *discordgo.InteractionSuccess:
 		user.interactionSuccessHandler(evt)
+	case *discordgo.ThreadListSync:
+		user.threadListSyncHandler(evt)
 	case *discordgo.Event:
 		// Ignore
 	default:
@@ -864,7 +846,7 @@ func (user *User) handlePrivateChannel(portal *Portal, meta *discordgo.Channel,
 		}
 	} else {
 		portal.UpdateInfo(user, meta)
-		portal.ForwardBackfillMissed(user, meta)
+		portal.ForwardBackfillMissed(user, meta.LastMessageID, nil)
 	}
 	user.MarkInPortal(database.UserPortal{
 		DiscordID: portal.Key.ChannelID,
@@ -954,8 +936,11 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp
 	guild.UpdateInfo(user, meta)
 	if len(meta.Channels) > 0 {
 		for _, ch := range meta.Channels {
+			if !user.channelIsBridgeable(ch) {
+				continue
+			}
 			portal := user.GetPortalByMeta(ch)
-			if guild.BridgingMode >= database.GuildBridgeEverything && portal.MXID == "" && user.channelIsBridgeable(ch) {
+			if guild.BridgingMode >= database.GuildBridgeEverything && portal.MXID == "" {
 				err := portal.CreateMatrixRoom(user, ch)
 				if err != nil {
 					user.log.Error().Err(err).
@@ -966,7 +951,7 @@ func (user *User) handleGuild(meta *discordgo.Guild, timestamp time.Time, isInSp
 			} else {
 				portal.UpdateInfo(user, ch)
 				if user.bridge.Config.Bridge.Backfill.MaxGuildMembers < 0 || meta.MemberCount < user.bridge.Config.Bridge.Backfill.MaxGuildMembers {
-					portal.ForwardBackfillMissed(user, ch)
+					portal.ForwardBackfillMissed(user, ch.LastMessageID, nil)
 				}
 			}
 		}
@@ -1018,6 +1003,10 @@ func (user *User) guildCreateHandler(g *discordgo.GuildCreate) {
 }
 
 func (user *User) guildDeleteHandler(g *discordgo.GuildDelete) {
+	if g.Unavailable {
+		user.log.Info().Str("guild_id", g.ID).Msg("Ignoring guild delete event with unavailable flag")
+		return
+	}
 	user.log.Info().Str("guild_id", g.ID).Msg("Got guild delete event")
 	user.MarkNotInPortal(g.ID)
 	guild := user.bridge.GetGuildByID(g.ID, false)
@@ -1038,6 +1027,30 @@ func (user *User) guildUpdateHandler(g *discordgo.GuildUpdate) {
 	user.handleGuild(g.Guild, time.Now(), user.IsInSpace(g.ID))
 }
 
+func (user *User) threadListSyncHandler(t *discordgo.ThreadListSync) {
+	for _, meta := range t.Threads {
+		log := user.log.With().
+			Str("action", "thread list sync").
+			Str("guild_id", t.GuildID).
+			Str("parent_id", meta.ParentID).
+			Str("thread_id", meta.ID).
+			Logger()
+		ctx := log.WithContext(context.Background())
+		thread := user.bridge.GetThreadByID(meta.ID, nil)
+		if thread == nil {
+			msg := user.bridge.DB.Message.GetByDiscordID(database.NewPortalKey(meta.ParentID, ""), meta.ID)
+			if len(msg) == 0 {
+				log.Debug().Msg("Found unknown thread in thread list sync and don't have message")
+			} else {
+				log.Debug().Msg("Found unknown thread in thread list sync for existing message, creating thread")
+				user.bridge.threadFound(ctx, user, msg[0], meta.ID, meta)
+			}
+		} else {
+			thread.Parent.ForwardBackfillMissed(user, meta.LastMessageID, thread)
+		}
+	}
+}
+
 func (user *User) channelCreateHandler(c *discordgo.ChannelCreate) {
 	if user.getGuildBridgingMode(c.GuildID) < database.GuildBridgeEverything {
 		user.log.Debug().
@@ -1169,11 +1182,21 @@ func (user *User) pushPortalMessage(msg interface{}, typeName, channelID, guildI
 		return
 	}
 
-	portal.discordMessages <- portalDiscordMessage{
+	wrappedMsg := portalDiscordMessage{
 		msg:    msg,
 		user:   user,
 		thread: thread,
 	}
+	select {
+	case portal.discordMessages <- wrappedMsg:
+	default:
+		user.log.Warn().
+			Str("discord_event", typeName).
+			Str("guild_id", guildID).
+			Str("channel_id", channelID).
+			Msg("Portal message buffer is full")
+		portal.discordMessages <- wrappedMsg
+	}
 }
 
 type CustomReadReceipt struct {
@@ -1409,6 +1432,8 @@ func (user *User) bridgeGuild(guildID string, everything bool) error {
 	}
 	if everything {
 		guild.BridgingMode = database.GuildBridgeEverything
+	} else {
+		guild.BridgingMode = database.GuildBridgeCreateOnMessage
 	}
 	guild.Update()