From a7864c28d848a4dee38f1194645d2df70290e0dd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 4 Feb 2023 16:10:03 +0200 Subject: [PATCH] Add support for converting lottie stickers --- CHANGELOG.md | 2 ++ Dockerfile | 7 +++- Dockerfile.ci | 7 +++- Dockerfile.dev | 7 +++- attachments.go | 87 +++++++++++++++++++++++++++++++++++++++++++++ config/bridge.go | 8 +++++ config/upgrade.go | 4 +++ example-config.yaml | 14 ++++++++ go.mod | 2 +- go.sum | 4 +-- portal.go | 9 ++++- 11 files changed, 144 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6114f5b..9b4d08f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ * Started automatically subscribing to bridged guilds. This fixes two problems: * Typing notifications should now work automatically in guilds. * Huge guilds now actually get messages bridged. +* Added support for converting animated lottie stickers to raster formats using + [lottieconverter](https://github.com/sot-tech/LottieConverter). * Improved markdown parsing to disable more features that don't exist on Discord. * Removed width from inline images (e.g. in the `guilds status` output) to handle non-square images properly. diff --git a/Dockerfile b/Dockerfile index df9f996..bb41541 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +FROM dock.mau.dev/tulir/lottieconverter AS lottie + FROM golang:1-alpine3.17 AS builder RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev @@ -11,8 +13,11 @@ FROM alpine:3.17 ENV UID=1337 \ GID=1337 -RUN apk add --no-cache ffmpeg su-exec ca-certificates olm bash jq yq curl +RUN apk add --no-cache ffmpeg su-exec ca-certificates olm bash jq yq curl \ + zlib libpng giflib libstdc++ libgcc +COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/ +COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter COPY --from=builder /usr/bin/mautrix-discord /usr/bin/mautrix-discord COPY --from=builder /build/example-config.yaml /opt/mautrix-discord/example-config.yaml COPY --from=builder /build/docker-run.sh /docker-run.sh diff --git a/Dockerfile.ci b/Dockerfile.ci index 294d35b..92f83a9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,10 +1,15 @@ +FROM dock.mau.dev/tulir/lottieconverter AS lottie + FROM alpine:3.17 ENV UID=1337 \ GID=1337 -RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq +RUN apk add --no-cache ffmpeg su-exec ca-certificates bash jq curl yq \ + zlib libpng giflib libstdc++ libgcc +COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/ +COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter ARG EXECUTABLE=./mautrix-discord COPY $EXECUTABLE /usr/bin/mautrix-discord COPY ./example-config.yaml /opt/mautrix-discord/example-config.yaml diff --git a/Dockerfile.dev b/Dockerfile.dev index d37c5e0..549c5eb 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,7 +1,12 @@ +FROM dock.mau.dev/tulir/lottieconverter AS lottie + FROM golang:1-alpine3.17 AS builder -RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl +RUN apk add --no-cache git ca-certificates build-base su-exec olm-dev bash jq yq curl \ + zlib libpng giflib libstdc++ libgcc +COPY --from=lottie /usr/lib/librlottie.so* /usr/lib/ +COPY --from=lottie /usr/local/bin/lottieconverter /usr/local/bin/lottieconverter COPY . /build WORKDIR /build RUN go build -o /usr/bin/mautrix-discord diff --git a/attachments.go b/attachments.go index fcc55b8..5bf0763 100644 --- a/attachments.go +++ b/attachments.go @@ -2,10 +2,15 @@ package main import ( "bytes" + "context" "fmt" "image" "io" "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" "strings" "time" @@ -18,6 +23,7 @@ import ( "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix/util" + "maunium.net/go/mautrix/util/ffmpeg" "go.mau.fi/mautrix-discord/database" ) @@ -151,6 +157,7 @@ type AttachmentMeta struct { MimeType string EmojiName string CopyIfMissing bool + Converter func([]byte) ([]byte, string, error) } var NoMeta = AttachmentMeta{} @@ -160,6 +167,78 @@ type attachmentKey struct { Encrypt bool } +func (br *DiscordBridge) convertLottie(data []byte) ([]byte, string, error) { + fps := br.Config.Bridge.AnimatedSticker.Args.FPS + width := br.Config.Bridge.AnimatedSticker.Args.Width + height := br.Config.Bridge.AnimatedSticker.Args.Height + target := br.Config.Bridge.AnimatedSticker.Target + var lottieTarget, outputMime string + switch target { + case "png": + lottieTarget = "png" + outputMime = "image/png" + fps = 1 + case "gif": + lottieTarget = "gif" + outputMime = "image/gif" + case "webm": + lottieTarget = "pngs" + outputMime = "video/webm" + case "webp": + lottieTarget = "pngs" + outputMime = "image/webp" + case "disable": + return data, "application/json", nil + default: + return nil, "", fmt.Errorf("invalid animated sticker target %q in bridge config", br.Config.Bridge.AnimatedSticker.Target) + } + + ctx := context.Background() + tempdir, err := os.MkdirTemp("", "mautrix_discord_lottie_") + if err != nil { + return nil, "", fmt.Errorf("failed to create temp dir: %w", err) + } + + lottieOutput := filepath.Join(tempdir, "out_") + if lottieTarget != "pngs" { + lottieOutput = filepath.Join(tempdir, "output."+lottieTarget) + } + cmd := exec.CommandContext(ctx, "lottieconverter", "-", lottieOutput, lottieTarget, fmt.Sprintf("%dx%d", width, height), strconv.Itoa(fps)) + cmd.Stdin = bytes.NewReader(data) + err = cmd.Run() + if err != nil { + return nil, "", fmt.Errorf("failed to run lottieconverter: %w", err) + } + var path string + if lottieTarget == "pngs" { + var videoCodec string + outputExtension := "." + target + if target == "webm" { + videoCodec = "libvpx-vp9" + } else if target == "webp" { + videoCodec = "libwebp_anim" + } else { + panic(fmt.Errorf("impossible case: unknown target %q", target)) + } + path, err = ffmpeg.ConvertPath( + ctx, lottieOutput+"*.png", outputExtension, + []string{"-framerate", strconv.Itoa(fps), "-pattern_type", "glob"}, + []string{"-c:v", videoCodec, "-pix_fmt", "yuva420p", "-f", target}, + false, + ) + if err != nil { + return nil, "", fmt.Errorf("failed to run ffmpeg: %w", err) + } + } else { + path = lottieOutput + } + data, err = os.ReadFile(path) + if err != nil { + return nil, "", fmt.Errorf("failed to read converted file: %w", err) + } + return data, outputMime, nil +} + func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, url string, encrypt bool, meta AttachmentMeta) (returnDBFile *database.File, returnErr error) { isCacheable := !encrypt returnDBFile = br.DB.File.Get(url, encrypt) @@ -180,6 +259,14 @@ func (br *DiscordBridge) copyAttachmentToMatrix(intent *appservice.IntentAPI, ur return } + if meta.Converter != nil { + data, meta.MimeType, onceErr = meta.Converter(data) + if onceErr != nil { + onceErr = fmt.Errorf("failed to convert attachment: %w", onceErr) + return + } + } + onceDBFile, onceErr = br.uploadMatrixAttachment(intent, data, url, encrypt, meta) if onceErr != nil { return diff --git a/config/bridge.go b/config/bridge.go index 797eedc..1ddd842 100644 --- a/config/bridge.go +++ b/config/bridge.go @@ -50,6 +50,14 @@ type BridgeConfig struct { DeletePortalOnChannelDelete bool `yaml:"delete_portal_on_channel_delete"` DeleteGuildOnLeave bool `yaml:"delete_guild_on_leave"` FederateRooms bool `yaml:"federate_rooms"` + AnimatedSticker struct { + Target string `yaml:"target"` + Args struct { + Width int `yaml:"width"` + Height int `yaml:"height"` + FPS int `yaml:"fps"` + } `yaml:"args"` + } `yaml:"animated_sticker"` DoublePuppetServerMap map[string]string `yaml:"double_puppet_server_map"` DoublePuppetAllowDiscovery bool `yaml:"double_puppet_allow_discovery"` diff --git a/config/upgrade.go b/config/upgrade.go index 625e4f7..173d328 100644 --- a/config/upgrade.go +++ b/config/upgrade.go @@ -45,6 +45,10 @@ func DoUpgrade(helper *up.Helper) { helper.Copy(up.Bool, "bridge", "delete_portal_on_channel_delete") helper.Copy(up.Bool, "bridge", "delete_guild_on_leave") helper.Copy(up.Bool, "bridge", "federate_rooms") + helper.Copy(up.Str, "bridge", "animated_sticker", "target") + helper.Copy(up.Int, "bridge", "animated_sticker", "args", "width") + helper.Copy(up.Int, "bridge", "animated_sticker", "args", "height") + helper.Copy(up.Int, "bridge", "animated_sticker", "args", "fps") helper.Copy(up.Map, "bridge", "double_puppet_server_map") helper.Copy(up.Bool, "bridge", "double_puppet_allow_discovery") helper.Copy(up.Map, "bridge", "login_shared_secret_map") diff --git a/example-config.yaml b/example-config.yaml index 4f2dd41..6ff48f1 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -140,6 +140,20 @@ bridge: # Whether or not created rooms should have federation enabled. # If false, created portal rooms will never be federated. federate_rooms: true + # Settings for converting animated stickers. + animated_sticker: + # Format to which animated stickers should be converted. + # disable - No conversion, send as-is (lottie JSON) + # png - converts to non-animated png (fastest) + # gif - converts to animated gif + # webm - converts to webm video, requires ffmpeg executable with vp9 codec and webm container support + # webp - converts to animated webp, requires ffmpeg executable with webp codec/container support + target: webp + # Arguments for converter. All converters take width and height. + args: + width: 256 + height: 256 + fps: 25 # only for webm, webp and gif (2, 5, 10, 20 or 25 recommended) # Servers to always allow double puppeting from double_puppet_server_map: example.com: https://example.com diff --git a/go.mod b/go.mod index 072abef..74f8911 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/yuin/goldmark v1.5.4 maunium.net/go/maulogger/v2 v2.3.2 - maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114 + maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6 ) require ( diff --git a/go.sum b/go.sum index e047797..57af0be 100644 --- a/go.sum +++ b/go.sum @@ -77,5 +77,5 @@ maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/maulogger/v2 v2.3.2 h1:1XmIYmMd3PoQfp9J+PaHhpt80zpfmMqaShzUTC7FwY0= maunium.net/go/maulogger/v2 v2.3.2/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= -maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114 h1:H6/OwVn9Z5PNhwzeWSvYCU/Cw3nvTbLTcvJo5HS/lyU= -maunium.net/go/mautrix v0.13.1-0.20230204122701-3c0e64060114/go.mod h1:3u2Fz3JY/eXVLOzxn2ODVL4rzCIjkkt0jlygEx4Qnaw= +maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6 h1:mSq0HwzhpM5XOk+YRgOsEx62AVG6N/lonmz/3iBwf+A= +maunium.net/go/mautrix v0.13.1-0.20230204140716-485b4f376dc6/go.mod h1:3u2Fz3JY/eXVLOzxn2ODVL4rzCIjkkt0jlygEx4Qnaw= diff --git a/portal.go b/portal.go index 162c9e9..d59b19e 100644 --- a/portal.go +++ b/portal.go @@ -555,7 +555,11 @@ func (portal *Portal) sendMediaFailedMessage(intent *appservice.IntentAPI, bridg const DiscordStickerSize = 160 func (portal *Portal) handleDiscordFile(typeName string, intent *appservice.IntentAPI, id, url string, content *event.MessageEventContent, ts time.Time, threadRelation *event.RelatesTo) *database.MessagePart { - dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, AttachmentMeta{AttachmentID: id, MimeType: content.Info.MimeType}) + meta := AttachmentMeta{AttachmentID: id, MimeType: content.Info.MimeType} + if typeName == "sticker" && content.Info.MimeType == "application/json" { + meta.Converter = portal.bridge.convertLottie + } + dbFile, err := portal.bridge.copyAttachmentToMatrix(intent, url, portal.Encrypted, meta) if err != nil { errorEventID := portal.sendMediaFailedMessage(intent, err) if errorEventID != "" { @@ -566,6 +570,9 @@ func (portal *Portal) handleDiscordFile(typeName string, intent *appservice.Inte } return nil } + if typeName == "sticker" && content.Info.MimeType == "application/json" { + content.Info.MimeType = dbFile.MimeType + } content.Info.Size = dbFile.Size if content.Info.Width == 0 && content.Info.Height == 0 { content.Info.Width = dbFile.Width