Refining Messages without suspension (#1167)

* Introduce `RefinableMessage.tryRefine` to refine without suspension.

* Extract `RefinableMessage` to separate file

* Always use `Bot` on `List<MsgComm.Msg>.toMessageChain`

* Introduce `MessageRefiner` and ensure MessageChain refined after transformation. Fix #1156, fix #1157

* Add basic tests

* Refine forward message contents

* Refine long message contents

* Move refinement from message internals to MiraiImpl public APIs

* Comment out unused `toMessageChainOffline`

* refinement tests part

* refinement tests part

* Full tests and minor internal improv.s

* Fix tests

* Fix compile

mirai-core/src/commonMain/kotlin/MiraiImpl.kt

@@ -27,10 +27,12 @@ import net.mamoe.mirai.internal.contact.*
 import net.mamoe.mirai.internal.contact.info.FriendInfoImpl
 import net.mamoe.mirai.internal.contact.info.MemberInfoImpl
 import net.mamoe.mirai.internal.message.*
+import net.mamoe.mirai.internal.message.DeepMessageRefiner.refineDeep
 import net.mamoe.mirai.internal.network.highway.*
 import net.mamoe.mirai.internal.network.protocol.data.jce.SvcDevLoginInfo
 import net.mamoe.mirai.internal.network.protocol.data.proto.ImMsgBody
 import net.mamoe.mirai.internal.network.protocol.data.proto.LongMsg
+import net.mamoe.mirai.internal.network.protocol.data.proto.MsgComm
 import net.mamoe.mirai.internal.network.protocol.data.proto.MsgTransmit
 import net.mamoe.mirai.internal.network.protocol.packet.chat.*
 import net.mamoe.mirai.internal.network.protocol.packet.chat.voice.PttStore
@@ -963,21 +965,29 @@ internal open class MiraiImpl : IMirai, LowLevelApiAccessor {
 
     override suspend fun downloadLongMessage(bot: Bot, resourceId: String): MessageChain {
         return downloadMultiMsgTransmit(bot, resourceId, ResourceKind.LONG_MESSAGE).msg
-            .toMessageChainNoSource(bot.id, 0, MessageSourceKind.GROUP)
+            .toMessageChainNoSource(bot, 0, MessageSourceKind.GROUP)
+            .refineDeep(bot)
     }
 
     override suspend fun downloadForwardMessage(bot: Bot, resourceId: String): List<ForwardMessage.Node> {
         return downloadMultiMsgTransmit(bot, resourceId, ResourceKind.FORWARD_MESSAGE).msg.map { msg ->
-            ForwardMessage.Node(
+            msg.toNode(bot)
+        }
+    }
+
+    protected open suspend fun MsgComm.Msg.toNode(bot: Bot): ForwardMessage.Node {
+        val msg = this
+        return ForwardMessage.Node(
             senderId = msg.msgHead.fromUin,
             time = msg.msgHead.msgTime,
             senderName = msg.msgHead.groupInfo?.groupCard
                 ?: msg.msgHead.fromNick.takeIf { it.isNotEmpty() }
                 ?: msg.msgHead.fromUin.toString(),
-                messageChain = listOf(msg).toMessageChainNoSource(bot.id, 0, MessageSourceKind.GROUP)
+            messageChain = listOf(msg)
+                .toMessageChainNoSource(bot, 0, MessageSourceKind.GROUP)
+                .refineDeep(bot)
         )
     }
-    }
 
     private suspend fun downloadMultiMsgTransmit(
         bot: Bot,
@@ -1026,7 +1036,7 @@ internal open class MiraiImpl : IMirai, LowLevelApiAccessor {
 
                 val down = longResp.msgDownRsp.single()
                 check(down.result == 0) {
-                    "Message download failed, result=${down.result}, resId=${down.msgResid}, msgContent=${down.msgContent.toUHexString()}"
+                    "Message download failed, result=${down.result}, resId=${down.msgResid.encodeToString()}, msgContent=${down.msgContent.toUHexString()}"
                 }
 
                 val content = down.msgContent.ungzip()

mirai-core/src/commonMain/kotlin/message/LongMessageInternal.kt

@@ -9,9 +9,10 @@
 
 package net.mamoe.mirai.internal.message
 
+import net.mamoe.mirai.Bot
 import net.mamoe.mirai.Mirai
-import net.mamoe.mirai.contact.Contact
 import net.mamoe.mirai.internal.asQQAndroidBot
+import net.mamoe.mirai.internal.message.DeepMessageRefiner.refineDeep
 import net.mamoe.mirai.message.data.*
 import net.mamoe.mirai.utils.safeCast
 
@@ -20,8 +21,8 @@ internal data class LongMessageInternal internal constructor(override val conten
     AbstractServiceMessage(), RefinableMessage {
     override val serviceId: Int get() = 35
 
-    override suspend fun refine(contact: Contact, context: MessageChain): Message {
-        val bot = contact.bot.asQQAndroidBot()
+    override suspend fun refine(bot: Bot, context: MessageChain): Message {
+        bot.asQQAndroidBot()
         val long = Mirai.downloadLongMessage(bot, resId)
 
         return MessageOrigin(SimpleServiceMessage(serviceId, content), resId, MessageOriginKind.LONG) + long
@@ -37,8 +38,8 @@ internal data class ForwardMessageInternal(override val content: String, val res
     RefinableMessage {
     override val serviceId: Int get() = 35
 
-    override suspend fun refine(contact: Contact, context: MessageChain): Message {
-        val bot = contact.bot.asQQAndroidBot()
+    override suspend fun refine(bot: Bot, context: MessageChain): Message {
+        bot.asQQAndroidBot()
 
         val msgXml = content.substringAfter("<msg", "")
         val xmlHead = msgXml.substringBefore("<item")
@@ -59,7 +60,11 @@ internal data class ForwardMessageInternal(override val content: String, val res
         val preview = titles
         val source = xmlFoot.findField("name")
 
-        return MessageOrigin(SimpleServiceMessage(serviceId, content), resId, MessageOriginKind.FORWARD) + ForwardMessage(
+        return MessageOrigin(
+            SimpleServiceMessage(serviceId, content),
+            resId,
+            MessageOriginKind.FORWARD
+        ) + ForwardMessage(
             preview = preview,
             title = title,
             brief = brief,
@@ -85,17 +90,3 @@ internal data class ForwardMessageInternal(override val content: String, val res
     }
 }
 
-/**
- * 在接收解析消息后会经过一层转换的消息.
- * @see MessageChain.refine
- */
-internal interface RefinableMessage : SingleMessage {
-
-    /**
-     * This message [RefinableMessage] will be replaced by return value of [refine]
-     */
-    suspend fun refine(
-        contact: Contact,
-        context: MessageChain,
-    ): Message?
-}

mirai-core/src/commonMain/kotlin/message/MarketFaceImpl.kt

@@ -13,7 +13,7 @@ package net.mamoe.mirai.internal.message
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.Transient
-import net.mamoe.mirai.contact.Contact
+import net.mamoe.mirai.Bot
 import net.mamoe.mirai.internal.network.protocol.data.proto.ImMsgBody
 import net.mamoe.mirai.message.data.Dice
 import net.mamoe.mirai.message.data.MarketFace
@@ -44,7 +44,7 @@ internal class MarketFaceInternal(
     override val name: String get() = delegate.faceName.decodeToString()
     override val id: Int get() = delegate.tabId
 
-    override suspend fun refine(contact: Contact, context: MessageChain): Message {
+    override fun tryRefine(bot: Bot, context: MessageChain): Message {
         delegate.toDiceOrNull()?.let { return it } // TODO: 2021/2/12 add dice origin, maybe rename MessageOrigin
         return MarketFaceImpl(delegate)
     }

mirai-core/src/commonMain/kotlin/message/ReceiveMessageHandler.kt

@@ -15,8 +15,8 @@ import kotlinx.io.core.readUInt
 import kotlinx.io.core.readUShort
 import kotlinx.serialization.json.Json
 import net.mamoe.mirai.Bot
-import net.mamoe.mirai.contact.Contact
-import net.mamoe.mirai.internal.asQQAndroidBot
+import net.mamoe.mirai.internal.message.DeepMessageRefiner.refineDeep
+import net.mamoe.mirai.internal.message.LightMessageRefiner.refineLight
 import net.mamoe.mirai.internal.message.ReceiveMessageTransformer.cleanupRubbishMessageElements
 import net.mamoe.mirai.internal.message.ReceiveMessageTransformer.joinToMessageChain
 import net.mamoe.mirai.internal.message.ReceiveMessageTransformer.toVoice
@@ -26,45 +26,48 @@ import net.mamoe.mirai.internal.utils.io.serialization.readProtoBuf
 import net.mamoe.mirai.message.data.*
 import net.mamoe.mirai.utils.*
 
+/**
+ * 只在手动构造 [OfflineMessageSource] 时调用
+ */
 internal fun ImMsgBody.SourceMsg.toMessageChainNoSource(
-    botId: Long,
+    bot: Bot,
     messageSourceKind: MessageSourceKind,
     groupIdOrZero: Long
 ): MessageChain {
     val elements = this.elems
     return buildMessageChain(elements.size + 1) {
-        joinToMessageChain(elements, groupIdOrZero, messageSourceKind, botId, this)
-    }.cleanupRubbishMessageElements()
+        joinToMessageChain(elements, groupIdOrZero, messageSourceKind, bot, this)
+    }.cleanupRubbishMessageElements().refineLight(bot)
 }
 
 
-internal fun List<MsgComm.Msg>.toMessageChainOnline(
+internal suspend fun List<MsgComm.Msg>.toMessageChainOnline(
     bot: Bot,
     groupIdOrZero: Long,
     messageSourceKind: MessageSourceKind
 ): MessageChain {
-    return toMessageChain(bot, bot.id, groupIdOrZero, true, messageSourceKind)
+    return toMessageChain(bot, groupIdOrZero, true, messageSourceKind).refineDeep(bot)
 }
 
-internal fun List<MsgComm.Msg>.toMessageChainOffline(
-    bot: Bot,
-    groupIdOrZero: Long,
-    messageSourceKind: MessageSourceKind
-): MessageChain {
-    return toMessageChain(bot, bot.id, groupIdOrZero, false, messageSourceKind)
-}
+//internal fun List<MsgComm.Msg>.toMessageChainOffline(
+//    bot: Bot,
+//    groupIdOrZero: Long,
+//    messageSourceKind: MessageSourceKind
+//): MessageChain {
+//    return toMessageChain(bot, groupIdOrZero, false, messageSourceKind).refineLight(bot)
+//}
 
 internal fun List<MsgComm.Msg>.toMessageChainNoSource(
-    botId: Long,
+    bot: Bot,
     groupIdOrZero: Long,
     messageSourceKind: MessageSourceKind
 ): MessageChain {
-    return toMessageChain(null, botId, groupIdOrZero, null, messageSourceKind)
+    return toMessageChain(bot, groupIdOrZero, null, messageSourceKind).refineLight(bot)
 }
 
+
 private fun List<MsgComm.Msg>.toMessageChain(
-    bot: Bot?,
-    botId: Long,
+    bot: Bot,
     groupIdOrZero: Long,
     onlineSource: Boolean?,
     messageSourceKind: MessageSourceKind
@@ -77,11 +80,10 @@ private fun List<MsgComm.Msg>.toMessageChain(
     val builder = MessageChainBuilder(elements.size)
 
     if (onlineSource != null) {
-        checkNotNull(bot)
         builder.add(ReceiveMessageTransformer.createMessageSource(bot, onlineSource, messageSourceKind, messageList))
     }
 
-    joinToMessageChain(elements, groupIdOrZero, messageSourceKind, botId, builder)
+    joinToMessageChain(elements, groupIdOrZero, messageSourceKind, bot, builder)
 
     for (msg in messageList) {
         msg.msgBody.richText.ptt?.toVoice()?.let { builder.add(it) }
@@ -94,7 +96,7 @@ private fun List<MsgComm.Msg>.toMessageChain(
  * 接收消息的解析器. 将 [MsgComm.Msg] 转换为对应的 [SingleMessage]
  * @see joinToMessageChain
  */
-private object ReceiveMessageTransformer {
+internal object ReceiveMessageTransformer {
     fun createMessageSource(
         bot: Bot,
         onlineSource: Boolean,
@@ -120,12 +122,13 @@ private object ReceiveMessageTransformer {
         elements: List<ImMsgBody.Elem>,
         groupIdOrZero: Long,
         messageSourceKind: MessageSourceKind,
-        botId: Long,
+        bot: Bot,
         builder: MessageChainBuilder
     ) {
+//        ProtoBuf.encodeToHexString(elements).soutv("join")
         // (this._miraiContentToString().soutv())
         for (element in elements) {
-            transformElement(element, groupIdOrZero, messageSourceKind, botId, builder)
+            transformElement(element, groupIdOrZero, messageSourceKind, bot, builder)
             when {
                 element.richMsg != null -> decodeRichMessage(element.richMsg, builder)
             }
@@ -136,11 +139,11 @@ private object ReceiveMessageTransformer {
         element: ImMsgBody.Elem,
         groupIdOrZero: Long,
         messageSourceKind: MessageSourceKind,
-        botId: Long,
+        bot: Bot,
         builder: MessageChainBuilder
     ) {
         when {
-            element.srcMsg != null -> decodeSrcMsg(element.srcMsg, builder, botId, messageSourceKind, groupIdOrZero)
+            element.srcMsg != null -> decodeSrcMsg(element.srcMsg, builder, bot, messageSourceKind, groupIdOrZero)
             element.notOnlineImage != null -> builder.add(OnlineFriendImageImpl(element.notOnlineImage))
             element.customFace != null -> decodeCustomFace(element.customFace, builder)
             element.face != null -> builder.add(Face(element.face.index))
@@ -281,11 +284,11 @@ private object ReceiveMessageTransformer {
     private fun decodeSrcMsg(
         srcMsg: ImMsgBody.SourceMsg,
         list: MessageChainBuilder,
-        botId: Long,
+        bot: Bot,
         messageSourceKind: MessageSourceKind,
         groupIdOrZero: Long
     ) {
-        list.add(QuoteReply(OfflineMessageSourceImplData(srcMsg, botId, messageSourceKind, groupIdOrZero)))
+        list.add(QuoteReply(OfflineMessageSourceImplData(srcMsg, bot, messageSourceKind, groupIdOrZero)))
     }
 
     private fun decodeCustomFace(
@@ -508,42 +511,3 @@ private object ReceiveMessageTransformer {
         kotlinx.io.core.String(downPara)
     )
 }
-
-/**
- * 解析 [ForwardMessageInternal], [LongMessageInternal]
- * 并处理换行符问题
- */
-internal suspend fun MessageChain.refine(contact: Contact): MessageChain {
-    val convertLineSeparator = contact.bot.asQQAndroidBot().configuration.convertLineSeparator
-
-    if (none {
-            it is RefinableMessage
-                    || (it is PlainText && convertLineSeparator && it.content.contains('\r'))
-        }
-    ) return this
-
-
-    val builder = MessageChainBuilder(this.size)
-    for (singleMessage in this) {
-        if (singleMessage is RefinableMessage) {
-            val v = singleMessage.refine(contact, this)
-            if (v != null) builder.add(v)
-        } else if (singleMessage is PlainText && convertLineSeparator) {
-            val content = singleMessage.content
-            if (content.contains('\r')) {
-                builder.add(
-                    PlainText(
-                        content
-                            .replace("\r\n", "\n")
-                            .replace('\r', '\n')
-                    )
-                )
-            } else {
-                builder.add(singleMessage)
-            }
-        } else {
-            builder.add(singleMessage)
-        }
-    }
-    return builder.build()
-}

mirai-core/src/commonMain/kotlin/message/RefinableMessage.kt

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+package net.mamoe.mirai.internal.message
+
+import net.mamoe.mirai.Bot
+import net.mamoe.mirai.message.data.*
+
+/**
+ * 在接收解析消息后会经过一层转换的消息.
+ * @see MessageChain.refineLight
+ */
+internal interface RefinableMessage : SingleMessage {
+
+    /**
+     * Refine if possible (without suspension), returns self otherwise.
+     * @since 2.6
+     */ // see #1157
+    fun tryRefine(
+        bot: Bot,
+        context: MessageChain,
+    ): Message? = this
+
+    /**
+     * This message [RefinableMessage] will be replaced by return value of [refineLight]
+     */
+    suspend fun refine(
+        bot: Bot,
+        context: MessageChain,
+    ): Message? = tryRefine(bot, context)
+}
+
+internal sealed class MessageRefiner {
+    protected inline fun MessageChain.refineImpl(
+        bot: Bot,
+        refineAction: (message: RefinableMessage) -> Message?
+    ): MessageChain {
+        val convertLineSeparator = bot.configuration.convertLineSeparator
+
+        if (none {
+                it is RefinableMessage
+                        || (it is PlainText && convertLineSeparator && it.content.contains('\r'))
+            }
+        ) return this
+
+
+        val builder = MessageChainBuilder(this.size)
+        for (singleMessage in this) {
+            if (singleMessage is RefinableMessage) {
+                val v = refineAction(singleMessage)
+                if (v != null) builder.add(v)
+            } else if (singleMessage is PlainText && convertLineSeparator) {
+                val content = singleMessage.content
+                if (content.contains('\r')) {
+                    builder.add(
+                        PlainText(
+                            content
+                                .replace("\r\n", "\n")
+                                .replace('\r', '\n')
+                        )
+                    )
+                } else {
+                    builder.add(singleMessage)
+                }
+            } else {
+                builder.add(singleMessage)
+            }
+        }
+        return builder.build()
+    }
+}
+
+internal object LightMessageRefiner : MessageRefiner() {
+    fun MessageChain.refineLight(bot: Bot): MessageChain {
+        return refineImpl(bot) { it.tryRefine(bot, this) }
+    }
+}
+
+internal object DeepMessageRefiner : MessageRefiner() {
+    suspend fun MessageChain.refineDeep(bot: Bot): MessageChain {
+        return refineImpl(bot) { it.refine(bot, this) }
+    }
+}

mirai-core/src/commonMain/kotlin/message/incomingSourceImpl.kt

@@ -48,7 +48,7 @@ internal class OnlineMessageSourceFromFriendImpl(
     } // other client 消息的这个是0
     override val time: Int = msg.first().msgHead.msgTime
     override val originalMessage: MessageChain by lazy {
-        msg.toMessageChainNoSource(bot.id, 0, MessageSourceKind.FRIEND)
+        msg.toMessageChainNoSource(bot, 0, MessageSourceKind.FRIEND)
     }
     override val sender: Friend = bot.getFriendOrFail(msg.first().msgHead.fromUin)
 
@@ -72,7 +72,7 @@ internal class OnlineMessageSourceFromStrangerImpl(
     } // other client 消息的这个是0
     override val time: Int = msg.first().msgHead.msgTime
     override val originalMessage: MessageChain by lazy {
-        msg.toMessageChainNoSource(bot.id, 0, MessageSourceKind.STRANGER)
+        msg.toMessageChainNoSource(bot, 0, MessageSourceKind.STRANGER)
     }
     override val sender: Stranger = bot.getStrangerOrFail(msg.first().msgHead.fromUin)
 
@@ -133,7 +133,7 @@ internal class OnlineMessageSourceFromTempImpl(
     override val ids: IntArray get() = sequenceIds//
     override val time: Int = msg.first().msgHead.msgTime
     override val originalMessage: MessageChain by lazy {
-        msg.toMessageChainNoSource(bot.id, groupIdOrZero = 0, MessageSourceKind.TEMP)
+        msg.toMessageChainNoSource(bot, groupIdOrZero = 0, MessageSourceKind.TEMP)
     }
     override val sender: Member = with(msg.first().msgHead) {
         bot.getGroupOrFail(c2cTmpMsgHead!!.groupUin).getOrFail(fromUin)
@@ -157,7 +157,7 @@ internal class OnlineMessageSourceFromGroupImpl(
     override val ids: IntArray get() = sequenceIds
     override val time: Int = msg.first().msgHead.msgTime
     override val originalMessage: MessageChain by lazy {
-        msg.toMessageChainNoSource(bot.id, groupIdOrZero = group.id, MessageSourceKind.GROUP)
+        msg.toMessageChainNoSource(bot, groupIdOrZero = group.id, MessageSourceKind.GROUP)
     }
 
     override val sender: Member by lazy {

mirai-core/src/commonMain/kotlin/message/lightApp.kt

@@ -12,7 +12,7 @@ package net.mamoe.mirai.internal.message
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
-import net.mamoe.mirai.contact.Contact
+import net.mamoe.mirai.Bot
 import net.mamoe.mirai.message.data.*
 import net.mamoe.mirai.utils.safeCast
 
@@ -22,7 +22,7 @@ internal data class LightAppInternal(
     companion object Key :
         AbstractPolymorphicMessageKey<RichMessage, LightAppInternal>(RichMessage, { it.safeCast() })
 
-    override suspend fun refine(contact: Contact, context: MessageChain): Message {
+    override fun tryRefine(bot: Bot, context: MessageChain): Message {
         val struct = tryDeserialize() ?: return LightApp(content)
         struct.run {
             if (meta.music != null) {

mirai-core/src/commonMain/kotlin/message/offlineSourceImpl.kt

@@ -117,7 +117,7 @@ internal fun OfflineMessageSourceImplData(
         fromId = head.fromUin,
         targetId = head.groupInfo?.groupCode ?: head.toUin,
         originalMessage = delegate.toMessageChainNoSource(
-            bot.id,
+            bot,
             groupIdOrZero = head.groupInfo?.groupCode ?: 0,
             messageSourceKind = kind
         ),
@@ -151,7 +151,7 @@ internal fun OfflineMessageSourceImplData(
 
 internal fun OfflineMessageSourceImplData(
     delegate: ImMsgBody.SourceMsg,
-    botId: Long,
+    bot: Bot,
     messageSourceKind: MessageSourceKind,
     groupIdOrZero: Long,
 ): OfflineMessageSourceImplData {
@@ -161,7 +161,7 @@ internal fun OfflineMessageSourceImplData(
         internalIds = delegate.pbReserve.loadAs(SourceMsg.ResvAttr.serializer())
             .origUids?.mapToIntArray { it.toInt() } ?: intArrayOf(),
         time = delegate.time,
-        originalMessageLazy = lazy { delegate.toMessageChainNoSource(botId, messageSourceKind, groupIdOrZero) },
+        originalMessageLazy = lazy { delegate.toMessageChainNoSource(bot, messageSourceKind, groupIdOrZero) },
         fromId = delegate.senderUin,
         targetId = when {
             groupIdOrZero != 0L -> groupIdOrZero
@@ -176,7 +176,7 @@ internal fun OfflineMessageSourceImplData(
             }"
             )*/
         },
-        botId = botId
+        botId = bot.id
     ).apply {
         jceData = delegate
     }

mirai-core/src/commonMain/kotlin/network/protocol/packet/chat/receive/MessageSvc.PbGetMsg.kt

@@ -32,7 +32,6 @@ import net.mamoe.mirai.internal.contact.info.GroupInfoImpl
 import net.mamoe.mirai.internal.contact.info.MemberInfoImpl
 import net.mamoe.mirai.internal.contact.info.StrangerInfoImpl
 import net.mamoe.mirai.internal.message.OnlineMessageSourceFromFriendImpl
-import net.mamoe.mirai.internal.message.refine
 import net.mamoe.mirai.internal.message.toMessageChainOnline
 import net.mamoe.mirai.internal.network.MultiPacket
 import net.mamoe.mirai.internal.network.Packet
@@ -402,13 +401,13 @@ internal suspend fun MsgComm.Msg.transform(bot: QQAndroidBot, fromSync: Boolean
                             if (fromSync) {
                                 FriendMessageSyncEvent(
                                     friend,
-                                    msgs.toMessageChainOnline(bot, 0, MessageSourceKind.FRIEND).refine(friend),
+                                    msgs.toMessageChainOnline(bot, 0, MessageSourceKind.FRIEND),
                                     msgHead.msgTime
                                 )
                             } else {
                                 FriendMessageEvent(
                                     friend,
-                                    msgs.toMessageChainOnline(bot, 0, MessageSourceKind.FRIEND).refine(friend),
+                                    msgs.toMessageChainOnline(bot, 0, MessageSourceKind.FRIEND),
                                     msgHead.msgTime
                                 )
                             }
@@ -427,13 +426,13 @@ internal suspend fun MsgComm.Msg.transform(bot: QQAndroidBot, fromSync: Boolean
                         if (fromSync) {
                             StrangerMessageSyncEvent(
                                 stranger,
-                                listOf(this).toMessageChainOnline(bot, 0, STRANGER).refine(stranger),
+                                listOf(this).toMessageChainOnline(bot, 0, STRANGER),
                                 msgHead.msgTime
                             )
                         } else {
                             StrangerMessageEvent(
                                 stranger,
-                                listOf(this).toMessageChainOnline(bot, 0, STRANGER).refine(stranger),
+                                listOf(this).toMessageChainOnline(bot, 0, STRANGER),
                                 msgHead.msgTime
                             )
                         }
@@ -507,13 +506,13 @@ internal suspend fun MsgComm.Msg.transform(bot: QQAndroidBot, fromSync: Boolean
                         return if (fromSync) {
                             GroupTempMessageSyncEvent(
                                 member,
-                                listOf(this).toMessageChainOnline(bot, 0, TEMP).refine(member),
+                                listOf(this).toMessageChainOnline(bot, 0, TEMP),
                                 msgHead.msgTime
                             )
                         } else {
                             GroupTempMessageEvent(
                                 member,
-                                listOf(this).toMessageChainOnline(bot, 0, TEMP).refine(member),
+                                listOf(this).toMessageChainOnline(bot, 0, TEMP),
                                 msgHead.msgTime
                             )
                         }

mirai-core/src/commonMain/kotlin/network/protocol/packet/chat/receive/OnlinePush.PbPushGroupMsg.kt

@@ -16,14 +16,12 @@ import net.mamoe.mirai.contact.Member
 import net.mamoe.mirai.contact.nameCardOrNick
 import net.mamoe.mirai.event.AbstractEvent
 import net.mamoe.mirai.event.Event
-import net.mamoe.mirai.event.broadcast
 import net.mamoe.mirai.event.events.GroupMessageEvent
 import net.mamoe.mirai.event.events.GroupMessageSyncEvent
 import net.mamoe.mirai.event.events.MemberCardChangeEvent
 import net.mamoe.mirai.internal.QQAndroidBot
 import net.mamoe.mirai.internal.contact.*
 import net.mamoe.mirai.internal.contact.info.MemberInfoImpl
-import net.mamoe.mirai.internal.message.refine
 import net.mamoe.mirai.internal.message.toMessageChainOnline
 import net.mamoe.mirai.internal.network.Packet
 import net.mamoe.mirai.internal.network.protocol.data.proto.ImMsgBody
@@ -122,7 +120,7 @@ internal object OnlinePushPbPushGroupMsg : IncomingPacketFactory<Packet?>("Onlin
 
         if (isFromSelfAccount) {
             return GroupMessageSyncEvent(
-                message = msgs.map { it.msg }.toMessageChainOnline(bot, group.id, GROUP).refine(group),
+                message = msgs.map { it.msg }.toMessageChainOnline(bot, group.id, GROUP),
                 time = msgHead.msgTime,
                 group = group,
                 sender = sender,
@@ -135,7 +133,7 @@ internal object OnlinePushPbPushGroupMsg : IncomingPacketFactory<Packet?>("Onlin
             return GroupMessageEvent(
                 senderName = name,
                 sender = sender,
-                message = msgs.map { it.msg }.toMessageChainOnline(bot, group.id, GROUP).refine(group),
+                message = msgs.map { it.msg }.toMessageChainOnline(bot, group.id, GROUP),
                 permission = sender.permission,
                 time = msgHead.msgTime
             )

mirai-core/src/commonTest/kotlin/test/utils.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+
+package net.mamoe.mirai.internal.test
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.runBlocking
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.EmptyCoroutineContext
+
+internal fun runBlockingUnit(
+    context: CoroutineContext = EmptyCoroutineContext,
+    block: suspend CoroutineScope.() -> Unit
+): Unit {
+    return runBlocking(context, block)
+}

mirai-core/src/jvmTest/kotlin/AbstractTestWithMiraiImpl.kt

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+
+package net.mamoe.mirai.internal
+
+import net.mamoe.mirai.Mirai
+import org.junit.jupiter.api.AfterEach
+import org.junit.jupiter.api.BeforeEach
+
+internal abstract class AbstractTestWithMiraiImpl : MiraiImpl() {
+    private val originalImpl = Mirai
+
+    @BeforeEach
+    fun setupMiraiImpl() {
+        @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+        net.mamoe.mirai._MiraiInstance.set(this)
+    }
+
+    @AfterEach
+    fun restoreMiraiImpl() {
+        @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+        net.mamoe.mirai._MiraiInstance.set(originalImpl)
+    }
+}

mirai-core/src/jvmTest/kotlin/message/data/MessageRefineTest.kt

@@ -0,0 +1,346 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+
+package net.mamoe.mirai.internal.message.data
+
+import kotlinx.serialization.decodeFromHexString
+import kotlinx.serialization.protobuf.ProtoBuf
+import net.mamoe.mirai.Bot
+import net.mamoe.mirai.internal.AbstractTestWithMiraiImpl
+import net.mamoe.mirai.internal.MiraiImpl
+import net.mamoe.mirai.internal.MockBot
+import net.mamoe.mirai.internal.message.DeepMessageRefiner.refineDeep
+import net.mamoe.mirai.internal.message.LightMessageRefiner.refineLight
+import net.mamoe.mirai.internal.message.OfflineMessageSourceImplData
+import net.mamoe.mirai.internal.message.ReceiveMessageTransformer
+import net.mamoe.mirai.internal.message.RefinableMessage
+import net.mamoe.mirai.internal.network.protocol.data.proto.ImMsgBody
+import net.mamoe.mirai.internal.network.protocol.data.proto.MsgComm
+import net.mamoe.mirai.internal.test.runBlockingUnit
+import net.mamoe.mirai.message.data.*
+import net.mamoe.mirai.message.data.MessageChain.Companion.serializeToJsonString
+import net.mamoe.mirai.utils.PlatformLogger
+import org.junit.jupiter.api.Test
+import kotlin.random.Random
+import kotlin.test.assertEquals
+
+
+open class TM(private val name: String = Random.nextInt().toString()) : SingleMessage {
+    override fun toString(): String = name
+    override fun contentToString(): String = name
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as TM
+
+        if (name != other.name) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int = name.hashCode()
+}
+
+private val bot = MockBot()
+
+private suspend fun testRefineAll(
+    before: Message,
+    after: MessageChain,
+) {
+    testRefineLight(before, after)
+    testRefineDeep(before, after)
+}
+
+private suspend fun testRefineDeep(
+    before: Message,
+    after: MessageChain
+) = assertEquals(after.toMessageChain(), before.toMessageChain().refineDeep(bot))
+
+private fun testRefineLight(
+    before: Message,
+    after: MessageChain
+) = assertEquals(after.toMessageChain(), before.toMessageChain().refineLight(bot))
+
+
+@Suppress("TestFunctionName")
+private fun RefinableMessage(
+    refine: (bot: Bot, context: MessageChain) -> Message?
+): RefinableMessage {
+    return object : RefinableMessage, TM() {
+        override fun tryRefine(bot: Bot, context: MessageChain): Message? {
+            return refine(bot, context)
+        }
+    }
+}
+
+@Suppress("TestFunctionName")
+private fun RefinableMessage0(
+    refine: () -> Message?
+): RefinableMessage {
+    return object : RefinableMessage, TM() {
+        override fun tryRefine(bot: Bot, context: MessageChain): Message? {
+            return refine()
+        }
+    }
+}
+
+private object MiraiImplForRefineTest : MiraiImpl() {
+    override suspend fun downloadForwardMessage(bot: Bot, resourceId: String): List<ForwardMessage.Node> {
+        return super.downloadForwardMessage(bot, resourceId)
+    }
+}
+
+internal class MessageRefineTest : AbstractTestWithMiraiImpl() {
+
+    @Test
+    fun `can remove self`() = runBlockingUnit {
+        testRefineAll(
+            RefinableMessage0 { null },
+            messageChainOf()
+        )
+    }
+
+    @Test
+    fun `can replace`() = runBlockingUnit {
+        testRefineAll(
+            RefinableMessage0 { TM("1") },
+            messageChainOf(TM("1"))
+        )
+    }
+
+    @Test
+    fun `ignore non-refinable`() = runBlockingUnit {
+        testRefineAll(
+            TM("1"),
+            messageChainOf(TM("1"))
+        )
+    }
+
+    @Test
+    fun `can replace flatten`() = runBlockingUnit {
+        testRefineAll(
+            buildMessageChain {
+                +RefinableMessage0 { TM("1") + TM("2") }
+                +TM("3")
+                +RefinableMessage0 { TM("4") + TM("5") }
+            },
+            messageChainOf(TM("1"), TM("2"), TM("3"), TM("4"), TM("5"))
+        )
+    }
+
+    private val testCases = object {
+        /**
+         * 单个 quote 包含 at 和 plain
+         */
+        val simpleQuote =
+            decodeProto("087aea027708a2fc1010d285d8cc0418f9e7b4830620012a0d0a0b0a09e999a4e99d9e363438420a18aedd90f380808080014a480a2d08d285d8cc0410d285d8cc04185228a2fc1030f9e7b4830638aedd90f380808080014a0608d285d8cc04e001011a170a15120d0a0b0a09e999a4e99d9e36343812044a0208591a0a180a0740e9bb84e889b21a0d00010000000300499602d20000050a030a01201a0a180a0740e9bb84e889b21a0d00010000000300499602d20000070a050a032073624baa02489a0145080120cb507800c80100f00100f80100900200c80200980300a00300b00300c00300d00300e803008a04021002900480c0829004b80400c00400ca0400f804dc8002880500044a0240011082010d0a076161617465737418012803")
+
+        /**
+         * 一个引用另一个 quote
+         */
+        val nestedQuote2 =
+            decodeProto("0631ea022e08a4fc1010d285d8cc041885e8b4830620012a0e0a0c0a0a40e9bb84e889b2207362420a1896fee2d386808080011b0a190a0840616161746573741a0d00010000000800499602d20000080a060a04207878784baa02489a0145080120cb507800c80100f00100f80100900200c80200980300a00300b00300c00300d00300e803008a04021003900480c0829004b80400c00400ca0400f804dc8002880500044a0240011082010d0a076161617465737418012803")
+
+        /**
+         * quote -> quote -> quote[at + plain]
+         */
+        val nestedQuote3 =
+            decodeProto("062aea022708a6fc1010d285d8cc0418b0e8b4830620012a070a050a03787878420a18b584a7ca80808080011b0a190a0840616161746573741a0d00010000000800499602d200000a0a080a062061616161614baa02489a0145080120cb507800c80100f00100f80100900200c80200980300a00300b00300c00300d00300e803008a04021003900480c0829004b80400c00400ca0400f804dc8002880500044a0240011082010d0a076161617465737418012803")
+
+        /**
+         * [Dice.value] 4
+         */
+        val dice4 =
+            decodeProto("056432620a0e5be99a8fe69cbae9aab0e5ad905d1006180122104823d3adb15df08014ce5d6796b76ee128c85930033a103430396532613639623136393138663950c80158c8016211727363547970653f313b76616c75653d336a0a0a0608c80110c8014001120a100a0e5be99a8fe69cbae9aab0e5ad905d4baa02489a014508017800900101c80100f00100f80100900200c80200980300a00300b00300c00300d00300e803008a04021003900480c0829004b80400c00400ca0400f804dc8002880500044a0240011082010d0a076161617465737418012803")
+
+        /**
+         * quote -> dice4
+         */
+        val quoteDice4 =
+            decodeProto("06e601ea02e20108adfc1010d285d8cc04188feab4830620012a120a100a0e5be99a8fe69cbae9aab0e5ad905d420a1894bbc6f481808080014aad010a2d08d285d8cc0410d285d8cc04185228adfc10308feab483063894bbc6f481808080014a0608d285d8cc04e001011a7c0a7a125e325c0a0e5be99a8fe69cbae9aab0e5ad905d1006180122104823d3adb15df08014ce5d6796b76ee128c85930033a1034303965326136396231363931386639480050c80158c8016211727363547970653f313b76616c75653d336a02400112120a100a0e5be99a8fe69cbae9aab0e5ad905d12044a0208001b0a190a0840616161746573741a0d00010000000800499602d20000080a060a04206162634baa02489a0145080120cb507800c80100f00100f80100900200c80200980300a00300b00300c00300d00300e803008a04021003900480c0829004b80400c00400ca0400f804dc8002880500044a0240011082010d0a076161617465737418012803")
+
+        /**
+         * forward[quote + dice + forward]
+         */
+        val complexForward =
+            "044b0a3a08d285d8cc041852288f831130dfffb6830638a7c4dfde87808080014a0e08d285d8cc042206e7b289e889b2a2010b10b0f083f7fbffffffff011a0d0a0b12050a030a016112024a00dc010a3a08d285d8cc0418522891831130e4ffb68306388fc594dd85808080014a0e08d285d8cc042206e7b289e889b2a2010b10b1f083f7fbffffffff011a9d010a9a011270ea026d088f831110d285d8cc0418dfffb6830620012a050a030a0161420a18a7c4dfde87808080014a400a2d08d285d8cc0410d285d8cc041852288f831130dfffb6830638a7c4dfde87808080014a0608d285d8cc04e001011a0f0a0d12050a030a016112044a02080050d285d8cc04121a0a180a0740e7b289e889b21a0d00010000000300499602d2000012060a040a02207212024a00520a3a08d285d8cc0418522892831130f2ffb6830638a8e4d1eb80808080014a0e08d285d8cc042206e7b289e889b2a2010b10b2f083f7fbffffffff011a140a12120c0a0a0a085be9aab0e5ad905d12024a00700a3a08d285d8cc04185228978311308d80b7830638a0e8bfa582808080014a0e08d285d8cc042206e7b289e889b2a2010b10b3f083f7fbffffffff011a320a30122a0a280a265be59088e5b9b6e8bdace58f915de8afb7e58d87e7baa7e696b0e78988e69cace69fa5e79c8b12024a00"
+                .let { s ->
+                    ProtoBuf.decodeFromHexString<List<MsgComm.Msg>>(s).flatMap { it.msgBody.richText.elems }
+                }
+
+        private fun decodeProto(p: String) = ProtoBuf.decodeFromHexString<List<ImMsgBody.Elem>>(p)
+    }
+
+//    override suspend fun downloadForwardMessage(bot: Bot, resourceId: String): List<ForwardMessage.Node> {
+//        return super.downloadForwardMessage(bot, resourceId)
+//    }
+
+    /**
+     * We cannot test LongMessage and MusicShare in unit tests (for now), but these tests will be sufficient
+     */
+    @Test
+    fun `recursive refinement`() = runBlockingUnit {
+        val map = listOf(
+            RefineTest(testCases.simpleQuote) {
+                expected {
+                    +QuoteReply(sourceStub(buildMessageChain {
+                        +"除非648"
+                    }))
+                    +At(1234567890) // sent by official client, redundant At?
+                    +" "
+                    +At(1234567890)
+                    +" sb"
+                }
+                light()
+                deep()
+            },
+            RefineTest(testCases.nestedQuote2) {
+                expected {
+                    +QuoteReply(sourceStub(buildMessageChain {
+                        +"@黄色 sb" // this is sent by official client.
+                    }))
+                    +At(1234567890) // mentions self
+                    +" xxx"
+                }
+                light()
+                deep()
+            },
+            RefineTest(testCases.nestedQuote3) {
+                expected {
+                    +QuoteReply(sourceStub(buildMessageChain {
+                        +"xxx" // official client does not handle nested quotes.
+                    }))
+                    +At(1234567890) // mentions self
+                    +" aaaaa"
+                }
+                light()
+                deep()
+            },
+            RefineTest(testCases.dice4) {
+                expected {
+                    +Dice(4)
+                }
+                light()
+                deep()
+            },
+            RefineTest(testCases.quoteDice4) {
+                expected {
+                    +QuoteReply(sourceStub(PlainText("[随机骰子]")))
+                    +At(1234567890)
+                    +" abc"
+                }
+                light()
+                deep()
+            },
+            RefineTest(testCases.complexForward) {
+                expected {
+                    +"a"
+                    +QuoteReply(sourceStub(PlainText("a")))
+                    +At(1234567890)
+                    +PlainText(" r")
+                    +PlainText("[骰子]") // client does not support
+                    +PlainText("[合并转发]请升级新版本查看") // client support but mirai does not.
+                }
+                deep() // deep only
+            }
+        )
+
+        for (test in map) {
+            if (test.testLight) {
+                testRecursiveRefine(test.list, test.expected, true)
+            }
+            if (test.testDeep) {
+                testRecursiveRefine(test.list, test.expected, false)
+            }
+        }
+    }
+}
+
+
+private fun sourceStub(
+    originalMessage: Message
+): OfflineMessageSourceImplData {
+    return OfflineMessageSourceImplData(
+        MessageSourceKind.GROUP, intArrayOf(), bot.id, 0, 0, 0, originalMessage.toMessageChain(), intArrayOf()
+    )
+}
+
+private suspend fun testRecursiveRefine(list: List<ImMsgBody.Elem>, expected: MessageChain, isLight: Boolean) {
+    val actual = buildMessageChain {
+        ReceiveMessageTransformer.joinToMessageChain(list, 0, MessageSourceKind.GROUP, bot, this)
+    }.let { c ->
+        if (isLight) {
+            c.refineLight(bot)
+        } else {
+            c.refineDeep(bot)
+        }
+    }
+    val color = object : PlatformLogger("") {
+        val yellow get() = Color.LIGHT_YELLOW.toString()
+        val green get() = Color.LIGHT_GREEN.toString()
+        val reset get() = Color.RESET.toString()
+    }
+
+    fun compare(expected: MessageChain, actual: MessageChain): Boolean {
+        if (expected.size != actual.size) return false
+        for ((e, a) in expected.zip(actual)) {
+            when (e) {
+                is QuoteReply -> {
+                    if (a !is QuoteReply) return false
+                    if (!compare(e.source.originalMessage, a.source.originalMessage)) return false
+                }
+                is MessageSource -> {
+                    if (a !is MessageSource) return false
+                    if (!compare(e.originalMessage, a.originalMessage)) return false
+                }
+                else -> {
+                    if (e != a) return false
+                }
+            }
+        }
+        return true
+    }
+
+    if (!compare(expected, actual))
+        throw AssertionError(
+            "\n" + """
+                Expected  str:${color.green}${expected}${color.reset}
+                Actual    str:${color.green}${actual}${color.reset}
+                
+                Expected json:${color.yellow}${expected.serializeToJsonString()}${color.reset}
+                Actual   json:${color.yellow}${actual.serializeToJsonString()}${color.reset}
+            """.trimIndent() + "\n"
+        )
+}
+
+private class RefineTest(
+    val list: List<ImMsgBody.Elem>,
+) {
+    lateinit var expected: MessageChain
+    fun expected(chain: MessageChainBuilder.() -> Unit) {
+        expected = buildMessageChain(chain)
+    }
+
+    var testLight: Boolean = false
+    var testDeep: Boolean = false
+    fun deep() {
+        testDeep = true
+    }
+
+    fun light() {
+        testLight = true
+    }
+}
+
+@Suppress("TestFunctionName")
+private fun RefineTest(list: List<ImMsgBody.Elem>, action: RefineTest.() -> Unit): RefineTest {
+    return RefineTest(list).apply(action)
+}