diff --git a/.gitignore b/.gitignore index fb2977862..da52815b9 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ bintray.key.txt /build-gpg-sign # Name for IDEA direction sorting build-secret-keys/ + +**/local.* \ No newline at end of file diff --git a/mirai-core/build.gradle.kts b/mirai-core/build.gradle.kts index e3503407b..e2753e28c 100644 --- a/mirai-core/build.gradle.kts +++ b/mirai-core/build.gradle.kts @@ -80,6 +80,7 @@ kotlin { commonTest { dependencies { implementation(kotlin("script-runtime")) + api(yamlkt) } } diff --git a/mirai-core/src/commonTest/kotlin/notice/RecordingNoticeHandler.kt b/mirai-core/src/commonTest/kotlin/notice/RecordingNoticeHandler.kt new file mode 100644 index 000000000..854c0b2d7 --- /dev/null +++ b/mirai-core/src/commonTest/kotlin/notice/RecordingNoticeHandler.kt @@ -0,0 +1,155 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package net.mamoe.mirai.internal.notice + +import kotlinx.atomicfu.atomic +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.Contextual +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.serializer +import net.mamoe.mirai.Mirai +import net.mamoe.mirai.internal.QQAndroidBot +import net.mamoe.mirai.internal.network.components.NoticeProcessorPipeline +import net.mamoe.mirai.internal.network.components.PipelineContext +import net.mamoe.mirai.internal.network.components.ProcessResult +import net.mamoe.mirai.internal.network.components.SimpleNoticeProcessor +import net.mamoe.mirai.internal.utils._miraiContentToString +import net.mamoe.mirai.internal.utils.io.ProtocolStruct +import net.mamoe.mirai.utils.* +import net.mamoe.yamlkt.Yaml +import net.mamoe.yamlkt.YamlBuilder +import kotlin.reflect.full.createType + +/** + * How to inject recorder? + * + * ``` + * bot.components[NoticeProcessorPipeline].registerProcessor(recorder) + * ``` + */ +internal class RecordingNoticeProcessor : SimpleNoticeProcessor(type()) { + private val id = atomic(0) + private val lock = Mutex() + + override suspend fun PipelineContext.processImpl(data: ProtocolStruct) { + lock.withLock { + id.getAndDecrement() + logger.info { "Recorded #${id.value} ${data::class.simpleName}" } + val serial = serialize(this, data) + logger.info { "original: $serial" } + logger.info { "desensitized: " + desensitize(serial) } + logger.info { "decoded: " + deserialize(desensitize(serial)).struct._miraiContentToString() } + } + } + + @Serializable + data class RecordNode( + val structType: String, + val struct: String, + val attributes: Map, + ) + + @Serializable + data class DeserializedRecord( + val attributes: TypeSafeMap, + val struct: ProtocolStruct + ) + + companion object { + private val logger = MiraiLogger.Factory.create(RecordingNoticeProcessor::class) + + private val yaml = Yaml { + // one-line + classSerialization = YamlBuilder.MapSerialization.FLOW_MAP + mapSerialization = YamlBuilder.MapSerialization.FLOW_MAP + listSerialization = YamlBuilder.ListSerialization.FLOW_SEQUENCE + stringSerialization = YamlBuilder.StringSerialization.DOUBLE_QUOTATION + encodeDefaultValues = false + } + + fun serialize(context: PipelineContext, data: ProtocolStruct): String { + return serialize(context.attributes.toMap(), data) + } + + fun serialize(attributes: Map, data: ProtocolStruct): String { + return yaml.encodeToString( + RecordNode( + data::class.java.name, + yaml.encodeToString(data), + attributes.mapValues { yaml.encodeToString(it.value) }) + ) + } + + fun deserialize(string: String): DeserializedRecord { + val (type, struct, attributes) = yaml.decodeFromString(RecordNode.serializer(), string) + val serializer = serializer(Class.forName(type).kotlin.createType()) + return DeserializedRecord( + TypeSafeMap(attributes.mapValues { yaml.decodeAnyFromString(it.value) }), + yaml.decodeFromString(serializer, struct).cast() + ) + } + + private val desensitizer by lateinitMutableProperty { + Desensitizer.create( + run> { + + val filename = + systemProp("mirai.network.recording.desensitization.filepath", "local.desensitization.yml") + + val file = + Thread.currentThread().contextClassLoader.getResource(filename) + ?: Thread.currentThread().contextClassLoader.getResource("recording/configs/$filename") + ?: error("Could not find desensitization configuration!") + + yaml.decodeFromString(file.readText()) + }.also { + logger.info { "Loaded ${it.size} desensitization rules." } + } + ) + } + + fun desensitize(string: String): String = desensitizer.desensitize(string) + } +} + +internal suspend fun NoticeProcessorPipeline.processRecording( + bot: QQAndroidBot, + record: RecordingNoticeProcessor.DeserializedRecord +): ProcessResult { + return this.process(bot, record.struct, record.attributes) +} + +internal class Desensitizer private constructor( + val rules: Map, +) { + companion object { + fun create(rules: Map): Desensitizer { + val map = HashMap() + map.putAll(rules) + rules.forEach { (t, u) -> + if (t.toLongOrNull() != null && u.toLongOrNull() != null) { + map.putIfAbsent( + Mirai.calculateGroupUinByGroupCode(t.toLong()).toString(), + Mirai.calculateGroupUinByGroupCode(u.toLong()).toString() + ) + } + } + return Desensitizer(rules) + } + } + + fun desensitize(value: String): String { + return rules.entries.fold(value) { acc, entry -> + acc.replace(entry.key, entry.value) + } + } +} \ No newline at end of file diff --git a/mirai-core/src/commonTest/kotlin/notice/test/RecordingNoticeProcessorTest.kt b/mirai-core/src/commonTest/kotlin/notice/test/RecordingNoticeProcessorTest.kt new file mode 100644 index 000000000..7a4568260 --- /dev/null +++ b/mirai-core/src/commonTest/kotlin/notice/test/RecordingNoticeProcessorTest.kt @@ -0,0 +1,87 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package net.mamoe.mirai.internal.notice.test + +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import net.mamoe.mirai.internal.MockBot +import net.mamoe.mirai.internal.network.components.AbstractPipelineContext +import net.mamoe.mirai.internal.network.components.ProcessResult +import net.mamoe.mirai.internal.notice.Desensitizer +import net.mamoe.mirai.internal.notice.RecordingNoticeProcessor +import net.mamoe.mirai.internal.test.AbstractTest +import net.mamoe.mirai.internal.utils.io.ProtocolStruct +import net.mamoe.mirai.utils.MutableTypeSafeMap +import net.mamoe.mirai.utils.TypeSafeMap +import net.mamoe.yamlkt.Yaml +import kotlin.test.Test +import kotlin.test.assertEquals + +internal class RecordingNoticeProcessorTest : AbstractTest() { + + class MyContext(attributes: TypeSafeMap) : AbstractPipelineContext(MockBot(), attributes) { + override suspend fun processAlso(data: ProtocolStruct, attributes: TypeSafeMap): ProcessResult { + throw UnsupportedOperationException() + } + } + + @Serializable + data class MyProtocolStruct( + val value: String + ) : ProtocolStruct + + @Test + fun `can serialize and deserialize reflectively`() { + val context = MyContext(MutableTypeSafeMap(mapOf("test" to "value"))) + val struct = MyProtocolStruct("vvv") + + val serialize = RecordingNoticeProcessor.serialize(context, struct) + println(serialize) + val deserialized = RecordingNoticeProcessor.deserialize(serialize) + + assertEquals(context.attributes, deserialized.attributes) + assertEquals(struct, deserialized.struct) + } + + @Test + fun `can read desensitization config`() { + val text = Thread.currentThread().contextClassLoader.getResource("recording/configs/test.desensitization.yml")!! + .readText() + val desensitizer = Desensitizer.create(Yaml.decodeFromString(text)) + assertEquals( + mapOf( + "123456789" to "111", + "987654321" to "111" + ), desensitizer.rules + ) + } + + @Test + fun `test desensitization`() { + val text = Thread.currentThread().contextClassLoader.getResource("recording/configs/test.desensitization.yml")!! + .readText() + val desensitizer = Desensitizer.create(Yaml.decodeFromString(text)) + + + assertEquals( + """ + "111": s1av12sad3 + "222": s1av12sad3 + """.trim(), + desensitizer.desensitize( + """ + "123456789": s1av12sad3 + "987654321": s1av12sad3 + """.trim() + ) + ) + + } +} \ No newline at end of file diff --git a/mirai-core/src/commonTest/resources/recording/configs/desensitization.yml b/mirai-core/src/commonTest/resources/recording/configs/desensitization.yml new file mode 100644 index 000000000..2e288e1c7 --- /dev/null +++ b/mirai-core/src/commonTest/resources/recording/configs/desensitization.yml @@ -0,0 +1,19 @@ +# Template for Desensitization in recordings +# +# Format: +# ``` +# : +# ``` +# +# If key is a number, its group uin counterpart will also be processed, with calculated replacer. +# +# For example, if your account id is 147258369, you may add: +# ``` +# 147258369: 123456 +# ``` +# Then your id will be replaced with 123456. +# +# +# To use desensitization, duplicate this file into name "local.desensitization.yml". + +123456789: 111 \ No newline at end of file diff --git a/mirai-core/src/commonTest/resources/recording/configs/test.desensitization.yml b/mirai-core/src/commonTest/resources/recording/configs/test.desensitization.yml new file mode 100644 index 000000000..43ca7ec12 --- /dev/null +++ b/mirai-core/src/commonTest/resources/recording/configs/test.desensitization.yml @@ -0,0 +1,2 @@ +123456789: 111 +987654321: 222 \ No newline at end of file