Secrets Protection

mirai-console/backend/mirai-console/src/internal/data/builtins/AutoLoginConfig.kt

@@ -11,6 +11,7 @@
 
 package net.mamoe.mirai.console.internal.data.builtins
 
+import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import net.mamoe.mirai.console.command.CommandSender
 import net.mamoe.mirai.console.command.descriptor.CommandValueArgumentParser
@@ -19,6 +20,8 @@ import net.mamoe.mirai.console.data.AutoSavePluginConfig
 import net.mamoe.mirai.console.data.ValueDescription
 import net.mamoe.mirai.console.data.value
 import net.mamoe.mirai.console.util.ConsoleExperimentalApi
+import net.mamoe.mirai.utils.SecretsProtection
+import net.mamoe.mirai.utils.readString
 import net.mamoe.yamlkt.Comment
 import net.mamoe.yamlkt.YamlDynamicSerializer
 
@@ -42,12 +45,24 @@ public object AutoLoginConfig : AutoSavePluginConfig("AutoLogin") {
         val configuration: Map<ConfigurationKey, @Serializable(with = YamlDynamicSerializer::class) Any> = mapOf(),
     ) {
         @Serializable
-        public data class Password(
+        public data class Password
+        /** @since 2.10.0 */
+        public constructor(
             @Comment("密码种类, 可选 PLAIN 或 MD5")
             val kind: PasswordKind,
+            /** @since 2.10.0 */
+            @SerialName("value")
             @Comment("密码内容, PLAIN 时为密码文本, MD5 时为 16 进制")
-            val value: String,
-        )
+            val value0: SecretsProtection.EscapedString,
+        ) {
+            public constructor(kind: PasswordKind, value: String) : this(
+                kind,
+                SecretsProtection.EscapedString(SecretsProtection.escape(value.toByteArray())),
+            )
+
+            internal val value: String
+                get() = value0.asString
+        }
 
         @Suppress("EnumEntryName")
         @Serializable

mirai-core-utils/src/commonMain/kotlin/IO.kt

@@ -18,7 +18,6 @@ import kotlinx.io.charsets.Charset
 import kotlinx.io.core.*
 import java.io.File
 import kotlin.text.String
-import java.nio.Buffer as JNioBuffer
 
 public val EMPTY_BYTE_ARRAY: ByteArray = ByteArray(0)
 
@@ -38,12 +37,6 @@ public inline fun ByteReadPacket.readPacketExact(
     n: Int = remaining.toInt()//not that safe but adequate
 ): ByteReadPacket = this.readBytes(n).toReadPacket()
 
-public inline var JNioBuffer.pos: Int
-    get() = position()
-    set(value) {
-        position(value)
-    }
-
 
 public typealias TlvMap = MutableMap<Int, ByteArray>
 

mirai-core-utils/src/commonMain/kotlin/JvmNioBuffer.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+@file:JvmMultifileClass
+@file:JvmName("MiraiUtils")
+
+@file:Suppress("NOTHING_TO_INLINE", "UsePropertyAccessSyntax")
+
+package net.mamoe.mirai.utils
+
+import java.nio.Buffer
+import java.nio.ByteBuffer
+import java.nio.CharBuffer
+import java.nio.charset.Charset
+
+public inline var Buffer.pos: Int
+    get() = position()
+    set(value) {
+        position(value)
+    }
+public inline val Buffer.remaining: Int
+    get() = remaining()
+public inline var Buffer.limit: Int
+    get() = limit()
+    set(value) {
+        limit(value)
+    }
+
+public inline fun Buffer.hasRemaining(size: Int): Boolean = remaining >= size
+
+public inline fun ByteBuffer.read(): Byte = get()
+public inline fun ByteBuffer.readInt(): Int = getInt()
+public inline fun ByteBuffer.readDouble(): Double = getDouble()
+public inline fun ByteBuffer.readShort(): Short = getShort()
+public inline fun ByteBuffer.readLong(): Long = getLong()
+public inline fun ByteBuffer.readChar(): Char = getChar()
+public inline fun ByteBuffer.readFloat(): Float = getFloat()
+
+public inline fun ByteBuffer.readBytes(dst: ByteArray) {
+    get(dst)
+}
+
+public fun ByteBuffer.readBytes(): ByteArray {
+    val rsp = ByteArray(remaining)
+    readBytes(rsp)
+    return rsp
+}
+
+public fun ByteBuffer.readToChars(charset: Charset = Charsets.UTF_8): CharBuffer {
+    return charset.decode(this)
+}
+
+public fun ByteBuffer.readString(charset: Charset = Charsets.UTF_8): String {
+    return readToChars(charset).toString()
+}

mirai-core-utils/src/commonMain/kotlin/SecretsProtection.kt

@@ -0,0 +1,210 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.mirai.utils
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.builtins.ByteArraySerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.serializer
+import java.nio.ByteBuffer
+import java.util.concurrent.ConcurrentLinkedDeque
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater
+import java.util.concurrent.locks.Lock
+import java.util.concurrent.locks.ReentrantLock
+
+/**
+ * 核心数据保护器
+ *
+ * ### Why
+ *
+ * 有时候可能会发生 `OutOfMemoryError`, 如果存在 `-XX:+HeapDumpOnOutOfMemoryError`, 则 JVM 会生成一份系统内存打印以供 debug.
+ * 该报告包含全部内存信息, 包括各种数据, 核心数据以及, 机密数据 (如密码)
+ *
+ * 该内存报告唯一没有包含的数据就是 Native层数据, 包括且不限于
+ *
+ * - `sun.misc.Unsafe.allocate()`
+ * - `java.nio.ByteBuffer.allocateDirect()` (Named `DirectByteBuffer`)
+ * - C/C++ (或其他语言) 的数据
+ *
+ * *试验数据来源 `openjdk version "17" 2021-09-14, 64-Bit Server VM (build 17+35-2724, mixed mode, sharing)`*
+ *
+ * ### How it works
+ *
+ * 因为 Heap Dump 不存在 `DirectByteBuffer` 的实际数据, 所以可以通过该类隐藏关键数据. 等需要的时候在读取出来.
+ * 因为数据并没有直接存在于某个类字段中, 缺少数据关联, 也很难分析相关数据是什么数据
+ */
+@Suppress("NOTHING_TO_INLINE", "UsePropertyAccessSyntax")
+//@MiraiExperimentalApi
+public object SecretsProtection {
+    private class NativeBufferWithLock(
+        @JvmField val buffer: ByteBuffer,
+        val lock: Lock = ReentrantLock(),
+        @JvmField @field:Volatile var lowRemainingHit: Int = 0,
+        @JvmField @field:Volatile var unusedHit: Int = 0,
+    ) {
+        companion object {
+            internal val lowRemainingHitUpdater = AtomicIntegerFieldUpdater.newUpdater(
+                NativeBufferWithLock::class.java, "lowRemainingHit"
+            )
+
+            internal val unusedHitUpdater = AtomicIntegerFieldUpdater.newUpdater(
+                NativeBufferWithLock::class.java, "unusedHit"
+            )
+        }
+    }
+
+    private val bufferSize = systemProp(
+        "mirai.secrets.protection.buffer.size", 0
+    ).toInt().coerceAtLeast(2048)
+
+    private val lowRemainingThreshold = bufferSize / 128
+
+    private val lowRemainingHitThreshold = systemProp(
+        "mirai.secrets.protection.threshold.low.remaining.hit", 10
+    ).toInt().coerceAtLeast(1)
+
+    private val pool = ConcurrentLinkedDeque<NativeBufferWithLock>()
+
+    init {
+        val tmbuffer = ByteBuffer.allocateDirect(bufferSize)
+        if (!systemProp("mirai.secrets.protection.ignore.warning", false)) {
+            if (tmbuffer.javaClass === ByteBuffer.allocate(1).javaClass) {
+                val ps = System.err
+                synchronized(ps) {
+                    ps.println("========================================================================================")
+                    ps.println("Mirai SecretsProtection WARNING:")
+                    ps.println()
+                    ps.println("当前 JRE 实现没有为 `ByteBuffer.allocateDirect` 直接分配本地内存, 请更换其他 JRE.")
+                    ps.println("这很有可能导致您的密码等敏感信息被带入内存报告中")
+                    ps.println("可添加 JVM 参数 -Dmirai.secrets.protection.ignore.warning=true 来忽略此警告")
+                    ps.println()
+                    ps.println("Current JRE Implementation not using native memory for `ByteBuffer.allocateDirect`.")
+                    ps.println("Please use another JRE.")
+                    ps.println("It may cause your passwords to be dumped by other processes.")
+                    ps.println("Suppress this warning by adding jvm option -Dmirai.secrets.protection.ignore.warning=true")
+                    ps.println()
+                    ps.println("========================================================================================")
+                }
+            }
+        }
+        pool.add(NativeBufferWithLock(tmbuffer))
+    }
+
+    /*
+    Implementation note:
+
+    1. 如果数据超过单个缓冲区大小，直接分配系统内存并返回
+    2. 查找首个可以放下数据的缓冲区，放入缓冲区并返回对应镜像
+    3. 如果没有可用缓冲区，分配新缓冲区并加入缓冲区池内
+
+     */
+    @JvmStatic
+    public fun allocate(size: Int): ByteBuffer {
+        if (size >= bufferSize) {
+            return ByteBuffer.allocateDirect(size)
+        }
+
+        fun putInto(buffer: NativeBufferWithLock): ByteBuffer {
+            // @context: buffer.locked = true
+            // @context: buffer.remaining >= buffer.size
+
+            // 返回存储 data 的数据的 DirectByteBuffer
+            //   ByteBuffer.slice(): DirectByteBuffer[start=pos, end=limit]
+            val mirror = buffer.buffer.let { buffer0 ->
+                buffer0.limit = buffer0.pos + size
+                buffer0.slice()
+            }
+            // 原始缓冲区复位
+            buffer.buffer.let { buf -> buf.limit = buf.capacity() }
+            buffer.buffer.pos += size
+
+            // 此缓冲区已无可用空间
+            if (!buffer.buffer.hasRemaining()) {
+                pool.remove(buffer)
+            }
+
+            return mirror
+        }
+
+        pool.forEach bufferLoop@{ buffer ->
+
+            val bufferRemaining = buffer.buffer.remaining
+            if (bufferRemaining >= size) {
+                if (buffer.lock.tryLock()) {
+                    try {
+                        if (buffer.buffer.hasRemaining(size)) {
+                            NativeBufferWithLock.unusedHitUpdater.getAndIncrement(buffer)
+                            return putInto(buffer)
+                        }
+                    } finally {
+                        buffer.lock.unlock()
+                    }
+                }
+            }
+            NativeBufferWithLock.unusedHitUpdater.getAndDecrement(buffer)
+
+            // OOM Avoid
+            if (bufferRemaining <= lowRemainingThreshold) {
+                NativeBufferWithLock.lowRemainingHitUpdater.getAndDecrement(buffer)
+                if (buffer.lowRemainingHit >= lowRemainingHitThreshold) {
+                    pool.remove(buffer)
+                }
+            }
+            if (buffer.unusedHit >= 20) {
+                pool.remove(buffer)
+            }
+
+        }
+        val newBuffer = NativeBufferWithLock(ByteBuffer.allocateDirect(bufferSize))
+        val rsp = putInto(newBuffer)
+        if (newBuffer.buffer.hasRemaining()) {
+            pool.add(newBuffer)
+        }
+        return rsp
+    }
+
+    @JvmStatic
+    public fun escape(data: ByteArray): ByteBuffer {
+        return allocate(data.size).also {
+            it.put(data)
+            it.pos = 0
+        }
+    }
+
+    @JvmInline
+    @Serializable(EscapedStringSerializer::class)
+    public value class EscapedString(
+        public val data: ByteBuffer,
+    ) {
+        public val asString: String
+            get() = data.duplicate().readString()
+    }
+
+    @JvmInline
+    @Serializable(EscapedByteBufferSerializer::class)
+    public value class EscapedByteBuffer(
+        public val data: ByteBuffer,
+    )
+
+    public object EscapedStringSerializer : KSerializer<EscapedString> by String.serializer().map(
+        String.serializer().descriptor.copy("EscapedString"),
+        deserialize = { EscapedString(escape(it.toByteArray())) },
+        serialize = { it.data.duplicate().readString() }
+    )
+
+    public object EscapedByteBufferSerializer : KSerializer<EscapedByteBuffer> by ByteArraySerializer().map(
+        ByteArraySerializer().descriptor.copy("EscapedByteBuffer"),
+        deserialize = { EscapedByteBuffer(escape(it)) },
+        serialize = { it.data.duplicate().readBytes() }
+    )
+
+
+}

mirai-core-utils/src/jvmTest/kotlin/SecretsProtectionTest.kt

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.mirai.utils
+
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+import kotlin.test.assertContentEquals
+import kotlin.test.assertTrue
+
+internal class SecretsProtectionTest {
+    @Test
+    fun chaosTest() = runBlocking<Unit> {
+        repeat(500) {
+            launch {
+                val data = ByteArray((1..255).random()) { (0..255).random().toByte() }
+                val buffer = SecretsProtection.escape(data)
+                assertContentEquals(
+                    data, buffer.duplicate().readBytes()
+                )
+                delay(100)
+                assertContentEquals(
+                    data, buffer.duplicate().readBytes()
+                )
+            }
+        }
+    }
+}

mirai-core/src/commonMain/kotlin/BotAccount.kt

@@ -10,25 +10,29 @@
 
 package net.mamoe.mirai.internal
 
-import net.mamoe.mirai.utils.MiraiExperimentalApi
-import net.mamoe.mirai.utils.md5
-import net.mamoe.mirai.utils.toUHexString
+import net.mamoe.mirai.utils.*
+import java.nio.ByteBuffer
 
 internal data class BotAccount(
     @JvmSynthetic
     internal val id: Long,
+
     @JvmSynthetic
     @MiraiExperimentalApi
-    val passwordMd5: ByteArray, // md5
+    val passwordMd5Buffer: ByteBuffer, // md5
 
     val phoneNumber: String = ""
 ) {
     init {
-        check(passwordMd5.size == 16) {
-            "Invalid passwordMd5: size must be 16 but got ${passwordMd5.size}. passwordMd5=${passwordMd5.toUHexString()}"
+        check(passwordMd5Buffer.remaining == 16) {
+            "Invalid passwordMd5: size must be 16 but got ${passwordMd5Buffer.remaining}. passwordMd5=${passwordMd5.toUHexString()}"
         }
     }
 
+    constructor(id: Long, passwordMd5: ByteArray, phoneNumber: String = "") : this(
+        id, SecretsProtection.escape(passwordMd5), phoneNumber
+    )
+
     constructor(id: Long, passwordPlainText: String, phoneNumber: String = "") : this(
         id,
         passwordPlainText.md5(),
@@ -37,6 +41,13 @@ internal data class BotAccount(
         require(passwordPlainText.length <= 16) { "Password length must be at most 16." }
     }
 
+    @get:JvmSynthetic
+    @MiraiExperimentalApi
+    val passwordMd5: ByteArray
+        get() {
+            return passwordMd5Buffer.duplicate().readBytes()
+        }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other == null || this::class != other::class) return false
@@ -44,7 +55,7 @@ internal data class BotAccount(
         other as BotAccount
 
         if (id != other.id) return false
-        if (!passwordMd5.contentEquals(other.passwordMd5)) return false
+        if (passwordMd5Buffer != other.passwordMd5Buffer) return false
 
         return true
     }
@@ -52,7 +63,7 @@ internal data class BotAccount(
 
     override fun hashCode(): Int {
         var result = id.hashCode()
-        result = 31 * result + passwordMd5.contentHashCode()
+        result = 31 * result + passwordMd5Buffer.hashCode()
         return result
     }
 }