From 6d16d77dadd06ea0ea76bba0d25e0e4523302014 Mon Sep 17 00:00:00 2001 From: Karlatemp Date: Thu, 23 Dec 2021 18:23:59 +0800 Subject: [PATCH] Secrets Protection --- .../internal/data/builtins/AutoLoginConfig.kt | 21 +- mirai-core-utils/src/commonMain/kotlin/IO.kt | 7 - .../src/commonMain/kotlin/JvmNioBuffer.kt | 61 +++++ .../commonMain/kotlin/SecretsProtection.kt | 210 ++++++++++++++++++ .../jvmTest/kotlin/SecretsProtectionTest.kt | 36 +++ .../src/commonMain/kotlin/BotAccount.kt | 27 ++- 6 files changed, 344 insertions(+), 18 deletions(-) create mode 100644 mirai-core-utils/src/commonMain/kotlin/JvmNioBuffer.kt create mode 100644 mirai-core-utils/src/commonMain/kotlin/SecretsProtection.kt create mode 100644 mirai-core-utils/src/jvmTest/kotlin/SecretsProtectionTest.kt diff --git a/mirai-console/backend/mirai-console/src/internal/data/builtins/AutoLoginConfig.kt b/mirai-console/backend/mirai-console/src/internal/data/builtins/AutoLoginConfig.kt index d6e8a862f..4d15974b2 100644 --- a/mirai-console/backend/mirai-console/src/internal/data/builtins/AutoLoginConfig.kt +++ b/mirai-console/backend/mirai-console/src/internal/data/builtins/AutoLoginConfig.kt @@ -11,6 +11,7 @@ package net.mamoe.mirai.console.internal.data.builtins +import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import net.mamoe.mirai.console.command.CommandSender import net.mamoe.mirai.console.command.descriptor.CommandValueArgumentParser @@ -19,6 +20,8 @@ import net.mamoe.mirai.console.data.AutoSavePluginConfig import net.mamoe.mirai.console.data.ValueDescription import net.mamoe.mirai.console.data.value import net.mamoe.mirai.console.util.ConsoleExperimentalApi +import net.mamoe.mirai.utils.SecretsProtection +import net.mamoe.mirai.utils.readString import net.mamoe.yamlkt.Comment import net.mamoe.yamlkt.YamlDynamicSerializer @@ -42,12 +45,24 @@ public object AutoLoginConfig : AutoSavePluginConfig("AutoLogin") { val configuration: Map = mapOf(), ) { @Serializable - public data class Password( + public data class Password + /** @since 2.10.0 */ + public constructor( @Comment("密码种类, 可选 PLAIN 或 MD5") val kind: PasswordKind, + /** @since 2.10.0 */ + @SerialName("value") @Comment("密码内容, PLAIN 时为密码文本, MD5 时为 16 进制") - val value: String, - ) + val value0: SecretsProtection.EscapedString, + ) { + public constructor(kind: PasswordKind, value: String) : this( + kind, + SecretsProtection.EscapedString(SecretsProtection.escape(value.toByteArray())), + ) + + internal val value: String + get() = value0.asString + } @Suppress("EnumEntryName") @Serializable diff --git a/mirai-core-utils/src/commonMain/kotlin/IO.kt b/mirai-core-utils/src/commonMain/kotlin/IO.kt index 2d4a73f48..2ecceb8aa 100644 --- a/mirai-core-utils/src/commonMain/kotlin/IO.kt +++ b/mirai-core-utils/src/commonMain/kotlin/IO.kt @@ -18,7 +18,6 @@ import kotlinx.io.charsets.Charset import kotlinx.io.core.* import java.io.File import kotlin.text.String -import java.nio.Buffer as JNioBuffer public val EMPTY_BYTE_ARRAY: ByteArray = ByteArray(0) @@ -38,12 +37,6 @@ public inline fun ByteReadPacket.readPacketExact( n: Int = remaining.toInt()//not that safe but adequate ): ByteReadPacket = this.readBytes(n).toReadPacket() -public inline var JNioBuffer.pos: Int - get() = position() - set(value) { - position(value) - } - public typealias TlvMap = MutableMap diff --git a/mirai-core-utils/src/commonMain/kotlin/JvmNioBuffer.kt b/mirai-core-utils/src/commonMain/kotlin/JvmNioBuffer.kt new file mode 100644 index 000000000..26152d0be --- /dev/null +++ b/mirai-core-utils/src/commonMain/kotlin/JvmNioBuffer.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +@file:JvmMultifileClass +@file:JvmName("MiraiUtils") + +@file:Suppress("NOTHING_TO_INLINE", "UsePropertyAccessSyntax") + +package net.mamoe.mirai.utils + +import java.nio.Buffer +import java.nio.ByteBuffer +import java.nio.CharBuffer +import java.nio.charset.Charset + +public inline var Buffer.pos: Int + get() = position() + set(value) { + position(value) + } +public inline val Buffer.remaining: Int + get() = remaining() +public inline var Buffer.limit: Int + get() = limit() + set(value) { + limit(value) + } + +public inline fun Buffer.hasRemaining(size: Int): Boolean = remaining >= size + +public inline fun ByteBuffer.read(): Byte = get() +public inline fun ByteBuffer.readInt(): Int = getInt() +public inline fun ByteBuffer.readDouble(): Double = getDouble() +public inline fun ByteBuffer.readShort(): Short = getShort() +public inline fun ByteBuffer.readLong(): Long = getLong() +public inline fun ByteBuffer.readChar(): Char = getChar() +public inline fun ByteBuffer.readFloat(): Float = getFloat() + +public inline fun ByteBuffer.readBytes(dst: ByteArray) { + get(dst) +} + +public fun ByteBuffer.readBytes(): ByteArray { + val rsp = ByteArray(remaining) + readBytes(rsp) + return rsp +} + +public fun ByteBuffer.readToChars(charset: Charset = Charsets.UTF_8): CharBuffer { + return charset.decode(this) +} + +public fun ByteBuffer.readString(charset: Charset = Charsets.UTF_8): String { + return readToChars(charset).toString() +} diff --git a/mirai-core-utils/src/commonMain/kotlin/SecretsProtection.kt b/mirai-core-utils/src/commonMain/kotlin/SecretsProtection.kt new file mode 100644 index 000000000..29984e9a3 --- /dev/null +++ b/mirai-core-utils/src/commonMain/kotlin/SecretsProtection.kt @@ -0,0 +1,210 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package net.mamoe.mirai.utils + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import kotlinx.serialization.builtins.ByteArraySerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.serializer +import java.nio.ByteBuffer +import java.util.concurrent.ConcurrentLinkedDeque +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock + +/** + * 核心数据保护器 + * + * ### Why + * + * 有时候可能会发生 `OutOfMemoryError`, 如果存在 `-XX:+HeapDumpOnOutOfMemoryError`, 则 JVM 会生成一份系统内存打印以供 debug. + * 该报告包含全部内存信息, 包括各种数据, 核心数据以及, 机密数据 (如密码) + * + * 该内存报告唯一没有包含的数据就是 Native层数据, 包括且不限于 + * + * - `sun.misc.Unsafe.allocate()` + * - `java.nio.ByteBuffer.allocateDirect()` (Named `DirectByteBuffer`) + * - C/C++ (或其他语言) 的数据 + * + * *试验数据来源 `openjdk version "17" 2021-09-14, 64-Bit Server VM (build 17+35-2724, mixed mode, sharing)`* + * + * ### How it works + * + * 因为 Heap Dump 不存在 `DirectByteBuffer` 的实际数据, 所以可以通过该类隐藏关键数据. 等需要的时候在读取出来. + * 因为数据并没有直接存在于某个类字段中, 缺少数据关联, 也很难分析相关数据是什么数据 + */ +@Suppress("NOTHING_TO_INLINE", "UsePropertyAccessSyntax") +//@MiraiExperimentalApi +public object SecretsProtection { + private class NativeBufferWithLock( + @JvmField val buffer: ByteBuffer, + val lock: Lock = ReentrantLock(), + @JvmField @field:Volatile var lowRemainingHit: Int = 0, + @JvmField @field:Volatile var unusedHit: Int = 0, + ) { + companion object { + internal val lowRemainingHitUpdater = AtomicIntegerFieldUpdater.newUpdater( + NativeBufferWithLock::class.java, "lowRemainingHit" + ) + + internal val unusedHitUpdater = AtomicIntegerFieldUpdater.newUpdater( + NativeBufferWithLock::class.java, "unusedHit" + ) + } + } + + private val bufferSize = systemProp( + "mirai.secrets.protection.buffer.size", 0 + ).toInt().coerceAtLeast(2048) + + private val lowRemainingThreshold = bufferSize / 128 + + private val lowRemainingHitThreshold = systemProp( + "mirai.secrets.protection.threshold.low.remaining.hit", 10 + ).toInt().coerceAtLeast(1) + + private val pool = ConcurrentLinkedDeque() + + init { + val tmbuffer = ByteBuffer.allocateDirect(bufferSize) + if (!systemProp("mirai.secrets.protection.ignore.warning", false)) { + if (tmbuffer.javaClass === ByteBuffer.allocate(1).javaClass) { + val ps = System.err + synchronized(ps) { + ps.println("========================================================================================") + ps.println("Mirai SecretsProtection WARNING:") + ps.println() + ps.println("当前 JRE 实现没有为 `ByteBuffer.allocateDirect` 直接分配本地内存, 请更换其他 JRE.") + ps.println("这很有可能导致您的密码等敏感信息被带入内存报告中") + ps.println("可添加 JVM 参数 -Dmirai.secrets.protection.ignore.warning=true 来忽略此警告") + ps.println() + ps.println("Current JRE Implementation not using native memory for `ByteBuffer.allocateDirect`.") + ps.println("Please use another JRE.") + ps.println("It may cause your passwords to be dumped by other processes.") + ps.println("Suppress this warning by adding jvm option -Dmirai.secrets.protection.ignore.warning=true") + ps.println() + ps.println("========================================================================================") + } + } + } + pool.add(NativeBufferWithLock(tmbuffer)) + } + + /* + Implementation note: + + 1. 如果数据超过单个缓冲区大小，直接分配系统内存并返回 + 2. 查找首个可以放下数据的缓冲区，放入缓冲区并返回对应镜像 + 3. 如果没有可用缓冲区，分配新缓冲区并加入缓冲区池内 + + */ + @JvmStatic + public fun allocate(size: Int): ByteBuffer { + if (size >= bufferSize) { + return ByteBuffer.allocateDirect(size) + } + + fun putInto(buffer: NativeBufferWithLock): ByteBuffer { + // @context: buffer.locked = true + // @context: buffer.remaining >= buffer.size + + // 返回存储 data 的数据的 DirectByteBuffer + // ByteBuffer.slice(): DirectByteBuffer[start=pos, end=limit] + val mirror = buffer.buffer.let { buffer0 -> + buffer0.limit = buffer0.pos + size + buffer0.slice() + } + // 原始缓冲区复位 + buffer.buffer.let { buf -> buf.limit = buf.capacity() } + buffer.buffer.pos += size + + // 此缓冲区已无可用空间 + if (!buffer.buffer.hasRemaining()) { + pool.remove(buffer) + } + + return mirror + } + + pool.forEach bufferLoop@{ buffer -> + + val bufferRemaining = buffer.buffer.remaining + if (bufferRemaining >= size) { + if (buffer.lock.tryLock()) { + try { + if (buffer.buffer.hasRemaining(size)) { + NativeBufferWithLock.unusedHitUpdater.getAndIncrement(buffer) + return putInto(buffer) + } + } finally { + buffer.lock.unlock() + } + } + } + NativeBufferWithLock.unusedHitUpdater.getAndDecrement(buffer) + + // OOM Avoid + if (bufferRemaining <= lowRemainingThreshold) { + NativeBufferWithLock.lowRemainingHitUpdater.getAndDecrement(buffer) + if (buffer.lowRemainingHit >= lowRemainingHitThreshold) { + pool.remove(buffer) + } + } + if (buffer.unusedHit >= 20) { + pool.remove(buffer) + } + + } + val newBuffer = NativeBufferWithLock(ByteBuffer.allocateDirect(bufferSize)) + val rsp = putInto(newBuffer) + if (newBuffer.buffer.hasRemaining()) { + pool.add(newBuffer) + } + return rsp + } + + @JvmStatic + public fun escape(data: ByteArray): ByteBuffer { + return allocate(data.size).also { + it.put(data) + it.pos = 0 + } + } + + @JvmInline + @Serializable(EscapedStringSerializer::class) + public value class EscapedString( + public val data: ByteBuffer, + ) { + public val asString: String + get() = data.duplicate().readString() + } + + @JvmInline + @Serializable(EscapedByteBufferSerializer::class) + public value class EscapedByteBuffer( + public val data: ByteBuffer, + ) + + public object EscapedStringSerializer : KSerializer by String.serializer().map( + String.serializer().descriptor.copy("EscapedString"), + deserialize = { EscapedString(escape(it.toByteArray())) }, + serialize = { it.data.duplicate().readString() } + ) + + public object EscapedByteBufferSerializer : KSerializer by ByteArraySerializer().map( + ByteArraySerializer().descriptor.copy("EscapedByteBuffer"), + deserialize = { EscapedByteBuffer(escape(it)) }, + serialize = { it.data.duplicate().readBytes() } + ) + + +} diff --git a/mirai-core-utils/src/jvmTest/kotlin/SecretsProtectionTest.kt b/mirai-core-utils/src/jvmTest/kotlin/SecretsProtectionTest.kt new file mode 100644 index 000000000..53617e6ff --- /dev/null +++ b/mirai-core-utils/src/jvmTest/kotlin/SecretsProtectionTest.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package net.mamoe.mirai.utils + +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertTrue + +internal class SecretsProtectionTest { + @Test + fun chaosTest() = runBlocking { + repeat(500) { + launch { + val data = ByteArray((1..255).random()) { (0..255).random().toByte() } + val buffer = SecretsProtection.escape(data) + assertContentEquals( + data, buffer.duplicate().readBytes() + ) + delay(100) + assertContentEquals( + data, buffer.duplicate().readBytes() + ) + } + } + } +} diff --git a/mirai-core/src/commonMain/kotlin/BotAccount.kt b/mirai-core/src/commonMain/kotlin/BotAccount.kt index 2ff298b90..f218c30ca 100644 --- a/mirai-core/src/commonMain/kotlin/BotAccount.kt +++ b/mirai-core/src/commonMain/kotlin/BotAccount.kt @@ -10,25 +10,29 @@ package net.mamoe.mirai.internal -import net.mamoe.mirai.utils.MiraiExperimentalApi -import net.mamoe.mirai.utils.md5 -import net.mamoe.mirai.utils.toUHexString +import net.mamoe.mirai.utils.* +import java.nio.ByteBuffer internal data class BotAccount( @JvmSynthetic internal val id: Long, + @JvmSynthetic @MiraiExperimentalApi - val passwordMd5: ByteArray, // md5 + val passwordMd5Buffer: ByteBuffer, // md5 val phoneNumber: String = "" ) { init { - check(passwordMd5.size == 16) { - "Invalid passwordMd5: size must be 16 but got ${passwordMd5.size}. passwordMd5=${passwordMd5.toUHexString()}" + check(passwordMd5Buffer.remaining == 16) { + "Invalid passwordMd5: size must be 16 but got ${passwordMd5Buffer.remaining}. passwordMd5=${passwordMd5.toUHexString()}" } } + constructor(id: Long, passwordMd5: ByteArray, phoneNumber: String = "") : this( + id, SecretsProtection.escape(passwordMd5), phoneNumber + ) + constructor(id: Long, passwordPlainText: String, phoneNumber: String = "") : this( id, passwordPlainText.md5(), @@ -37,6 +41,13 @@ internal data class BotAccount( require(passwordPlainText.length <= 16) { "Password length must be at most 16." } } + @get:JvmSynthetic + @MiraiExperimentalApi + val passwordMd5: ByteArray + get() { + return passwordMd5Buffer.duplicate().readBytes() + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other == null || this::class != other::class) return false @@ -44,7 +55,7 @@ internal data class BotAccount( other as BotAccount if (id != other.id) return false - if (!passwordMd5.contentEquals(other.passwordMd5)) return false + if (passwordMd5Buffer != other.passwordMd5Buffer) return false return true } @@ -52,7 +63,7 @@ internal data class BotAccount( override fun hashCode(): Int { var result = id.hashCode() - result = 31 * result + passwordMd5.contentHashCode() + result = 31 * result + passwordMd5Buffer.hashCode() return result } } \ No newline at end of file