Simplify cache configuration.

Improve cache saving: - Use ProtoBuf to serialize BdhSession - More readable servers.json
2025-03-25 06:50:09 +08:00 · 2021-02-10 15:20:39 +08:00 · 2021-02-10 15:20:39 +08:00 · cda6aaa79b
commit cda6aaa79b
parent 4668a2d91b
10 changed files with 124 additions and 106 deletions
--- a/binary-compatibility-validator/api/binary-compatibility-validator.api
+++ b/binary-compatibility-validator/api/binary-compatibility-validator.api
@ -5365,12 +5365,11 @@ public class net/mamoe/mirai/utils/BotConfiguration {
 	public static synthetic fun fileBasedDeviceInfo$default (Lnet/mamoe/mirai/utils/BotConfiguration;Ljava/lang/String;ILjava/lang/Object;)V
 	public final fun getAutoReconnectOnForceOffline ()Z
 	public final fun getBotLoggerSupplier ()Lkotlin/jvm/functions/Function1;
-	public final fun getCacheDirSupplier ()Lkotlin/jvm/functions/Function0;
+	public final fun getCacheDir ()Ljava/io/File;
+	public final fun getContactListCache ()Lnet/mamoe/mirai/utils/BotConfiguration$ContactListCache;
 	public static final fun getDefault ()Lnet/mamoe/mirai/utils/BotConfiguration;
 	public final fun getDeviceInfo ()Lkotlin/jvm/functions/Function1;
 	public final fun getFirstReconnectDelayMillis ()J
-	public final fun getFriendListCache ()Lnet/mamoe/mirai/utils/BotConfiguration$FriendListCache;
-	public final fun getGroupMemberListCache ()Lnet/mamoe/mirai/utils/BotConfiguration$GroupMemberListCache;
 	public final fun getHeartbeatPeriodMillis ()J
 	public final fun getHeartbeatTimeoutMillis ()J
 	public final fun getHighwayUploadCoroutineCount ()I
@ -5407,11 +5406,10 @@ public class net/mamoe/mirai/utils/BotConfiguration {
 	public static synthetic fun redirectNetworkLogToFile$default (Lnet/mamoe/mirai/utils/BotConfiguration;Ljava/io/File;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V
 	public final fun setAutoReconnectOnForceOffline (Z)V
 	public final fun setBotLoggerSupplier (Lkotlin/jvm/functions/Function1;)V
-	public final fun setCacheDirSupplier (Lkotlin/jvm/functions/Function0;)V
+	public final fun setCacheDir (Ljava/io/File;)V
+	public final fun setContactListCache (Lnet/mamoe/mirai/utils/BotConfiguration$ContactListCache;)V
 	public final fun setDeviceInfo (Lkotlin/jvm/functions/Function1;)V
 	public final fun setFirstReconnectDelayMillis (J)V
-	public final fun setFriendListCache (Lnet/mamoe/mirai/utils/BotConfiguration$FriendListCache;)V
-	public final fun setGroupMemberListCache (Lnet/mamoe/mirai/utils/BotConfiguration$GroupMemberListCache;)V
 	public final fun setHeartbeatPeriodMillis (J)V
 	public final fun setHeartbeatTimeoutMillis (J)V
 	public final fun setHighwayUploadCoroutineCount (I)V
@ -5432,22 +5430,14 @@ public final class net/mamoe/mirai/utils/BotConfiguration$Companion {
 public abstract interface annotation class net/mamoe/mirai/utils/BotConfiguration$ConfigurationDsl : java/lang/annotation/Annotation {
 }

-public final class net/mamoe/mirai/utils/BotConfiguration$FriendListCache {
+public final class net/mamoe/mirai/utils/BotConfiguration$ContactListCache {
 	public fun <init> ()V
-	public fun <init> (Ljava/io/File;)V
-	public fun <init> (Ljava/io/File;J)V
-	public synthetic fun <init> (Ljava/io/File;JILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun getCacheFile ()Ljava/io/File;
-	public final fun getSaveIntervalMillis ()J
-}
-
-public final class net/mamoe/mirai/utils/BotConfiguration$GroupMemberListCache {
-	public fun <init> ()V
-	public fun <init> (Ljava/io/File;)V
-	public fun <init> (Ljava/io/File;J)V
-	public synthetic fun <init> (Ljava/io/File;JILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun getCacheDir ()Ljava/io/File;
+	public final fun getFriendListCacheEnabled ()Z
+	public final fun getGroupMemberListCacheEnabled ()Z
 	public final fun getSaveIntervalMillis ()J
+	public final fun setFriendListCacheEnabled (Z)V
+	public final fun setGroupMemberListCacheEnabled (Z)V
+	public final fun setSaveIntervalMillis (J)V
 }

 public final class net/mamoe/mirai/utils/BotConfiguration$MiraiProtocol : java/lang/Enum {
--- a/mirai-core-api/src/commonMain/kotlin/utils/BotConfiguration.kt
+++ b/mirai-core-api/src/commonMain/kotlin/utils/BotConfiguration.kt
@ -56,11 +56,6 @@ public open class BotConfiguration { // open for Java
     */
    public var workingDir: File = File(".")

-    /**
-     * 缓存数据目录
-     */
-    public var cacheDirSupplier: (() -> File) = { workingDir.resolve("cache") }
-
    /**
     * Json 序列化器, 使用 'kotlinx.serialization'
     */
@ -388,29 +383,37 @@ public open class BotConfiguration { // open for Java
    //////////////////////////////////////////////////////////////////////////

    /**
-     * 非 `null` 时启用好友列表缓存, 加快初始化速度. 在启用后将会在下载好友列表后保存到文件, 并在修改时自动保存.
+     * 缓存数据目录, 相对于 [workingDir]
     * @since 2.4
-     * @see disableFriendListCache
     */
-    public var friendListCache: FriendListCache? = FriendListCache()
+    public var cacheDir: File = File("cache")

    /**
-     * 好友列表缓存设置.
+     * 联系人信息缓存配置. 将会保存在 [cacheDir] 中 `contacts` 目录
     * @since 2.4
-     * @see friendListCache
     */
-    public class FriendListCache @JvmOverloads constructor(
+    public var contactListCache: ContactListCache = ContactListCache()
+
+    /**
+     * 联系人信息缓存配置
+     * @since 2.4
+     */
+    public class ContactListCache {
        /**
-         * 缓存文件位置, 相对于 [cacheDirSupplier] 的路径.
-         *
-         * 注意: 保存的文件仅供内部使用, 将来可能会变化.
+         * 在有修改时自动保存间隔. 默认 60 秒. 在每次登录完成后有修改时都会立即保存一次.
         */
-        public val cacheFile: File = File("friends.json"),
+        public var saveIntervalMillis: Long = 60_000
+
        /**
-         * 在有好友列表修改时自动保存间隔
+         * 开启好友列表缓存
         */
-        public val saveIntervalMillis: Long = 60_000,
-    )
+        public var friendListCacheEnabled: Boolean = true
+
+        /**
+         * 开启好友列表缓存
+         */
+        public var groupMemberListCacheEnabled: Boolean = true
+    }

    /**
     * 禁用好友列表缓存.
@ -418,53 +421,26 @@ public open class BotConfiguration { // open for Java
     */
    @ConfigurationDsl
    public fun disableFriendListCache() {
-        friendListCache = null
+        contactListCache.friendListCacheEnabled = false
    }

-
-
-    /**
-     * 非 `null` 时启用群成员列表缓存, 加快初始化速度. 在启用后将会在下载群成员列表后保存到文件, 并在修改时自动保存.
-     * @since 2.4
-     * @see disableGroupMemberListCache
-     */
-    public var groupMemberListCache: GroupMemberListCache? = GroupMemberListCache()
-
-    /**
-     * 群成员列表缓存设置.
-     * @since 2.4
-     * @see groupMemberListCache
-     */
-    public class GroupMemberListCache @JvmOverloads constructor(
-        /**
-         * 缓存目录位置, 相对于 [cacheDirSupplier] 的路径.
-         *
-         * 注意: 保存的文件仅供内部使用, 将来可能会变化.
-         */
-        public val cacheDir: File = File("groups"),
-        /**
-         * 在有成员列表修改时自动保存间隔
-         */
-        public val saveIntervalMillis: Long = 60_000,
-    )
-
    /**
     * 禁用群成员列表缓存.
     * @since 2.4
     */
    @ConfigurationDsl
    public fun disableGroupMemberListCache() {
-        groupMemberListCache = null
+        contactListCache.groupMemberListCacheEnabled = false
    }

    /**
-     * 禁用好友列表, 群成员列表, 陌生人列表的缓存.
+     * 禁用好友列表和群成员列表的缓存.
     * @since 2.4
     */
    @ConfigurationDsl
    public fun disableContactCaches() {
-        disableFriendListCache()
-        disableGroupMemberListCache()
+        contactListCache.friendListCacheEnabled = false
+        contactListCache.groupMemberListCacheEnabled = false
    }

    ///////////////////////////////////////////////////////////////////////////
@ -475,7 +451,6 @@ public open class BotConfiguration { // open for Java
        return BotConfiguration().also { new ->
            // To structural order
            new.workingDir = workingDir
-            new.cacheDirSupplier = cacheDirSupplier
            new.json = json
            new.parentCoroutineContext = parentCoroutineContext
            new.heartbeatPeriodMillis = heartbeatPeriodMillis
@ -490,8 +465,8 @@ public open class BotConfiguration { // open for Java
            new.deviceInfo = deviceInfo
            new.botLoggerSupplier = botLoggerSupplier
            new.networkLoggerSupplier = networkLoggerSupplier
-            new.friendListCache = friendListCache
-            new.groupMemberListCache = groupMemberListCache
+            new.cacheDir = cacheDir
+            new.contactListCache = contactListCache
        }
    }

--- a/mirai-core-utils/src/commonMain/kotlin/IO.kt
+++ b/mirai-core-utils/src/commonMain/kotlin/IO.kt
@ -131,4 +131,10 @@ public fun File.createFileIfNotExists() {
        this.parentFile.mkdirs()
        this.createNewFile()
    }
-}
+}
+
+public fun File.resolveCreateFile(relative: String): File = this.resolve(relative).apply { createFileIfNotExists() }
+public fun File.resolveCreateFile(relative: File): File = this.resolve(relative).apply { createFileIfNotExists() }
+
+public fun File.resolveMkdir(relative: String): File = this.resolve(relative).apply { mkdirs() }
+public fun File.resolveMkdir(relative: File): File = this.resolve(relative).apply { mkdirs() }
--- a/mirai-core-utils/src/commonMain/kotlin/RandomUtils.kt
+++ b/mirai-core-utils/src/commonMain/kotlin/RandomUtils.kt
@ -7,6 +7,10 @@
 *  https://github.com/mamoe/mirai/blob/master/LICENSE
 */

+@file:JvmMultifileClass
+@file:JvmName("MiraiUtils")
+
+
 package net.mamoe.mirai.utils


--- a/mirai-core-utils/src/commonMain/kotlin/Serialization.kt
+++ b/mirai-core-utils/src/commonMain/kotlin/Serialization.kt
@ -7,13 +7,17 @@
 * https://github.com/mamoe/mirai/blob/master/LICENSE
 */

+@file:JvmMultifileClass
+@file:JvmName("MiraiUtils")
+
+
 package net.mamoe.mirai.utils

 import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.StringFormat
 import java.io.File

-public fun <T> File.loadAs(
+public fun <T> File.loadNotBlankAs(
    serializer: DeserializationStrategy<T>,
    stringFormat: StringFormat,
 ): T? {
--- a/mirai-core-utils/src/commonMain/kotlin/WeakRef.kt
+++ b/mirai-core-utils/src/commonMain/kotlin/WeakRef.kt
@ -7,6 +7,10 @@
 *  https://github.com/mamoe/mirai/blob/master/LICENSE
 */

+@file:JvmMultifileClass
+@file:JvmName("MiraiUtils")
+
+
@file:Suppress("unused", "NOTHING_TO_INLINE")

 package net.mamoe.mirai.utils
--- a/mirai-core/src/commonMain/kotlin/QQAndroidBot.kt
+++ b/mirai-core/src/commonMain/kotlin/QQAndroidBot.kt
@ -32,6 +32,7 @@ import net.mamoe.mirai.internal.network.protocol.packet.OutgoingPacketWithRespTy
 import net.mamoe.mirai.internal.network.protocol.packet.chat.*
 import net.mamoe.mirai.internal.network.protocol.packet.login.StatSvc
 import net.mamoe.mirai.internal.utils.ScheduledJob
+import net.mamoe.mirai.internal.utils.friendCacheFile
 import net.mamoe.mirai.message.data.*
 import net.mamoe.mirai.network.LoginFailedException
 import net.mamoe.mirai.utils.*
@ -81,8 +82,9 @@ internal class QQAndroidBot constructor(
    override val friends: ContactList<Friend> = ContactList()

    val friendListCache: FriendListCache? by lazy {
-        configuration.friendListCache?.cacheFile?.let { cacheFile ->
-            val ret = configuration.cacheDirSupplier().resolve(cacheFile).loadAs(FriendListCache.serializer(), JsonForCache) ?: FriendListCache()
+        if (!configuration.contactListCache.friendListCacheEnabled) return@lazy null
+            val file = configuration.friendCacheFile()
+            val ret = file.loadNotBlankAs(FriendListCache.serializer(), JsonForCache) ?: FriendListCache()

            @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
            bot.eventChannel.parentScope(this@QQAndroidBot)
@ -90,31 +92,29 @@ internal class QQAndroidBot constructor(
                    friendListSaver?.notice()
                }
            ret
-        }
    }

    val groupMemberListCaches: GroupMemberListCaches? by lazy {
-        if (configuration.groupMemberListCache!= null) {
+        if (!configuration.contactListCache.groupMemberListCacheEnabled) {
+            return@lazy null
+        }
            GroupMemberListCaches(this)
-        } else null
    }

    private val friendListSaver by lazy {
-        configuration.friendListCache?.let { friendListCache: BotConfiguration.FriendListCache ->
-
-            ScheduledJob(coroutineContext, friendListCache.saveIntervalMillis.milliseconds) {
+        if (!configuration.contactListCache.friendListCacheEnabled) return@lazy null
+            ScheduledJob(coroutineContext, configuration.contactListCache.saveIntervalMillis.milliseconds) {
                runBIO { saveFriendCache() }
            }
-        }
    }
+
    fun saveFriendCache() {
-        val friendListCache = friendListCache
-        if (friendListCache != null) {
-            configuration.friendListCache?.cacheFile?.let { configuration.workingDir.resolve(it) }?.run {
-                createFileIfNotExists()
-                writeText(JsonForCache.encodeToString(FriendListCache.serializer(), friendListCache))
-                bot.network.logger.info { "Saved ${friendListCache.list.size} friends to local cache." }
-            }
+        val friendListCache = friendListCache ?: return
+
+        configuration.friendCacheFile().run {
+            createFileIfNotExists()
+            writeText(JsonForCache.encodeToString(FriendListCache.serializer(), friendListCache))
+            bot.network.logger.info { "Saved ${friendListCache.list.size} friends to local cache." }
        }
    }

--- a/mirai-core/src/commonMain/kotlin/network/ContactListCache.kt
+++ b/mirai-core/src/commonMain/kotlin/network/ContactListCache.kt
@ -11,11 +11,13 @@ package net.mamoe.mirai.internal.network

 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.protobuf.ProtoBuf
 import net.mamoe.mirai.internal.QQAndroidBot
 import net.mamoe.mirai.internal.contact.info.FriendInfoImpl
 import net.mamoe.mirai.internal.contact.info.MemberInfoImpl
 import net.mamoe.mirai.internal.network.protocol.data.jce.StTroopNum
 import net.mamoe.mirai.internal.utils.ScheduledJob
+import net.mamoe.mirai.internal.utils.groupCacheDir
 import net.mamoe.mirai.utils.createFileIfNotExists
 import net.mamoe.mirai.utils.info
 import net.mamoe.mirai.utils.runBIO
@ -28,6 +30,11 @@ internal val JsonForCache = Json {
    encodeDefaults = true
    ignoreUnknownKeys = true
    isLenient = true
+    prettyPrint = true
+}
+
+internal val ProtoBufForCache = ProtoBuf {
+    encodeDefaults = true
 }

@Serializable
@ -64,7 +71,7 @@ internal class GroupMemberListCaches(

    private val changedGroups: MutableCollection<Long> = ConcurrentLinkedQueue()
    private val groupListSaver by lazy {
-        ScheduledJob(bot.coroutineContext, bot.configuration.groupMemberListCache!!.saveIntervalMillis.milliseconds) {
+        ScheduledJob(bot.coroutineContext, bot.configuration.contactListCache.saveIntervalMillis.milliseconds) {
            runBIO { saveGroupCaches() }
        }
    }
@ -83,9 +90,7 @@ internal class GroupMemberListCaches(
        return ret
    }

-    private val cacheDir by lazy {
-        bot.configuration.groupMemberListCache!!.cacheDir.let { bot.configuration.cacheDirSupplier().resolve(it) }
-    }
+    private val cacheDir by lazy { bot.configuration.groupCacheDir() }

    private fun resolveCacheFile(groupCode: Long): File {
        cacheDir.mkdirs()
--- a/mirai-core/src/commonMain/kotlin/network/handler/BdhSessionSyncer.kt
+++ b/mirai-core/src/commonMain/kotlin/network/handler/BdhSessionSyncer.kt
@ -12,16 +12,23 @@ package net.mamoe.mirai.internal.network.handler
 import kotlinx.coroutines.CompletableDeferred
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
 import kotlinx.serialization.builtins.ListSerializer
-import kotlinx.serialization.builtins.PairSerializer
-import kotlinx.serialization.builtins.serializer
 import net.mamoe.mirai.internal.QQAndroidBot
 import net.mamoe.mirai.internal.network.BdhSession
 import net.mamoe.mirai.internal.network.JsonForCache
+import net.mamoe.mirai.internal.network.ProtoBufForCache
+import net.mamoe.mirai.internal.utils.actualCacheDir
 import java.io.File

-private val ServerListSerializer: KSerializer<List<Pair<String, Int>>> =
-    ListSerializer(PairSerializer(String.serializer(), Int.serializer()))
+@Serializable
+private data class ServerHostAndPort(
+    val host: String,
+    val port: Int,
+)
+
+private val ServerListSerializer: KSerializer<List<ServerHostAndPort>> =
+    ListSerializer(ServerHostAndPort.serializer())

@OptIn(ExperimentalCoroutinesApi::class)
 internal class BdhSessionSyncer(
@ -42,9 +49,9 @@ internal class BdhSessionSyncer(
    }

    private val sessionCacheFile: File
-        get() = bot.configuration.cacheDirSupplier().resolve("session.json")
+        get() = bot.configuration.actualCacheDir().resolve("session.bin")
    private val serverListCacheFile: File
-        get() = bot.configuration.cacheDirSupplier().resolve("serverlist.json")
+        get() = bot.configuration.actualCacheDir().resolve("servers.json")

    fun loadServerListFromCache() {
        val serverListCacheFile = this.serverListCacheFile
@ -53,7 +60,7 @@ internal class BdhSessionSyncer(
            kotlin.runCatching {
                val list = JsonForCache.decodeFromString(ServerListSerializer, serverListCacheFile.readText())
                bot.serverList.clear()
-                bot.serverList.addAll(list)
+                bot.serverList.addAll(list.map { it.host to it.port })
            }.onFailure {
                bot.network.logger.warning("Error in loading server list from cache", it)
            }
@ -68,10 +75,11 @@ internal class BdhSessionSyncer(
            bot.network.logger.verbose("Loading BdhSession from cache file")
            kotlin.runCatching {
                overrideSession(
-                    JsonForCache.decodeFromString(BdhSession.serializer(), sessionCacheFile.readText()),
+                    ProtoBufForCache.decodeFromByteArray(BdhSession.serializer(), sessionCacheFile.readBytes()),
                    doSave = false
                )
            }.onFailure {
+                kotlin.runCatching { sessionCacheFile.delete() }
                bot.network.logger.warning("Error in loading BdhSession from cache", it)
            }
        } else {
@ -88,7 +96,7 @@ internal class BdhSessionSyncer(
            serverListCacheFile.writeText(
                JsonForCache.encodeToString(
                    ServerListSerializer,
-                    bot.serverList
+                    bot.serverList.map { ServerHostAndPort(it.first, it.second) }
                )
            )
        }.onFailure {
@ -102,8 +110,8 @@ internal class BdhSessionSyncer(
        if (bdhSession.isCompleted) {
            bot.network.logger.verbose("Saving bdh session to cache")
            kotlin.runCatching {
-                sessionCacheFile.writeText(
-                    JsonForCache.encodeToString(
+                sessionCacheFile.writeBytes(
+                    ProtoBufForCache.encodeToByteArray(
                        BdhSession.serializer(),
                        bdhSession.getCompleted()
                    )
--- a/mirai-core/src/commonMain/kotlin/utils/BotConfigurationExt.kt
+++ b/mirai-core/src/commonMain/kotlin/utils/BotConfigurationExt.kt
@ -0,0 +1,22 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ *  此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ *  Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ *  https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+package net.mamoe.mirai.internal.utils
+
+import net.mamoe.mirai.utils.BotConfiguration
+import net.mamoe.mirai.utils.resolveCreateFile
+import net.mamoe.mirai.utils.resolveMkdir
+import java.io.File
+
+
+internal fun BotConfiguration.actualCacheDir(): File = workingDir.resolveMkdir(cacheDir)
+internal fun BotConfiguration.contactCacheDir(): File = actualCacheDir().resolveMkdir("contacts")
+internal fun BotConfiguration.friendCacheFile(): File = contactCacheDir().resolveCreateFile("friends.json")
+internal fun BotConfiguration.groupCacheDir(): File = contactCacheDir().resolveMkdir("groups")
+internal fun BotConfiguration.groupCacheFile(groupId: Long): File = groupCacheDir().resolveCreateFile("$groupId.json")