From 4c810ee3ee382e5c95ea155e33a3d4936c2f945f Mon Sep 17 00:00:00 2001 From: Karlatemp Date: Sat, 3 Jul 2021 22:05:12 +0800 Subject: [PATCH] ExternalResourceLeakObserver (#1383) * ExternalResourceLeakObserver * Avoid exceptions of user-defined run-when-close actions * Fix build * Release references * Move `ExternalResourceLeakObserver` to mirai-core-api * Make internal * Make `close()` thread-safely * typo * Don't track `ExternalResource` creation stack by default * Update mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt Co-authored-by: Him188 --- .../internal/utils/ExternalImageImpls.kt | 91 ++++++++++++-- .../utils/ExternalResourceLeakObserver.kt | 111 ++++++++++++++++++ 2 files changed, 189 insertions(+), 13 deletions(-) create mode 100644 mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt diff --git a/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalImageImpls.kt b/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalImageImpls.kt index 84aff7c6d..0c02a3085 100644 --- a/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalImageImpls.kt +++ b/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalImageImpls.kt @@ -9,8 +9,11 @@ package net.mamoe.mirai.internal.utils +import kotlinx.atomicfu.atomic import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Deferred import net.mamoe.mirai.utils.* +import java.io.Closeable import java.io.InputStream import java.io.RandomAccessFile @@ -27,7 +30,18 @@ internal class ExternalResourceImplByFileWithMd5( private val file: RandomAccessFile, override val md5: ByteArray, formatName: String? -) : ExternalResource { +) : ExternalResourceInternal { + internal class ResourceHolder( + @JvmField internal val file: RandomAccessFile, + ) : ExternalResourceHolder() { + override val closed: CompletableDeferred = CompletableDeferred() + override fun closeImpl() { + file.close() + } + } + + override val holder: ResourceHolder = ResourceHolder(file) + override val sha1: ByteArray by lazy { inputStream().sha1() } override val size: Long = file.length() override val formatName: String by lazy { @@ -39,22 +53,67 @@ internal class ExternalResourceImplByFileWithMd5( return file.inputStream() } - override val closed: CompletableDeferred = CompletableDeferred() + override val closed: CompletableDeferred get() = holder.closed + override fun close() = holder.close() + init { + registerToLeakObserver(this) + } +} + +internal abstract class ExternalResourceHolder : Closeable { + /** + * Mirror of [ExternalResource.closed] + */ + abstract val closed: Deferred + val isClosed: Boolean get() = _closed.value + val createStackTrace: Array? = if (isExternalResourceCreationStackEnabled) { + Thread.currentThread().stackTrace + } else null + + private val _closed = atomic(false) + protected abstract fun closeImpl() override fun close() { + if (!_closed.compareAndSet(false, true)) return try { - file.close() + closeImpl() } finally { - kotlin.runCatching { closed.complete(Unit) } + kotlin.runCatching { + val closed = this.closed + if (closed is CompletableDeferred) { + closed.complete(Unit) + } else { + closed.cancel() + } + } } } } +internal interface ExternalResourceInternal : ExternalResource { + val holder: ExternalResourceHolder +} + internal class ExternalResourceImplByFile( private val file: RandomAccessFile, formatName: String?, - private val closeOriginalFileOnClose: Boolean = true -) : ExternalResource { + closeOriginalFileOnClose: Boolean = true +) : ExternalResourceInternal { + internal class ResourceHolder( + @JvmField internal val closeOriginalFileOnClose: Boolean, + @JvmField internal val file: RandomAccessFile, + ) : ExternalResourceHolder() { + override val closed: CompletableDeferred = CompletableDeferred() + override fun closeImpl() { + if (closeOriginalFileOnClose) file.close() + } + } + + override val holder: ResourceHolder = ResourceHolder( + closeOriginalFileOnClose, + file, + ) + override val size: Long = file.length() override val md5: ByteArray by lazy { inputStream().md5() } override val sha1: ByteArray by lazy { inputStream().sha1() } @@ -67,13 +126,11 @@ internal class ExternalResourceImplByFile( return file.inputStream() } - override val closed: CompletableDeferred = CompletableDeferred() - override fun close() { - try { - if (closeOriginalFileOnClose) file.close() - } finally { - kotlin.runCatching { closed.complete(Unit) } - } + override val closed: CompletableDeferred get() = holder.closed + override fun close() = holder.close() + + init { + registerToLeakObserver(this) } } @@ -108,6 +165,14 @@ private fun RandomAccessFile.inputStream(): InputStream { }.buffered() } +private fun registerToLeakObserver(resource: ExternalResourceInternal) { + ExternalResourceLeakObserver.register(resource) +} + +internal const val isExternalResourceCreationStackEnabledName = "mirai.resource.creation.stack.enabled" +internal val isExternalResourceCreationStackEnabled by lazy { + systemProp(isExternalResourceCreationStackEnabledName, false) +} /* * ImgType: diff --git a/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt b/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt new file mode 100644 index 000000000..7b251bbd6 --- /dev/null +++ b/mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt @@ -0,0 +1,111 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package net.mamoe.mirai.internal.utils + +import net.mamoe.mirai.utils.ExternalResource +import net.mamoe.mirai.utils.MiraiLogger +import net.mamoe.mirai.utils.error +import net.mamoe.mirai.utils.warning +import java.lang.ref.ReferenceQueue +import java.lang.ref.WeakReference +import java.util.concurrent.ConcurrentLinkedDeque + +internal object ExternalResourceLeakObserver : Runnable { + private val queue = ReferenceQueue() + private val references = ConcurrentLinkedDeque() + private val logger by lazy { + MiraiLogger.create("ExternalResourceLeakObserver") + } + + internal class ERReference( + resourceInternal: ExternalResourceInternal + ) : WeakReference(resourceInternal, queue) { + @JvmField + internal val holder: ExternalResourceHolder = resourceInternal.holder + } + + class ExternalResourceCreateStackTrace : Throwable() { + override fun fillInStackTrace(): Throwable { + return this + } + } + + + @JvmStatic + fun register(resource: ExternalResource) { + if (resource !is ExternalResourceInternal) return + references.add(ERReference(resource)) + } + + init { + val thread = Thread(this, "Mirai ExternalResource Leak Observer Thread") + thread.isDaemon = true + thread.start() + } + + override fun run() { + while (true) { + + try { + loop@ + while (true) { + val reference = queue.poll() ?: break@loop + if (reference !is ERReference) { + logger.warning { "Unknown reference $reference (#${reference.javaClass}) was entered queue. Skipping" } + reference.clear() + continue@loop + } + val holder = reference.holder + reference.clear() + references.remove(reference) + if (holder.isClosed) { + continue@loop + } + val stackException = holder.createStackTrace?.let { stack -> + ExternalResourceCreateStackTrace().also { it.stackTrace = stack } + } + kotlin.runCatching { // Observer should avoid all possible errors + logger.error( + { + "A resource leak occurred, use ExternalResource.close to avoid it!! (holder=$holder)" + if (isExternalResourceCreationStackEnabled) { + "" + } else ". Add jvm option `-D$isExternalResourceCreationStackEnabledName=true` to show creation stack track" + }, + stackException + ) + } + try { + holder.close() + } catch (exceptionInClose: Throwable) { + kotlin.runCatching { // Observer should avoid all possible errors + logger.error( + { "Exception in closing a leaked resource (holder=$holder)" }, + exceptionInClose.also { + if (stackException != null) { + it.addSuppressed(stackException) + } + } + ) + } + } + } + } catch (throwable: Throwable) { + kotlin.runCatching { // Observer should avoid all possible errors + logger.error( + "Exception in queue loop", + throwable + ) + } + } + + Thread.sleep(60 * 1000L) + } + } +}