ExternalResourceLeakObserver (#1383)

* ExternalResourceLeakObserver

* Avoid exceptions of user-defined run-when-close actions

* Fix build

* Release references

* Move `ExternalResourceLeakObserver` to mirai-core-api

* Make internal

* Make `close()` thread-safely

* typo

* Don't track `ExternalResource` creation stack by default

* Update mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt

Co-authored-by: Him188 <Him188@mamoe.net>

mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalImageImpls.kt

@@ -9,8 +9,11 @@
 
 package net.mamoe.mirai.internal.utils
 
+import kotlinx.atomicfu.atomic
 import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.Deferred
 import net.mamoe.mirai.utils.*
+import java.io.Closeable
 import java.io.InputStream
 import java.io.RandomAccessFile
 
@@ -27,7 +30,18 @@ internal class ExternalResourceImplByFileWithMd5(
     private val file: RandomAccessFile,
     override val md5: ByteArray,
     formatName: String?
-) : ExternalResource {
+) : ExternalResourceInternal {
+    internal class ResourceHolder(
+        @JvmField internal val file: RandomAccessFile,
+    ) : ExternalResourceHolder() {
+        override val closed: CompletableDeferred<Unit> = CompletableDeferred()
+        override fun closeImpl() {
+            file.close()
+        }
+    }
+
+    override val holder: ResourceHolder = ResourceHolder(file)
+
     override val sha1: ByteArray by lazy { inputStream().sha1() }
     override val size: Long = file.length()
     override val formatName: String by lazy {
@@ -39,22 +53,67 @@ internal class ExternalResourceImplByFileWithMd5(
         return file.inputStream()
     }
 
-    override val closed: CompletableDeferred<Unit> = CompletableDeferred()
+    override val closed: CompletableDeferred<Unit> get() = holder.closed
+    override fun close() = holder.close()
 
+    init {
+        registerToLeakObserver(this)
+    }
+}
+
+internal abstract class ExternalResourceHolder : Closeable {
+    /**
+     * Mirror of [ExternalResource.closed]
+     */
+    abstract val closed: Deferred<Unit>
+    val isClosed: Boolean get() = _closed.value
+    val createStackTrace: Array<StackTraceElement>? = if (isExternalResourceCreationStackEnabled) {
+        Thread.currentThread().stackTrace
+    } else null
+
+    private val _closed = atomic(false)
+    protected abstract fun closeImpl()
     override fun close() {
+        if (!_closed.compareAndSet(false, true)) return
         try {
-            file.close()
+            closeImpl()
         } finally {
-            kotlin.runCatching { closed.complete(Unit) }
+            kotlin.runCatching {
+                val closed = this.closed
+                if (closed is CompletableDeferred<Unit>) {
+                    closed.complete(Unit)
+                } else {
+                    closed.cancel()
+                }
+            }
         }
     }
 }
 
+internal interface ExternalResourceInternal : ExternalResource {
+    val holder: ExternalResourceHolder
+}
+
 internal class ExternalResourceImplByFile(
     private val file: RandomAccessFile,
     formatName: String?,
-    private val closeOriginalFileOnClose: Boolean = true
-) : ExternalResource {
+    closeOriginalFileOnClose: Boolean = true
+) : ExternalResourceInternal {
+    internal class ResourceHolder(
+        @JvmField internal val closeOriginalFileOnClose: Boolean,
+        @JvmField internal val file: RandomAccessFile,
+    ) : ExternalResourceHolder() {
+        override val closed: CompletableDeferred<Unit> = CompletableDeferred()
+        override fun closeImpl() {
+            if (closeOriginalFileOnClose) file.close()
+        }
+    }
+
+    override val holder: ResourceHolder = ResourceHolder(
+        closeOriginalFileOnClose,
+        file,
+    )
+
     override val size: Long = file.length()
     override val md5: ByteArray by lazy { inputStream().md5() }
     override val sha1: ByteArray by lazy { inputStream().sha1() }
@@ -67,13 +126,11 @@ internal class ExternalResourceImplByFile(
         return file.inputStream()
     }
 
-    override val closed: CompletableDeferred<Unit> = CompletableDeferred()
-    override fun close() {
-        try {
-            if (closeOriginalFileOnClose) file.close()
-        } finally {
-            kotlin.runCatching { closed.complete(Unit) }
-        }
+    override val closed: CompletableDeferred<Unit> get() = holder.closed
+    override fun close() = holder.close()
+
+    init {
+        registerToLeakObserver(this)
     }
 }
 
@@ -108,6 +165,14 @@ private fun RandomAccessFile.inputStream(): InputStream {
     }.buffered()
 }
 
+private fun registerToLeakObserver(resource: ExternalResourceInternal) {
+    ExternalResourceLeakObserver.register(resource)
+}
+
+internal const val isExternalResourceCreationStackEnabledName = "mirai.resource.creation.stack.enabled"
+internal val isExternalResourceCreationStackEnabled by lazy {
+    systemProp(isExternalResourceCreationStackEnabledName, false)
+}
 
 /*
  * ImgType:

mirai-core-api/src/commonMain/kotlin/internal/utils/ExternalResourceLeakObserver.kt

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2019-2021 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/dev/LICENSE
+ */
+
+package net.mamoe.mirai.internal.utils
+
+import net.mamoe.mirai.utils.ExternalResource
+import net.mamoe.mirai.utils.MiraiLogger
+import net.mamoe.mirai.utils.error
+import net.mamoe.mirai.utils.warning
+import java.lang.ref.ReferenceQueue
+import java.lang.ref.WeakReference
+import java.util.concurrent.ConcurrentLinkedDeque
+
+internal object ExternalResourceLeakObserver : Runnable {
+    private val queue = ReferenceQueue<Any>()
+    private val references = ConcurrentLinkedDeque<ERReference>()
+    private val logger by lazy {
+        MiraiLogger.create("ExternalResourceLeakObserver")
+    }
+
+    internal class ERReference(
+        resourceInternal: ExternalResourceInternal
+    ) : WeakReference<ExternalResource>(resourceInternal, queue) {
+        @JvmField
+        internal val holder: ExternalResourceHolder = resourceInternal.holder
+    }
+
+    class ExternalResourceCreateStackTrace : Throwable() {
+        override fun fillInStackTrace(): Throwable {
+            return this
+        }
+    }
+
+
+    @JvmStatic
+    fun register(resource: ExternalResource) {
+        if (resource !is ExternalResourceInternal) return
+        references.add(ERReference(resource))
+    }
+
+    init {
+        val thread = Thread(this, "Mirai ExternalResource Leak Observer Thread")
+        thread.isDaemon = true
+        thread.start()
+    }
+
+    override fun run() {
+        while (true) {
+
+            try {
+                loop@
+                while (true) {
+                    val reference = queue.poll() ?: break@loop
+                    if (reference !is ERReference) {
+                        logger.warning { "Unknown reference $reference (#${reference.javaClass}) was entered queue. Skipping" }
+                        reference.clear()
+                        continue@loop
+                    }
+                    val holder = reference.holder
+                    reference.clear()
+                    references.remove(reference)
+                    if (holder.isClosed) {
+                        continue@loop
+                    }
+                    val stackException = holder.createStackTrace?.let { stack ->
+                        ExternalResourceCreateStackTrace().also { it.stackTrace = stack }
+                    }
+                    kotlin.runCatching { // Observer should avoid all possible errors
+                        logger.error(
+                            {
+                                "A resource leak occurred, use ExternalResource.close to avoid it!! (holder=$holder)" + if (isExternalResourceCreationStackEnabled) {
+                                    ""
+                                } else ". Add jvm option `-D$isExternalResourceCreationStackEnabledName=true` to show creation stack track"
+                            },
+                            stackException
+                        )
+                    }
+                    try {
+                        holder.close()
+                    } catch (exceptionInClose: Throwable) {
+                        kotlin.runCatching { // Observer should avoid all possible errors
+                            logger.error(
+                                { "Exception in closing a leaked resource (holder=$holder)" },
+                                exceptionInClose.also {
+                                    if (stackException != null) {
+                                        it.addSuppressed(stackException)
+                                    }
+                                }
+                            )
+                        }
+                    }
+                }
+            } catch (throwable: Throwable) {
+                kotlin.runCatching { // Observer should avoid all possible errors
+                    logger.error(
+                        "Exception in queue loop",
+                        throwable
+                    )
+                }
+            }
+
+            Thread.sleep(60 * 1000L)
+        }
+    }
+}