Cui cloud & GitHub cloud uploading

.github/workflows/cui.yml

@@ -0,0 +1,48 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CuiCloud Publish
+
+# Controls when the action will run. Triggers the workflow on push or pull request 
+# events but only for the master branch
+on:
+  release:
+    types:
+      - created
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Gradle clean
+        run: ./gradlew clean
+      - name: Gradle build
+        run: ./gradlew build # if test's failed, don't publish
+      - name: Gradle :mirai-console:cuiCloudUpload
+        run: ./gradlew :mirai-console:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
+      - name: Gradle :mirai-console-qqandroid:cuiCloudUpload
+        run: ./gradlew :mirai-console-graphical:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
+
+
+#      - name: Upload artifact
+#        uses: actions/upload-artifact@v1.0.0
+#        with:
+#          # Artifact name
+#          name: mirai-core
+#          # Directory containing files to upload
+#          path: "mirai-core/build/libs/mirai-core-*-all.jar"
+#      - name: Upload artifact
+#        uses: actions/upload-artifact@v1.0.0
+#        with:
+#          # Artifact name
+#          name: mirai-core-qqandroid-all
+#          # Directory containing files to upload
+#          path: "mirai-core-qqandroid/build/libs/mirai-core-qqandroid-*-all.jar"

.github/workflows/shadow.yml

@@ -0,0 +1,48 @@
+# This is a basic workflow to help you get started with Actions
+
+name: mirai-repo Publish
+
+# Controls when the action will run. Triggers the workflow on push or pull request 
+# events but only for the master branch
+on:
+  release:
+    types:
+      - created
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Gradle clean
+        run: ./gradlew clean
+      - name: Gradle build
+        run: ./gradlew build # if test's failed, don't publish
+      - name: Gradle :mirai-console:githubUpload
+        run: ./gradlew :mirai-console:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }}
+      - name: Gradle :mirai-console-qqandroid:githubUpload
+        run: ./gradlew :mirai-console-graphical:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }}
+
+
+#      - name: Upload artifact
+#        uses: actions/upload-artifact@v1.0.0
+#        with:
+#          # Artifact name
+#          name: mirai-core
+#          # Directory containing files to upload
+#          path: "mirai-core/build/libs/mirai-core-*-all.jar"
+#      - name: Upload artifact
+#        uses: actions/upload-artifact@v1.0.0
+#        with:
+#          # Artifact name
+#          name: mirai-core-qqandroid-all
+#          # Directory containing files to upload
+#          path: "mirai-core-qqandroid/build/libs/mirai-core-qqandroid-*-all.jar"

build.gradle.kts

@@ -1,3 +1,7 @@
+@file:Suppress("UnstableApiUsage")
+
+import kotlin.math.pow
+
 buildscript {
     repositories {
         maven(url = "https://dl.bintray.com/kotlin/kotlin-eap")
@@ -23,4 +27,77 @@ allprojects {
         jcenter()
         mavenCentral()
     }
+}
+
+subprojects {
+    afterEvaluate {
+        apply(plugin = "com.github.johnrengelman.shadow")
+        val kotlin =
+            (this as ExtensionAware).extensions.getByName("kotlin") as? org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension
+                ?: return@afterEvaluate
+
+        val githubUpload by tasks.creating {
+            group = "mirai"
+            dependsOn(tasks.getByName("shadowJar"))
+
+            doFirst {
+                timeout.set(java.time.Duration.ofHours(3))
+                findLatestFile()?.let { (_, file) ->
+                    val filename = file.name
+                    println("Uploading file $filename")
+                    runCatching {
+                        upload.GitHub.upload(
+                            file,
+                            "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename",
+                            project
+                        )
+                    }.exceptionOrNull()?.let {
+                        System.err.println("GitHub Upload failed")
+                        it.printStackTrace() // force show stacktrace
+                        throw it
+                    }
+                }
+            }
+        }
+
+        val cuiCloudUpload by tasks.creating {
+            group = "mirai"
+            dependsOn(tasks.getByName("shadowJar"))
+
+            doFirst {
+                timeout.set(java.time.Duration.ofHours(3))
+                findLatestFile()?.let { (_, file) ->
+                    val filename = file.name
+                    println("Uploading file $filename")
+                    runCatching {
+                        upload.CuiCloud.upload(
+                            file,
+                            project
+                        )
+                    }.exceptionOrNull()?.let {
+                        System.err.println("CuiCloud Upload failed")
+                        it.printStackTrace() // force show stacktrace
+                        throw it
+                    }
+                }
+            }
+
+        }
+    }
+}
+
+
+fun Project.findLatestFile(): Map.Entry<String, File>? {
+    return File(projectDir, "build/libs").walk()
+        .filter { it.isFile }
+        .onEach { println("all files=$it") }
+        .filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) }
+        .onEach { println("matched file: ${it.name}") }
+        .associateBy { it.nameWithoutExtension.substringAfterLast('-') }
+        .onEach { println("versions: $it") }
+        .maxBy {
+            it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int ->
+                acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0)
+            }
+        }
 }

buildSrc/build.gradle.kts

@@ -4,4 +4,24 @@ plugins {
 
 repositories {
     jcenter()
+}
+
+kotlin {
+    sourceSets.all {
+        languageSettings.useExperimentalAnnotation("kotlin.Experimental")
+        languageSettings.useExperimentalAnnotation("kotlin.RequiresOptIn")
+    }
+}
+
+dependencies {
+    fun kotlinx(id: String, version: String) = "org.jetbrains.kotlinx:kotlinx-$id:$version"
+    fun ktor(id: String, version: String) = "io.ktor:ktor-$id:$version"
+
+    api("org.jsoup:jsoup:1.12.1")
+
+    api("com.google.code.gson:gson:2.8.6")
+    api(kotlinx("coroutines-core", "1.3.3"))
+    api(ktor("client-core", "1.3.2"))
+    api(ktor("client-cio", "1.3.2"))
+    api(ktor("client-json", "1.3.2"))
 }

buildSrc/src/main/kotlin/upload/CuiCloud.kt

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2020 Mamoe Technologies and contributors.
+ *
+ * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
+ * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
+ *
+ * https://github.com/mamoe/mirai/blob/master/LICENSE
+ */
+
+package upload
+
+import io.ktor.client.request.forms.MultiPartFormDataContent
+import io.ktor.client.request.forms.formData
+import io.ktor.client.request.post
+import io.ktor.client.statement.HttpResponse
+import io.ktor.http.isSuccess
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.provideDelegate
+import java.io.File
+import java.util.*
+
+@Suppress("DEPRECATION")
+object CuiCloud {
+    private fun getUrl(project: Project): String {
+        kotlin.runCatching {
+            @Suppress("UNUSED_VARIABLE", "LocalVariableName")
+            val cui_cloud_url: String by project
+            return cui_cloud_url
+        }
+
+        System.getProperty("cui_cloud_url", null)?.let {
+            return it.trim()
+        }
+        File(File(System.getProperty("user.dir")).parent, "/cuiUrl.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+
+        File(File(System.getProperty("user.dir")), "/cuiUrl.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+        error("cannot find url for CuiCloud")
+    }
+
+    private fun getKey(project: Project): String {
+        kotlin.runCatching {
+            @Suppress("UNUSED_VARIABLE", "LocalVariableName")
+            val cui_cloud_key: String by project
+            return cui_cloud_key
+        }
+
+        System.getProperty("cui_cloud_key", null)?.let {
+            return it.trim()
+        }
+        File(File(System.getProperty("user.dir")).parent, "/cuiToken.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+
+        File(File(System.getProperty("user.dir")), "/cuiToken.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+
+        error("cannot find key for CuiCloud")
+    }
+
+    fun upload(file: File, project: Project) {
+        val cuiCloudUrl = getUrl(project)
+        val key = getKey(project)
+
+        runBlocking {
+            uploadToCuiCloud(
+                cuiCloudUrl,
+                key,
+                "/mirai/${project.name}/${file.nameWithoutExtension}.mp4",
+                file.readBytes()
+            )
+        }
+    }
+
+    @UseExperimental(ExperimentalStdlibApi::class)
+    private suspend fun uploadToCuiCloud(
+        cuiCloudUrl: String,
+        cuiToken: String,
+        filePath: String,
+        content: ByteArray
+    ) {
+        println("filePath=$filePath")
+        println("content=${content.size / 1024 / 1024} MB")
+
+        val response = withContext(Dispatchers.IO) {
+            Http.post<HttpResponse>(cuiCloudUrl) {
+                body = MultiPartFormDataContent(
+                    formData {
+                        append("base64", Base64.getEncoder().encodeToString(content))
+                        append("filePath", filePath)
+                        append("large", "true")
+                        append("key", cuiToken)
+                    }
+                )
+            }
+        }
+        println(response.status)
+
+        val buffer = ByteArray(4096)
+        val resp = buildList<Byte> {
+            while (true) {
+                val read = response.content.readAvailable(buffer, 0, buffer.size)
+                if (read == -1) {
+                    break
+                }
+                addAll(buffer.toList().take(read))
+            }
+        }
+        println(String(resp.toByteArray()))
+
+        if (!response.status.isSuccess()) {
+            error("Cui cloud response: ${response.status}")
+        }
+    }
+}
+
+inline fun <E> buildList(builderAction: MutableList<E>.() -> Unit): List<E> {
+    return ArrayList<E>().apply(builderAction)
+}

buildSrc/src/main/kotlin/upload/GitHub.kt

@@ -0,0 +1,178 @@
+@file:Suppress("EXPERIMENTAL_API_USAGE")
+
+package upload
+
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
+import io.ktor.client.HttpClient
+import io.ktor.client.engine.cio.CIO
+import io.ktor.client.features.HttpTimeout
+import io.ktor.client.request.put
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withContext
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.provideDelegate
+import org.jsoup.Connection
+import org.jsoup.Jsoup
+import java.io.File
+import java.util.*
+
+internal val Http = HttpClient(CIO) {
+    engine {
+        requestTimeout = 600_000
+    }
+    install(HttpTimeout) {
+        socketTimeoutMillis = 600_000
+        requestTimeoutMillis = 600_000
+        connectTimeoutMillis = 600_000
+    }
+}
+
+object GitHub {
+
+    private fun getGithubToken(project: Project): String {
+        kotlin.runCatching {
+            @Suppress("UNUSED_VARIABLE", "LocalVariableName")
+            val github_token: String by project
+            return github_token
+        }
+
+        System.getProperty("github_token", null)?.let {
+            return it.trim()
+        }
+
+        File(File(System.getProperty("user.dir")).parent, "/token.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+
+        File(File(System.getProperty("user.dir")), "/token.txt").let { local ->
+            if (local.exists()) {
+                return local.readText().trim()
+            }
+        }
+
+        error(
+            "Cannot find github token, " +
+                    "please specify by creating a file token.txt in project dir, " +
+                    "or by providing JVM parameter 'github_token'"
+        )
+    }
+
+    fun upload(file: File, url: String, project: Project) = runBlocking {
+        val token = getGithubToken(project)
+        println("token.length=${token.length}")
+        Http.put<String>("$url?access_token=$token") {
+            val sha = getGithubSha("mirai-repo", "shadow/${project.name}/${file.name}", "master", project)
+            println("sha=$sha")
+            val content = String(Base64.getEncoder().encode(file.readBytes()))
+            body = """
+                    {
+                      "message": "automatically upload on release",
+                      "content": "$content"
+                      ${if (sha == null) "" else """, "sha": "$sha" """}
+                    }
+                """.trimIndent()
+        }.let {
+            println("Upload response: $it")
+        }
+    }
+
+
+    private suspend fun getGithubSha(
+        repo: String,
+        filePath: String,
+        branch: String,
+        project: Project
+    ): String? {
+        fun String.asJson(): JsonObject {
+            return JsonParser.parseString(this).asJsonObject
+        }
+
+        /*
+        * 只能获取1M以内/branch为master的sha
+        * */
+        class TargetTooLargeException() : Exception("Target TOO Large")
+
+        suspend fun getShaSmart(repo: String, filePath: String, project: Project): String? {
+            return withContext(Dispatchers.IO) {
+                val response = Jsoup
+                    .connect(
+                        "https://api.github.com/repos/mamoe/$repo/contents/$filePath?access_token=" + getGithubToken(
+                            project
+                        )
+                    )
+                    .ignoreContentType(true)
+                    .ignoreHttpErrors(true)
+                    .method(Connection.Method.GET)
+                    .execute()
+                if (response.statusCode() == 404) {
+                    null
+                } else {
+                    val p = response.body().asJson()
+                    if (p.has("message") && p["message"].asString == "This API returns blobs up to 1 MB in size. The requested blob is too large to fetch via the API, but you can use the Git Data API to request blobs up to 100 MB in size.") {
+                        throw TargetTooLargeException()
+                    }
+                    p.get("sha").asString
+                }
+            }
+        }
+
+        suspend fun getShaStupid(
+            repo: String,
+            filePath: String,
+            branch: String,
+            project: Project
+        ): String? {
+            val resp = withContext(Dispatchers.IO) {
+                Jsoup
+                    .connect(
+                        "https://api.github.com/repos/mamoe/$repo/git/ref/heads/$branch?access_token=" + getGithubToken(
+                            project
+                        )
+                    )
+                    .ignoreContentType(true)
+                    .ignoreHttpErrors(true)
+                    .method(Connection.Method.GET)
+                    .execute()
+            }
+            if (resp.statusCode() == 404) {
+                println("Branch Not Found")
+                return null
+            }
+            val info = resp.body().asJson().get("object").asJsonObject.get("url").asString
+            var parentNode = withContext(Dispatchers.IO) {
+                Jsoup.connect(info + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
+                    .method(Connection.Method.GET)
+                    .execute().body().asJson().get("tree").asJsonObject.get("url").asString
+            }
+            filePath.split("/").forEach { subPath ->
+                withContext(Dispatchers.IO) {
+                    Jsoup.connect(parentNode + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
+                        .method(Connection.Method.GET).execute().body().asJson().get("tree").asJsonArray
+                }.forEach list@{
+                    with(it.asJsonObject) {
+                        if (this.get("path").asString == subPath) {
+                            parentNode = this.get("url").asString
+                            return@list
+                        }
+                    }
+                }
+            }
+            check(parentNode.contains("/blobs/"))
+            return parentNode.substringAfterLast("/")
+        }
+
+        return if (branch == "master") {
+            try {
+                getShaSmart(repo, filePath, project)
+            } catch (e: TargetTooLargeException) {
+                getShaStupid(repo, filePath, branch, project)
+            }
+        } else {
+            getShaStupid(repo, filePath, branch, project)
+        }
+    }
+}

mirai-console/src/main/java/net/mamoe/mirai/console/events/Events.java

@@ -13,7 +13,6 @@ import kotlinx.coroutines.GlobalScope;
 import net.mamoe.mirai.event.Event;
 import net.mamoe.mirai.event.Listener;
 import net.mamoe.mirai.event.ListeningStatus;
-import net.mamoe.mirai.event.internal.EventInternalJvmKt;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.function.Consumer;
@@ -35,7 +34,7 @@ public final class Events {
      */
     @NotNull
     public static <E extends Event> Listener<E> subscribe(@NotNull Class<E> eventClass, @NotNull Function<E, ListeningStatus> onEvent) {
-        return EventInternalJvmKt._subscribeEventForJaptOnly(eventClass, GlobalScope.INSTANCE, onEvent);
+        return EventsImplKt.subscribeEventForJaptOnly(eventClass, GlobalScope.INSTANCE, onEvent);
     }
 
     /**
@@ -49,7 +48,7 @@ public final class Events {
      */
     @NotNull
     public static <E extends Event> Listener<E> subscribeAlways(@NotNull Class<E> eventClass, @NotNull Consumer<E> onEvent) {
-        return EventInternalJvmKt._subscribeEventForJaptOnly(eventClass, GlobalScope.INSTANCE, onEvent);
+        return EventsImplKt.subscribeEventForJaptOnly(eventClass, GlobalScope.INSTANCE, onEvent);
     }
 
 

mirai-console/src/main/kotlin/net/mamoe/mirai/console/events/EventsImpl.kt

@@ -7,10 +7,26 @@
  * https://github.com/mamoe/mirai/blob/master/LICENSE
  */
 
+@file:Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+
 package net.mamoe.mirai.console.events;
 
+import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.runBlocking
 import net.mamoe.mirai.event.Event
+import net.mamoe.mirai.event.Listener
+import net.mamoe.mirai.event.ListeningStatus
 import net.mamoe.mirai.event.broadcast
+import net.mamoe.mirai.event.internal._subscribeEventForJaptOnly
+import java.util.function.Consumer
+import java.util.function.Function
 
 internal fun <E : Event> broadcast(e: E): E = runBlocking { e.broadcast() }
+
+internal fun <E : Event> Class<E>.subscribeEventForJaptOnly(
+    scope: CoroutineScope,
+    onEvent: Function<E, ListeningStatus>
+): Listener<E> = _subscribeEventForJaptOnly(scope, onEvent)
+
+internal fun <E : Event> Class<E>.subscribeEventForJaptOnly(scope: CoroutineScope, onEvent: Consumer<E>): Listener<E> =
+    _subscribeEventForJaptOnly(scope, onEvent)