From 05f6259cb360dce1ca592d810d71bdd2323e2bf6 Mon Sep 17 00:00:00 2001 From: Him188 Date: Tue, 31 Mar 2020 09:52:49 +0800 Subject: [PATCH] Cui cloud publishing & GitHub republishing (#177) * Add CuiCloud tasks * Add CuiCloud workflow * Fix shadowJar * Fix `reply` function prohibition in MessageSelectBuilder * Fix doc * Update CI * Update CI * Test CI * Fix file * Increase jvm memory limitation * Add comments * Remove unnecessary experimental api use * Increase upload task timeout * Fix sha * Increase timeout * Fix github * Trigger publishing on release --- .github/workflows/cui.yml | 48 +++++++++ .github/workflows/shadow.yml | 4 +- build.gradle.kts | 80 ++++++++++----- buildSrc/build.gradle.kts | 10 ++ buildSrc/settings.gradle.kts | 0 buildSrc/src/main/kotlin/upload/CuiCloud.kt | 82 +++++++++++++++ buildSrc/src/main/kotlin/upload/GitHub.kt | 106 +++++++++++++++++++- gradle.properties | 3 +- 8 files changed, 303 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/cui.yml create mode 100644 buildSrc/settings.gradle.kts create mode 100644 buildSrc/src/main/kotlin/upload/CuiCloud.kt diff --git a/.github/workflows/cui.yml b/.github/workflows/cui.yml new file mode 100644 index 000000000..6a76d541a --- /dev/null +++ b/.github/workflows/cui.yml @@ -0,0 +1,48 @@ +# This is a basic workflow to help you get started with Actions + +name: CuiCloud Publish + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + release: + types: + - created + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Gradle clean + run: ./gradlew clean + - name: Gradle build + run: ./gradlew build # if test's failed, don't publish + - name: Gradle :mirai-core:cuiCloudUpload + run: ./gradlew :mirai-core:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} + - name: Gradle :mirai-core-qqandroid:cuiCloudUpload + run: ./gradlew :mirai-core-qqandroid:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} + + +# - name: Upload artifact +# uses: actions/upload-artifact@v1.0.0 +# with: +# # Artifact name +# name: mirai-core +# # Directory containing files to upload +# path: "mirai-core/build/libs/mirai-core-*-all.jar" +# - name: Upload artifact +# uses: actions/upload-artifact@v1.0.0 +# with: +# # Artifact name +# name: mirai-core-qqandroid-all +# # Directory containing files to upload +# path: "mirai-core-qqandroid/build/libs/mirai-core-qqandroid-*-all.jar" diff --git a/.github/workflows/shadow.yml b/.github/workflows/shadow.yml index 88fe6e1fa..f366bae15 100644 --- a/.github/workflows/shadow.yml +++ b/.github/workflows/shadow.yml @@ -1,6 +1,6 @@ # This is a basic workflow to help you get started with Actions -name: Shadow Publish +name: mirai-repo Publish # Controls when the action will run. Triggers the workflow on push or pull request # events but only for the master branch @@ -25,7 +25,7 @@ jobs: - name: Gradle clean run: ./gradlew clean - name: Gradle build - run: ./gradlew build + run: ./gradlew build # if test's failed, don't publish - name: Gradle :mirai-core:githubUpload run: ./gradlew :mirai-core:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }} - name: Gradle :mirai-core-qqandroid:githubUpload diff --git a/build.gradle.kts b/build.gradle.kts index 2e9ee0371..399728a73 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -84,36 +84,64 @@ subprojects { dependsOn(shadowJvmJar) doFirst { - timeout.set(Duration.ofMinutes(10)) - File(projectDir, "build/libs").walk() - .filter { it.isFile } - .onEach { println("all files=$it") } - .filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) } - .onEach { println("matched file: ${it.name}") } - .associateBy { it.nameWithoutExtension.substringAfterLast('-') } - .onEach { println("versions: $it") } - .maxBy { - it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int -> - acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0) - } - }?.let { (_, file) -> - val filename = file.name - println("Uploading file $filename") - runCatching { - upload.GitHub.upload( - file, - "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename", - project - ) - }.exceptionOrNull()?.let { - System.err.println("Upload failed") - it.printStackTrace() // force show stacktrace - throw it - } + timeout.set(Duration.ofHours(3)) + findLatestFile()?.let { (_, file) -> + val filename = file.name + println("Uploading file $filename") + runCatching { + upload.GitHub.upload( + file, + "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename", + project + ) + }.exceptionOrNull()?.let { + System.err.println("GitHub Upload failed") + it.printStackTrace() // force show stacktrace + throw it } + } + } + } + + val cuiCloudUpload by tasks.creating { + group = "mirai" + dependsOn(shadowJvmJar) + + doFirst { + timeout.set(Duration.ofHours(3)) + findLatestFile()?.let { (_, file) -> + val filename = file.name + println("Uploading file $filename") + runCatching { + upload.CuiCloud.upload( + file, + project + ) + }.exceptionOrNull()?.let { + System.err.println("CuiCloud Upload failed") + it.printStackTrace() // force show stacktrace + throw it + } + } } } } +} + + +fun Project.findLatestFile(): Map.Entry? { + return File(projectDir, "build/libs").walk() + .filter { it.isFile } + .onEach { println("all files=$it") } + .filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) } + .onEach { println("matched file: ${it.name}") } + .associateBy { it.nameWithoutExtension.substringAfterLast('-') } + .onEach { println("versions: $it") } + .maxBy { + it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int -> + acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0) + } + } } \ No newline at end of file diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index d4e362d00..a0b4b2f9b 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -6,10 +6,20 @@ repositories { jcenter() } +kotlin { + sourceSets.all { + languageSettings.useExperimentalAnnotation("kotlin.Experimental") + languageSettings.useExperimentalAnnotation("kotlin.RequiresOptIn") + } +} + dependencies { fun kotlinx(id: String, version: String) = "org.jetbrains.kotlinx:kotlinx-$id:$version" fun ktor(id: String, version: String) = "io.ktor:ktor-$id:$version" + api("org.jsoup:jsoup:1.12.1") + + api("com.google.code.gson:gson:2.8.6") api(kotlinx("coroutines-core", "1.3.3")) api(ktor("client-core", "1.3.2")) api(ktor("client-cio", "1.3.2")) diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts new file mode 100644 index 000000000..e69de29bb diff --git a/buildSrc/src/main/kotlin/upload/CuiCloud.kt b/buildSrc/src/main/kotlin/upload/CuiCloud.kt new file mode 100644 index 000000000..d229d74eb --- /dev/null +++ b/buildSrc/src/main/kotlin/upload/CuiCloud.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/master/LICENSE + */ + +package upload + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.gradle.api.Project +import org.gradle.kotlin.dsl.provideDelegate +import org.jsoup.Connection +import org.jsoup.Jsoup +import java.io.File +import java.util.* + +object CuiCloud { + private fun getUrl(project: Project): String { + kotlin.runCatching { + @Suppress("UNUSED_VARIABLE", "LocalVariableName") + val cui_cloud_url: String by project + return cui_cloud_url + } + + System.getProperty("cui_cloud_url", null)?.let { + return it.trim() + } + error("cannot find url for CuiCloud") + } + + private fun getKey(project: Project): String { + kotlin.runCatching { + @Suppress("UNUSED_VARIABLE", "LocalVariableName") + val cui_cloud_key: String by project + return cui_cloud_key + } + + System.getProperty("cui_cloud_key", null)?.let { + return it.trim() + } + error("cannot find key for CuiCloud") + } + + fun upload(file: File, project: Project) { + val cuiCloudUrl = getUrl(project) + val key = getKey(project) + + runBlocking { + uploadToCuiCloud( + cuiCloudUrl, + key, + "/mirai/${project.name}/${file.nameWithoutExtension}.mp4", + file.readBytes() + ) + } + } + + private suspend fun uploadToCuiCloud( + cuiCloudUrl: String, + cuiToken: String, + filePath: String, + content: ByteArray + ) { + val response = withContext(Dispatchers.IO) { + Jsoup.connect(cuiCloudUrl).method(Connection.Method.POST) + .data("base64", Base64.getEncoder().encodeToString(content)) + .data("filePath", filePath) + .data("key", cuiToken) + .timeout(Int.MAX_VALUE) + .execute() + } + if (response.statusCode() != 200) { + println(response.body()) + error("Cui Cloud Does Not Return 200") + } + } +} \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/upload/GitHub.kt b/buildSrc/src/main/kotlin/upload/GitHub.kt index 46d8e2374..d273f22f7 100644 --- a/buildSrc/src/main/kotlin/upload/GitHub.kt +++ b/buildSrc/src/main/kotlin/upload/GitHub.kt @@ -2,13 +2,19 @@ package upload +import com.google.gson.JsonObject +import com.google.gson.JsonParser import io.ktor.client.HttpClient import io.ktor.client.engine.cio.CIO import io.ktor.client.features.HttpTimeout import io.ktor.client.request.put +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.gradle.api.Project import org.gradle.kotlin.dsl.provideDelegate +import org.jsoup.Connection +import org.jsoup.Jsoup import java.io.File import java.util.* @@ -57,16 +63,114 @@ object GitHub { connectTimeoutMillis = 600_000 } }.put("$url?access_token=$token") { - //header("token", token) + val sha = getGithubSha("mirai-repo", "shadow/${project.name}/${file.name}", "master", project) + println("sha=$sha") val content = String(Base64.getEncoder().encode(file.readBytes())) body = """ { "message": "automatically upload on release", "content": "$content" + ${if (sha == null) "" else """, "sha": "$sha" """} } """.trimIndent() }.let { println("Upload response: $it") } } + + + private suspend fun getGithubSha( + repo: String, + filePath: String, + branch: String, + project: Project + ): String? { + fun String.asJson(): JsonObject { + return JsonParser.parseString(this).asJsonObject + } + + /* + * 只能获取1M以内/branch为master的sha + * */ + class TargetTooLargeException() : Exception("Target TOO Large") + + suspend fun getShaSmart(repo: String, filePath: String, project: Project): String? { + return withContext(Dispatchers.IO) { + val response = Jsoup + .connect( + "https://api.github.com/repos/mamoe/$repo/contents/$filePath?access_token=" + getGithubToken( + project + ) + ) + .ignoreContentType(true) + .ignoreHttpErrors(true) + .method(Connection.Method.GET) + .execute() + if (response.statusCode() == 404) { + null + } else { + val p = response.body().asJson() + if (p.has("message") && p["message"].asString == "This API returns blobs up to 1 MB in size. The requested blob is too large to fetch via the API, but you can use the Git Data API to request blobs up to 100 MB in size.") { + throw TargetTooLargeException() + } + p.get("sha").asString + } + } + } + + suspend fun getShaStupid( + repo: String, + filePath: String, + branch: String, + project: Project + ): String? { + val resp = withContext(Dispatchers.IO) { + Jsoup + .connect( + "https://api.github.com/repos/mamoe/$repo/git/ref/heads/$branch?access_token=" + getGithubToken( + project + ) + ) + .ignoreContentType(true) + .ignoreHttpErrors(true) + .method(Connection.Method.GET) + .execute() + } + if (resp.statusCode() == 404) { + println("Branch Not Found") + return null + } + val info = resp.body().asJson().get("object").asJsonObject.get("url").asString + var parentNode = withContext(Dispatchers.IO) { + Jsoup.connect(info + "?access_token=" + getGithubToken(project)).ignoreContentType(true) + .method(Connection.Method.GET) + .execute().body().asJson().get("tree").asJsonObject.get("url").asString + } + filePath.split("/").forEach { subPath -> + withContext(Dispatchers.IO) { + Jsoup.connect(parentNode + "?access_token=" + getGithubToken(project)).ignoreContentType(true) + .method(Connection.Method.GET).execute().body().asJson().get("tree").asJsonArray + }.forEach list@{ + with(it.asJsonObject) { + if (this.get("path").asString == subPath) { + parentNode = this.get("url").asString + return@list + } + } + } + } + check(parentNode.contains("/blobs/")) + return parentNode.substringAfterLast("/") + } + + return if (branch == "master") { + try { + getShaSmart(repo, filePath, project) + } catch (e: TargetTooLargeException) { + getShaStupid(repo, filePath, branch, project) + } + } else { + getShaStupid(repo, filePath, branch, project) + } + } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index f6a30e62f..330f03d63 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,4 +2,5 @@ kotlin.code.style=official # config kotlin.incremental.multiplatform=true -kotlin.parallel.tasks.in.project=true \ No newline at end of file +kotlin.parallel.tasks.in.project=true +org.gradle.jvmargs=-Xmx4096m -XX:MaxPermSize=512m -Dfile.encoding=UTF-8 \ No newline at end of file