From 531ef65f5ef8dad8e5e6bc58bf67b55a5561f6fb Mon Sep 17 00:00:00 2001 From: Karlatemp Date: Mon, 28 Nov 2022 00:05:51 +0800 Subject: [PATCH] [publish] Rewrite publishing --- .github/workflows/release.yml | 77 +++++-- ci-release-helper/.gitignore | 2 + ci-release-helper/build.gradle.kts | 47 +++++ ci-release-helper/src/CiHelper.kt | 311 +++++++++++++++++++++++++++++ ci-release-helper/src/package.kt | 11 + 5 files changed, 428 insertions(+), 20 deletions(-) create mode 100644 ci-release-helper/.gitignore create mode 100644 ci-release-helper/src/CiHelper.kt create mode 100644 ci-release-helper/src/package.kt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 746089236..189f0e2f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,10 +11,37 @@ on: jobs: + initialize-sonatype-stage: + name: "Initialize sonatype staging repository" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - run: chmod -R 777 * + + - name: Create publishing staging repository + run: ./gradlew runcihelper --args create-stage-repo --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" "-Pcihelper.cert.profileid=${{ secrets.SONATYPE_PROFILEID }}" + + - name: Cache staging repository id + uses: actions/upload-artifact@v3 + with: + name: publish-stage-id + path: ci-release-helper/repoid publish-others: name: "Others (${{ matrix.os }})" runs-on: ${{ matrix.os }} + needs: [ initialize-sonatype-stage ] strategy: fail-fast: false matrix: @@ -43,14 +70,9 @@ jobs: mkdir build-gpg-sign echo "$GPG_PRIVATE" > build-gpg-sign/keys.gpg echo "$GPG_PUBLIC_" > build-gpg-sign/keys.gpg.pub - mkdir build-secret-keys - echo "$SONATYPE_USER" > build-secret-keys/sonatype.key - echo "$SONATYPE_KEY" >> build-secret-keys/sonatype.key env: GPG_PRIVATE: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PUBLIC_: ${{ secrets.GPG_PUBLIC_KEY }} - SONATYPE_USER: ${{ secrets.SONATYPE_USER }} - SONATYPE_KEY: ${{ secrets.SONATYPE_KEY }} - name: Setup Gradle uses: gradle/gradle-build-action@v2 @@ -81,9 +103,6 @@ jobs: - name: Clean and download dependencies run: ./gradlew clean ${{ env.gradleArgs }} - - name: Check keys - run: ./gradlew ensureMavenCentralAvailable ${{ env.gradleArgs }} - - name: "Assemble" run: ./gradlew assemble ${{ env.gradleArgs }} @@ -98,15 +117,28 @@ jobs: name: Ensure KDoc valid run: ./gradlew dokkaHtmlMultiModule ${{ env.gradleArgs }} + - name: Initialize Publishing Caching Repository + run: ./gradlew runcihelper --args sync-maven-metadata ${{ env.gradleArgs }} + - name: Publish if: ${{ env.isMac == 'true' }} - run: ./gradlew publish ${{ env.gradleArgs }} + run: ./gradlew publishAllPublicationsToMiraiStageRepoRepository ${{ env.gradleArgs }} + + - name: Restore staging repository id + uses: actions/download-artifact@v3 + with: + name: publish-stage-id + path: ci-release-helper/repoid + + - name: Publish to maven central + run: ./gradlew runcihelper --args publish-to-maven-central --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" - name: Publish Gradle plugin run: ./gradlew :mirai-console-gradle:publishPlugins ${{ env.gradleArgs }} -Dgradle.publish.key=${{ secrets.GRADLE_PUBLISH_KEY }} -Pgradle.publish.key=${{ secrets.GRADLE_PUBLISH_KEY }} -Dgradle.publish.secret=${{ secrets.GRADLE_PUBLISH_SECRET }} -Pgradle.publish.secret=${{ secrets.GRADLE_PUBLISH_SECRET }} + continue-on-error: true publish-core-native: name: "Native (${{ matrix.os }})" @@ -154,14 +186,9 @@ jobs: mkdir build-gpg-sign echo "$GPG_PRIVATE" > build-gpg-sign/keys.gpg echo "$GPG_PUBLIC_" > build-gpg-sign/keys.gpg.pub - mkdir build-secret-keys - echo "$SONATYPE_USER" > build-secret-keys/sonatype.key - echo "$SONATYPE_KEY" >> build-secret-keys/sonatype.key env: GPG_PRIVATE: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PUBLIC_: ${{ secrets.GPG_PUBLIC_KEY }} - SONATYPE_USER: ${{ secrets.SONATYPE_USER }} - SONATYPE_KEY: ${{ secrets.SONATYPE_KEY }} - name: Setup Gradle uses: gradle/gradle-build-action@v2 @@ -228,9 +255,6 @@ jobs: - name: Clean and download dependencies run: ./gradlew clean ${{ env.gradleArgs }} - - name: Check keys - run: ./gradlew ensureMavenCentralAvailable ${{ env.gradleArgs }} - - name: "Test mirai-core-utils for ${{ matrix.os }}" run: ./gradlew :mirai-core-utils:${{ matrix.targetName }}Test ${{ env.gradleArgs }} @@ -240,21 +264,34 @@ jobs: - name: "Test mirai-core for ${{ matrix.os }}" run: ./gradlew :mirai-core:${{ matrix.targetName }}Test ${{ env.gradleArgs }} + - name: Initialize Publishing Caching Repository + run: ./gradlew runcihelper --args sync-maven-metadata ${{ env.gradleArgs }} + # # Parallel compilation will exhaust machine memory causing OOM # - name: Assemble # run: ./gradlew assemble ${{ env.gradleArgs }} "-Porg.gradle.parallel=${{ matrix.parallelCompilation }}" - name: Publish MingwX64 if: ${{ env.isWindows == 'true' }} - run: ./gradlew publishMingwX64PublicationToMavenCentralRepository ${{ env.gradleArgs }} + run: ./gradlew publishMingwX64PublicationToMiraiStageRepoRepository ${{ env.gradleArgs }} - name: Publish LinuxX64 if: ${{ env.isUbuntu == 'true' }} - run: ./gradlew publishLinuxX64PublicationToMavenCentralRepository ${{ env.gradleArgs }} + run: ./gradlew publishLinuxX64PublicationToMiraiStageRepoRepository ${{ env.gradleArgs }} - name: Publish macOSX64 if: ${{ env.isMac == 'true' }} - run: ./gradlew publishMacosX64PublicationToMavenCentralRepository ${{ env.gradleArgs }} + run: ./gradlew publishMacosX64PublicationToMiraiStageRepoRepository ${{ env.gradleArgs }} + + - name: Restore staging repository id + uses: actions/download-artifact@v3 + with: + name: publish-stage-id + path: ci-release-helper/repoid + + - name: Publish to maven central + run: ./gradlew runcihelper --args publish-to-maven-central --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" + # # close-repository: # runs-on: macos-12 diff --git a/ci-release-helper/.gitignore b/ci-release-helper/.gitignore new file mode 100644 index 000000000..31c67be59 --- /dev/null +++ b/ci-release-helper/.gitignore @@ -0,0 +1,2 @@ +/stage-repo +/stage-* diff --git a/ci-release-helper/build.gradle.kts b/ci-release-helper/build.gradle.kts index 2893233b9..3cf7c1088 100644 --- a/ci-release-helper/build.gradle.kts +++ b/ci-release-helper/build.gradle.kts @@ -7,10 +7,53 @@ * https://github.com/mamoe/mirai/blob/dev/LICENSE */ import keys.SecretKeys +import kotlinx.validation.sourceSets import java.io.ByteArrayOutputStream plugins { id("io.codearte.nexus-staging") version "0.22.0" + kotlin("jvm") +} + +tasks.register("runcihelper") { + this.classpath = sourceSets["main"].runtimeClasspath + this.mainClass.set("cihelper.CiHelperKt") + this.workingDir = rootProject.projectDir + + fun Project.findPublishingExt(): PublishingExtension? { + val exts = (this@findPublishingExt as ExtensionAware).extensions + return exts.findByName("publishing") as PublishingExtension? + } + + + doFirst { + @Suppress("USELESS_CAST") + environment("PROJ_VERSION", (project.version as Any?).toString()) + rootProject.allprojects.asSequence() + .mapNotNull { it.findPublishingExt() } + .flatMap { it.publications.asSequence() } + .mapNotNull { it as? MavenPublication } + .map { it.artifactId } + .joinToString("|") + .let { environment("PROJ_ARTIFACTS", it) } + + rootProject.allprojects.asSequence() + .mapNotNull { it.findPublishingExt() } + .flatMap { it.repositories.asSequence() } + .mapNotNull { it as? MavenArtifactRepository } + .filter { it.name == "MiraiStageRepo" } + .first().url + .let { environment("PROJ_MiraiStageRepo", it.toString()) } + + val additionProperties = rootProject.properties.asSequence() + .filter { (k, _) -> k.startsWith("cihelper.") } + .map { (k, v) -> "-D$k=$v" } + .toList() + if (additionProperties.isNotEmpty()) { + val currentJvmArgs = jvmArgs ?: emptyList() + jvmArgs = currentJvmArgs + additionProperties + } + } } description = "Mirai CI Methods for Releasing" @@ -22,6 +65,10 @@ nexusStaging { password = keys.password } +dependencies { + implementation(`kotlinx-serialization-json`) +} + tasks.register("updateSnapshotVersion") { group = "mirai" diff --git a/ci-release-helper/src/CiHelper.kt b/ci-release-helper/src/CiHelper.kt new file mode 100644 index 000000000..402822566 --- /dev/null +++ b/ci-release-helper/src/CiHelper.kt @@ -0,0 +1,311 @@ +/* + * Copyright 2019-2022 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ +@file:JvmName("CiHelperKt") + +package cihelper + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.io.File +import java.io.OutputStream +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.charset.Charset +import java.nio.file.FileSystems +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.attribute.PosixFilePermissions +import java.security.MessageDigest +import java.util.* +import java.util.stream.Collectors +import kotlin.io.path.* + +private val hexTemplate: CharArray = "0123456789abcdef".toCharArray() +private const val useragent = "Gradle/7.3.1 (Windows 10;10.0;amd64) (Azul Systems, Inc.;18.0.2.1;18.0.2.1+1)" + +fun ByteArray.hexToString(): String { + val sb = StringBuilder(this.size * 2) + forEach { sbyte -> + sb.append(hexTemplate[sbyte.toInt().shr(4).and(0xF)]) + sb.append(hexTemplate[sbyte.toInt().and(0xF)]) + } + return sb.toString() +} + +private fun getAuth(): String { + + val cert_username = + System.getenv("CERT_USERNAME") ?: System.getProperty("cihelper.cert.username") ?: error("CERT_USERNAME") + val cert_password = + System.getenv("CERT_PASSWORD") ?: System.getProperty("cihelper.cert.password") ?: error("CERT_PASSWORD") + + return "Basic " + Base64.getEncoder().encodeToString( + ("$cert_username:$cert_password").toByteArray() + ) +} + +@Suppress("Since15") +fun main(args: Array) { + val projVer = System.getenv("PROJ_VERSION") ?: error("Please use `./gradlew runcihelper --args XXXX`") + val projArtifacts = System.getenv("PROJ_ARTIFACTS")!!.split("|") + val repoLoc = System.getenv("PROJ_MiraiStageRepo")!!.let { Paths.get(URI.create(it)) } + + if (args.isEmpty()) error("no action") + + val relatedRepoLoc = repoLoc.resolve("net/mamoe") + + val httpc = HttpClient.newBuilder().build() + + when (args[0]) { + + "sync-maven-metadata" -> { + // https://repo1.maven.org/maven2/net/mamoe/mirai-core-all/maven-metadata.xml + + projArtifacts.forEach { projArtifact -> + val savedLoc = relatedRepoLoc.resolve(projArtifact) + .createDirectories() + .resolve("maven-metadata.xml") + + println("[metadata.xml] Syncing $projArtifact") + + val verPath = relatedRepoLoc.resolve(projArtifact).resolve(projVer) + + val isNotEmpty = if (verPath.exists()) { + Files.newDirectoryStream(verPath).use { it.iterator().hasNext() } + } else false + + if (isNotEmpty) { + println("[metadata.xml] Skipped $projArtifact because it was published to stage.") + return@forEach + } + + + val rsp = httpc.send( + HttpRequest.newBuilder( + URI.create("https://repo1.maven.org/maven2/net/mamoe/$projArtifact/maven-metadata.xml") + ).GET().build(), + HttpResponse.BodyHandlers.ofFile(savedLoc) + ) + if (rsp.statusCode() != 200) { + if (rsp.statusCode() == 404) { + savedLoc.deleteIfExists() + return@forEach + } + error("$rsp -> " + savedLoc.takeIf { it.isRegularFile() }?.readText()) + } + } + } + + "create-stage-repo" -> { + val rsp = httpc.send( + HttpRequest.newBuilder( + URI.create( + "https://oss.sonatype.org/service/local/staging/profiles/${ + System.getProperty( + "cihelper.cert.profileid" + ) + }/start" + ) + ) + .header("User-Agent", useragent) + .header("Authorization", getAuth()) + .header("Content-Type", "application/json;charset=utf-8") + .POST(HttpRequest.BodyPublishers.ofString("{\"data\":{\"description\": \"mamoe/mirai release $projVer\"}}")) + .build(), + HttpResponse.BodyHandlers.ofString() + ) + if (rsp.statusCode() != 201) { + error(rsp.toString()) + } + val rspx = Json.decodeFromString(JsonObject.serializer(), rsp.body()) + val stagedRepositoryId = rspx["data"]!!.jsonObject["stagedRepositoryId"]!!.jsonPrimitive.content + + File("ci-release-helper").also { it.mkdirs() } + .resolve("repoid").writeText(stagedRepositoryId) + } + + "publish-to-maven-central" -> { + // https://oss.sonatype.org/service/local/staging/deploy/maven2 + relatedRepoLoc.listDirectoryEntries().forEach { subdir -> + val verpath = subdir.resolve(projVer) + val doDelete = if (!verpath.isDirectory()) { + true + } else { + verpath.listDirectoryEntries().isEmpty() + } + if (doDelete) { + subdir.toFile().deleteRecursively() + } + } + val pendingFiles = Files.walk(relatedRepoLoc) + .filter { it.isRegularFile() } + .filter { !it.name.endsWith(".md5") && !it.name.endsWith(".sha1") } + .filter { !it.name.endsWith(".asc") } + .use { stream -> stream.collect(Collectors.toList()) } + + run `sign artifacts`@{ + // build-gpg-sign/keys.gpg + // build-gpg-sign/keys.gpg.pub + val bgs = Paths.get("build-gpg-sign").toAbsolutePath() + if (!bgs.isDirectory()) return@`sign artifacts` + val gpgHomeDir = bgs.resolve("homedir") + val bgsFile = bgs.toFile() + + fun execGpg(vararg cmd: String) { + println("::group::${cmd.joinToString(" ")}") + try { + val exitcode = ProcessBuilder("gpg", "--homedir", "homedir", "--batch", "--no-tty", *cmd) + .directory(bgsFile) + .inheritIO() + .start() + .waitFor() + if (exitcode != 0) { + error("Exit code $exitcode != 0") + } + } finally { + println("::endgroup::") + } + } + + + if (!gpgHomeDir.resolve("pubring.kbx").exists()) { + + val keys = arrayOf("keys.gpg", "keys.gpg.pub") + if (!keys.all { bgs.resolve(it).isRegularFile() }) return@`sign artifacts` + + val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix") + val dirPermissions = PosixFilePermissions.asFileAttribute( + EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE + ) + ) + + Files.createDirectories( + gpgHomeDir, + *if (isPosix) arrayOf(dirPermissions) else arrayOf(), + ) + + keys.forEach { execGpg("--import", it) } + } + + + println("::group::Signing artifacts") + pendingFiles.toList().asSequence().filterNot { it.name == "maven-metadata.xml" } + .forEach { pendingFile -> + val pt = pendingFile.absolutePathString() + val ascFile = pendingFile.resolveSibling(pendingFile.name + ".asc") + ascFile.deleteIfExists() + execGpg("-a", "--detach-sig", "--sign", pt) + + pendingFiles.add(ascFile) + } + println("::endgroup::") + } + + run `calc msg digest`@{ + pendingFiles.toList().forEach { pendingFile -> + val sha1MD = MessageDigest.getInstance("SHA-1") + val md5MD = MessageDigest.getInstance("MD5") + + pendingFile.inputStream().use { content -> + content.copyTo(object : OutputStream() { + override fun write(b: Int) { + sha1MD.update(b.toByte()) + md5MD.update(b.toByte()) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + sha1MD.update(b, off, len) + md5MD.update(b, off, len) + } + }) + } + + val sha1 = sha1MD.digest().hexToString() + val mg5 = md5MD.digest().hexToString() + + val pfname = pendingFile.name + val sha1File = pendingFile.resolveSibling("$pfname.sha1") + val md5File = pendingFile.resolveSibling("$pfname.md5") + + sha1File.writeText(sha1) + md5File.writeText(mg5) + + pendingFiles.add(sha1File) + pendingFiles.add(md5File) + } + } + + pendingFiles.sort() + + println("::group::Publishing to Maven Central") + + val authorization = getAuth() + val errors = mutableListOf() + + fun resolveSonatypeRepoLoc(): String { + val repoIdPath = Paths.get("ci-release-helper/repoid") + + var repoId = "" + if (repoIdPath.isRegularFile()) { + repoId = repoIdPath.readText().trim() + } else if (repoIdPath.isDirectory()) { + val files = repoIdPath.listDirectoryEntries().filter { it.isRegularFile() } + if (files.size == 1) { + repoId = files.first().readText().trim() + } + } + if (repoId.isNotBlank()) { + return "https://oss.sonatype.org/service/local/staging/deployByRepositoryId/$repoId/" + } + + return "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + } + + val repoServerLocation = resolveSonatypeRepoLoc() + + pendingFiles.forEach { pending -> + val netpath = repoLoc.relativize(pending) + + val uri = repoServerLocation + (netpath.toString().replace("\\", "/")) + + println("Processing $uri") + + val rsp = httpc.send( + HttpRequest.newBuilder( + URI.create(uri) + ).PUT(HttpRequest.BodyPublishers.ofFile(pending)) + .header("Authorization", authorization) + .header("User-Agent", useragent) + .build(), + HttpResponse.BodyHandlers.ofByteArray() + ) + if (rsp.statusCode() / 100 != 2) { + val errmsg = "$rsp -> " + String(rsp.body(), Charset.defaultCharset()) + errors.add(errmsg) + println(errmsg) + } + } + + println("::endgroup::") + if (errors.isNotEmpty()) { + error(errors.joinToString("\n\n", prefix = "\n")) + } + } + + else -> error("Unknown command: " + args.joinToString(" ")) + } +} diff --git a/ci-release-helper/src/package.kt b/ci-release-helper/src/package.kt new file mode 100644 index 000000000..86476e349 --- /dev/null +++ b/ci-release-helper/src/package.kt @@ -0,0 +1,11 @@ +/* + * Copyright 2019-2022 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/dev/LICENSE + */ + +package cihelper +