tursom/TursomServer
0
0
Fork 0
mirror of https://github.com/tursom/TursomServer.git synced 2025-03-03 22:30:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化 Web 写入性能

Browse Source
This commit is contained in:
tursom 2019-11-21 16:26:46 +08:00
parent 367b3067c0
commit 3993a8af1f
7 changed files with 309 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -4,4 +4,5 @@ gradle
gradlew gradlew
gradlew.bat gradlew.bat
build build
*/build/ */build/
gradle.properties

22
build.gradle
View File

@ -1,5 +1,5 @@
buildscript { buildscript {
ext.kotlinVersion = '1.3.50' ext.kotlinVersion = '1.3.60'
repositories { repositories {
mavenLocal() mavenLocal()
@ -11,6 +11,7 @@ buildscript {
} }
allprojects { allprojects {
apply plugin: "maven-publish"
apply plugin: 'java' apply plugin: 'java'
apply plugin: 'kotlin' apply plugin: 'kotlin'
@ -46,4 +47,23 @@ allprojects {
artifacts { artifacts {
archives sourcesJar archives sourcesJar
} }
publishing {
repositories {
maven {
name = "GitHubPackages"
url = uri("https://maven.pkg.github.com/tursom/TursomServer")
credentials {
println project.findProperty("gpr.user")
//username = project.findProperty("gpr.user") ?: System.getenv("USERNAME")
username = "tursom"
password = project.findProperty("gpr.key") ?: System.getenv("PASSWORD")
}
}
}
publications {
gpr(MavenPublication) {
from(components.java)
}
}
}
} }

470
src/main/kotlin/cn/tursom/core/datastruct/StringRadixTree.kt
View File

@ -6,257 +6,257 @@ package cn.tursom.core.datastruct
* 删除节点的时候可能会导致冗余，需要额外的检查，以保持基数树的特性 * 删除节点的时候可能会导致冗余，需要额外的检查，以保持基数树的特性
*/ */
class StringRadixTree<T> { class StringRadixTree<T> {
private val root = Node<T>() private val root = Node<T>()
/** /**
* 为一个位置设置一个值 * 为一个位置设置一个值
* @param value 需要设置的值，为 null 时会自动删除这个节点 * @param value 需要设置的值，为 null 时会自动删除这个节点
* @return 节点的旧值 * @return 节点的旧值
*/ */
operator fun set(route: String, value: T?): T? { operator fun set(route: String, value: T?): T? {
val context = Context(route) val context = Context(route)
var node: Node<T>? = root var node: Node<T>? = root
var prev: Node<T> = root var prev: Node<T> = root
var oldValue: T? = null var oldValue: T? = null
while (node != null) { while (node != null) {
var nodeLocation = 0 // 节点匹配位置的指针，本应该封装起来，但没必要 var nodeLocation = 0 // 节点匹配位置的指针，本应该封装起来，但没必要
// 查找直到节点或路径被搜索完毕 // 查找直到节点或路径被搜索完毕
while (nodeLocation < node.length && !context.end) { while (nodeLocation < node.length && !context.end) {
if (node[nodeLocation] != context.peek) { if (node[nodeLocation] != context.peek) {
// 这里是一个适合插入的节点，进行插入操作 // 这里是一个适合插入的节点，进行插入操作
insert(node, nodeLocation, context, value) insert(node, nodeLocation, context, value)
return oldValue return oldValue
} }
context.add context.add
nodeLocation++ nodeLocation++
} }
if (context.end) { if (context.end) {
// 如果路径被搜索完毕 // 如果路径被搜索完毕
if (nodeLocation == node.length) { if (nodeLocation == node.length) {
// 如果 node 与路径正好匹配 // 如果 node 与路径正好匹配
oldValue = node.value oldValue = node.value
node.value = value node.value = value
testNode(node) testNode(node)
} else { } else {
// 不是正好匹配的，只能打断这个节点以插入数据 // 不是正好匹配的，只能打断这个节点以插入数据
branchNode(node, nodeLocation, context, value) branchNode(node, nodeLocation, context, value)
} }
return oldValue return oldValue
} }
prev = node prev = node
node = node.subNodes[context.peek] node = node.subNodes[context.peek]
} }
// 没找到合适的节点，直接对查找的最后节点进行插入操作 // 没找到合适的节点，直接对查找的最后节点进行插入操作
insert(prev, prev.length, context, value) insert(prev, prev.length, context, value)
return oldValue return oldValue
} }
/** /**
* @return 节点是否有改变 * @return 节点是否有改变
*/ */
private fun testNode(node: Node<T>): Boolean { private fun testNode(node: Node<T>): Boolean {
if (node.value == null) { if (node.value == null) {
val nodeChanged = when (node.subNodes.size) { val nodeChanged = when (node.subNodes.size) {
0 -> removeNode(node) 0 -> removeNode(node)
1 -> mergeNode(node) 1 -> mergeNode(node)
else -> false else -> false
} }
var parent = node.parent ?: return nodeChanged var parent = node.parent ?: return nodeChanged
while (testNode(parent)) { while (testNode(parent)) {
parent = parent.parent ?: return nodeChanged parent = parent.parent ?: return nodeChanged
} }
} }
return false return false
} }
/** /**
* 移除节点，不会进行后续的检查工作 * 移除节点，不会进行后续的检查工作
* @return 节点是否有改变 * @return 节点是否有改变
*/ */
private fun removeNode(node: Node<T>): Boolean { private fun removeNode(node: Node<T>): Boolean {
val parent = node.parent val parent = node.parent
return if (parent != null) { return if (parent != null) {
parent.remove(node[0]) parent.remove(node[0])
true true
} else { } else {
node.str = "" node.str = ""
false false
} }
} }
/** /**
* 合并节点与其子节点，不会进行后续的检查工作 * 合并节点与其子节点，不会进行后续的检查工作
* @return 节点是否有改变 * @return 节点是否有改变
*/ */
private fun mergeNode(node: Node<T>): Boolean { private fun mergeNode(node: Node<T>): Boolean {
// 只有节点值为0且只有一个字节点的时候才会进行合并 // 只有节点值为0且只有一个字节点的时候才会进行合并
if (node.value != null || node.subNodes.size != 1) return false if (node.value != null || node.subNodes.size != 1) return false
// 魔法操作（笑） // 魔法操作（笑）
val subNode = node.subNodes.first()!! val subNode = node.subNodes.first()!!
node.str += subNode.str node.str += subNode.str
node.subNodes = subNode.subNodes node.subNodes = subNode.subNodes
node.subNodes.forEach { (_, u) -> u.parent = subNode } node.subNodes.forEach { (_, u) -> u.parent = subNode }
node.value = subNode.value node.value = subNode.value
return true return true
} }
/** /**
* 将一个节点打断成一对父子节点，并插入一个新子节点 * 将一个节点打断成一对父子节点，并插入一个新子节点
*/ */
private fun branchNode(node: Node<T>, nodeLocation: Int, context: Context, value: T?) { private fun branchNode(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return value ?: return
Node(node, node.str.substring(nodeLocation, node.str.length), node.value) Node(node, node.str.substring(nodeLocation, node.str.length), node.value)
node.str = node.str.substring(0, nodeLocation) node.str = node.str.substring(0, nodeLocation)
if (context.end) { if (context.end) {
node.value = value node.value = value
} else { } else {
node.value = null node.value = null
//node.subNodes[context.peek] = Node(context.remains, value, node) //node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value) node.addSubNode(context.remains, value)
} }
} }
/** /**
* 针对一个节点插入数据 * 针对一个节点插入数据
*/ */
private fun insert(node: Node<T>, nodeLocation: Int, context: Context, value: T?) { private fun insert(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return value ?: return
if (node.value == null) { if (node.value == null) {
if (node.subNodes.isEmpty()) { if (node.subNodes.isEmpty()) {
node.value = value node.value = value
node.str = context.remains node.str = context.remains
return return
} else { } else {
if (node.str.isEmpty()) { if (node.str.isEmpty()) {
//node.subNodes[context.peek] = Node(context.remains, value, node) //node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value) node.addSubNode(context.remains, value)
return return
} else if (node.str == context.remains) { } else if (node.str == context.remains) {
node.value = value node.value = value
return return
} }
} }
} }
if (nodeLocation != node.length) { if (nodeLocation != node.length) {
branchNode(node, nodeLocation, context, value) branchNode(node, nodeLocation, context, value)
} else { } else {
//val subNode = Node(context.remains, value, node) //val subNode = Node(context.remains, value, node)
//node.subNodes[subNode[0]] = subNode //node.subNodes[subNode[0]] = subNode
node.addSubNode(context.remains, value) node.addSubNode(context.remains, value)
} }
} }
operator fun get(route: String): T? { operator fun get(route: String): T? {
val context = Context(route) val context = Context(route)
var node: Node<T>? = root var node: Node<T>? = root
while (node != null) { while (node != null) {
var nodeLocation = 0 var nodeLocation = 0
while (nodeLocation < node.length) { while (nodeLocation < node.length) {
if (node[nodeLocation] != context.get) return null if (context.end || node[nodeLocation] != context.get) return null
nodeLocation++ nodeLocation++
} }
if (context.end) return node.value if (context.end) return node.value
node = node.subNodes[context.peek] node = node.subNodes[context.peek]
} }
return null return null
} }
fun listGet(route: String): List<Pair<T?, Int>>? { fun listGet(route: String): List<Pair<T?, Int>>? {
val context = Context(route) val context = Context(route)
var node: Node<T>? = root var node: Node<T>? = root
val result = ArrayList<Pair<T?, Int>>() val result = ArrayList<Pair<T?, Int>>()
var recodedSize = 0 var recodedSize = 0
while (node != null) { while (node != null) {
var nodeLocation = 0 var nodeLocation = 0
while (nodeLocation < node.length) { while (nodeLocation < node.length) {
if (node[nodeLocation] != context.get) { if (node[nodeLocation] != context.get) {
result.add(node.value to recodedSize) result.add(node.value to recodedSize)
result.add(null to route.length) result.add(null to route.length)
return result return result
} }
nodeLocation++ nodeLocation++
recodedSize++ recodedSize++
} }
result.add(node.value to recodedSize) result.add(node.value to recodedSize)
if (context.end) { if (context.end) {
return result return result
} }
node = node.subNodes[context.peek] node = node.subNodes[context.peek]
} }
return result return result
} }
private fun toString(node: Node<T>, stringBuilder: StringBuilder, indentation: String) { private fun toString(node: Node<T>, stringBuilder: StringBuilder, indentation: String) {
if (indentation.isEmpty()) { if (indentation.isEmpty()) {
stringBuilder.append("\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n") stringBuilder.append("\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode -> node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, " ") toString(subNode.value, stringBuilder, " ")
} }
} else { } else {
stringBuilder.append("$indentation|--\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n") stringBuilder.append("$indentation|--\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode -> node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, "$indentation| ") toString(subNode.value, stringBuilder, "$indentation| ")
} }
} }
} }
override fun toString(): String { override fun toString(): String {
val stringBuilder = StringBuilder() val stringBuilder = StringBuilder()
toString(root, stringBuilder, "") toString(root, stringBuilder, "")
if (stringBuilder.isNotEmpty()) stringBuilder.deleteCharAt(stringBuilder.length - 1) if (stringBuilder.isNotEmpty()) stringBuilder.deleteCharAt(stringBuilder.length - 1)
return stringBuilder.toString() return stringBuilder.toString()
} }
/** /**
* 基数树的节点，用来储存数据和子节点 * 基数树的节点，用来储存数据和子节点
*/ */
private data class Node<T>(var str: String = "", var value: T? = null, var parent: Node<T>? = null, var subNodes: SimpMap<Char, Node<T>> = ArrayMap(0)) { private data class Node<T>(var str: String = "", var value: T? = null, var parent: Node<T>? = null, var subNodes: SimpMap<Char, Node<T>> = ArrayMap(0)) {
constructor(parent: Node<T>, str: String = "", value: T? = null) : this(str, value, parent, parent.subNodes) { constructor(parent: Node<T>, str: String = "", value: T? = null) : this(str, value, parent, parent.subNodes) {
parent.subNodes = ArrayMap(1) parent.subNodes = ArrayMap(1)
parent.subNodes.setAndGet(this[0], this) parent.subNodes.setAndGet(this[0], this)
subNodes.forEach { (_, u) -> u.parent = this } subNodes.forEach { (_, u) -> u.parent = this }
} }
val length get() = str.length val length get() = str.length
operator fun get(index: Int) = str[index] operator fun get(index: Int) = str[index]
fun addSubNode(key: String, value: T?) { fun addSubNode(key: String, value: T?) {
if (subNodes is ArrayMap && subNodes.size > 16) { if (subNodes is ArrayMap && subNodes.size > 16) {
val oldNodes = subNodes val oldNodes = subNodes
subNodes = SimpHashMap() subNodes = SimpHashMap()
subNodes.putAll(oldNodes) subNodes.putAll(oldNodes)
} }
subNodes.setAndGet(key[0], Node(key, value, this)) subNodes.setAndGet(key[0], Node(key, value, this))
} }
fun remove(key: Char) { fun remove(key: Char) {
if (subNodes is SimpHashMap && subNodes.size < 8) { if (subNodes is SimpHashMap && subNodes.size < 8) {
val oldNodes = subNodes val oldNodes = subNodes
subNodes = ArrayMap() subNodes = ArrayMap()
subNodes.putAll(oldNodes) subNodes.putAll(oldNodes)
} }
subNodes.delete(key) subNodes.delete(key)
} }
override fun toString(): String { override fun toString(): String {
val sb = StringBuilder("Node(str=\"$str\", value=\"$value\", parent=\"${parent?.str}\", subNodes=") val sb = StringBuilder("Node(str=\"$str\", value=\"$value\", parent=\"${parent?.str}\", subNodes=")
subNodes.forEach { (t, _) -> subNodes.forEach { (t, _) ->
sb.append(t) sb.append(t)
} }
return sb.toString() return sb.toString()
} }
} }
/** /**
* 路径匹配时需要的环境 * 路径匹配时需要的环境
*/ */
private data class Context(private val route: String, private var location: Int = 0) { private data class Context(private val route: String, private var location: Int = 0) {
val peek get() = route[location] val peek get() = route[location]
val get get() = route[location++] val get get() = route[location++]
val add get() = location++ val add get() = location++
val end get() = location == route.length val end get() = location == route.length
val remains get() = if (location <= 0) route else route.substring(location, route.length) val remains get() = if (location <= 0) route else route.substring(location, route.length)
} }
} }
//fun main() { //fun main() {

37
web/netty-web/src/main/kotlin/cn/tursom/web/netty/NettyHttpContent.kt
View File

@ -5,6 +5,7 @@ import cn.tursom.core.buffer.ByteBuffer
import cn.tursom.web.AdvanceHttpContent import cn.tursom.web.AdvanceHttpContent
import cn.tursom.web.utils.Chunked import cn.tursom.web.utils.Chunked
import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBuf
import io.netty.buffer.CompositeByteBuf
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import io.netty.channel.ChannelHandlerContext import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.http.* import io.netty.handler.codec.http.*
@ -41,11 +42,12 @@ open class NettyHttpContent(
val responseMap = HashMap<String, Any>() val responseMap = HashMap<String, Any>()
val responseListMap = HashMap<String, ArrayList<Any>>() val responseListMap = HashMap<String, ArrayList<Any>>()
override val responseBody = ByteArrayOutputStream() //override val responseBody = ByteArrayOutputStream()
override var responseCode: Int = 200 override var responseCode: Int = 200
override var responseMessage: String? = null override var responseMessage: String? = null
override val method: String get() = httpMethod.name() override val method: String get() = httpMethod.name()
val chunkedList = ArrayList<ByteBuffer>() val chunkedList = ArrayList<ByteBuffer>()
val responseBodyBuf: CompositeByteBuf = ctx.alloc().compositeBuffer()!!
override fun getHeader(header: String): String? { override fun getHeader(header: String): String? {
return headers[header] return headers[header]
@ -88,27 +90,36 @@ open class NettyHttpContent(
} }
override fun write(message: String) { override fun write(message: String) {
responseBody.write(message.toByteArray()) responseBodyBuf.addComponent(Unpooled.wrappedBuffer(message.toByteArray()))
//responseBody.write(message.toByteArray())
} }
override fun write(byte: Byte) { override fun write(byte: Byte) {
responseBody.write(byte.toInt()) val buffer = ctx.alloc().buffer(1).writeByte(byte.toInt())
responseBodyBuf.addComponent(buffer)
//responseBody.write(byte.toInt())
} }
override fun write(bytes: ByteArray, offset: Int, size: Int) { override fun write(bytes: ByteArray, offset: Int, size: Int) {
responseBody.write(bytes, offset, size) responseBodyBuf.addComponent(Unpooled.wrappedBuffer(bytes, offset, size))
//responseBody.write(bytes, offset, size)
} }
override fun write(buffer: ByteBuffer) { override fun write(buffer: ByteBuffer) {
buffer.writeTo(responseBody) //buffer.writeTo(responseBody)
responseBodyBuf.addComponent(if (buffer is NettyByteBuffer) {
buffer.byteBuf
} else {
Unpooled.wrappedBuffer(buffer.readBuffer())
})
} }
override fun reset() { override fun reset() {
responseBody.reset() responseBodyBuf.clear()
} }
override fun finish() { override fun finish() {
finish(responseBody.buf, 0, responseBody.size()) finish(responseBodyBuf)
} }
override fun finish(buffer: ByteArray, offset: Int, size: Int) { override fun finish(buffer: ByteArray, offset: Int, size: Int) {
@ -132,13 +143,13 @@ open class NettyHttpContent(
fun finish(response: FullHttpResponse) { fun finish(response: FullHttpResponse) {
val heads = response.headers() val heads = response.headers()
addHeaders( addHeaders(
heads, mapOf( heads,
HttpHeaderNames.CONTENT_TYPE to "${HttpHeaderValues.TEXT_PLAIN}; charset=UTF-8", mapOf(
HttpHeaderNames.CONTENT_LENGTH to response.content().readableBytes(), HttpHeaderNames.CONTENT_TYPE to "${HttpHeaderValues.TEXT_PLAIN}; charset=UTF-8",
HttpHeaderNames.CONNECTION to HttpHeaderValues.KEEP_ALIVE HttpHeaderNames.CONTENT_LENGTH to response.content().readableBytes(),
HttpHeaderNames.CONNECTION to HttpHeaderValues.KEEP_ALIVE
)
) )
)
ctx.writeAndFlush(response) ctx.writeAndFlush(response)
} }

34
web/netty-web/src/main/kotlin/cn/tursom/web/netty/NettyHttpHandler.kt
View File

@ -8,25 +8,25 @@ import io.netty.handler.codec.http.FullHttpRequest
@ChannelHandler.Sharable @ChannelHandler.Sharable
class NettyHttpHandler( class NettyHttpHandler(
private val handler: HttpHandler<NettyHttpContent, NettyExceptionContent> private val handler: HttpHandler<NettyHttpContent, NettyExceptionContent>
) : SimpleChannelInboundHandler<FullHttpRequest>() { ) : SimpleChannelInboundHandler<FullHttpRequest>() {
override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) { override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) {
val handlerContext = NettyHttpContent(ctx, msg) val handlerContext = NettyHttpContent(ctx, msg)
try { try {
handler.handle(handlerContext) handler.handle(handlerContext)
} catch (e: Throwable) { } catch (e: Throwable) {
handlerContext.write("${e.javaClass}: ${e.message}") handlerContext.finish("${e.javaClass}: ${e.message}")
} }
} }
override fun channelReadComplete(ctx: ChannelHandlerContext) { override fun channelReadComplete(ctx: ChannelHandlerContext) {
super.channelReadComplete(ctx) super.channelReadComplete(ctx)
ctx.flush() ctx.flush()
} }
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable?) { override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable?) {
if (cause != null) handler.exception(NettyExceptionContent(ctx, cause)) if (cause != null) handler.exception(NettyExceptionContent(ctx, cause))
ctx.close() ctx.close()
} }
} }

16
web/src/main/kotlin/cn/tursom/web/HttpContent.kt
View File

@ -20,7 +20,7 @@ interface HttpContent {
val body: ByteBuffer? val body: ByteBuffer?
val clientIp: SocketAddress val clientIp: SocketAddress
val method: String val method: String
val responseBody: ByteArrayOutputStream //val responseBody: ByteArrayOutputStream
val cookieMap: Map<String, Cookie> val cookieMap: Map<String, Cookie>
val realIp val realIp
get() = getHeader("X-Forwarded-For") ?: clientIp.toString().let { str -> get() = getHeader("X-Forwarded-For") ?: clientIp.toString().let { str ->
@ -46,7 +46,9 @@ interface HttpContent {
fun write(buffer: ByteBuffer) fun write(buffer: ByteBuffer)
fun reset() fun reset()
fun finish() = finish(responseBody.buf, 0, responseBody.count) //fun finish() = finish(responseBody.buf, 0, responseBody.count)
fun finish()
fun finish(buffer: ByteArray, offset: Int = 0, size: Int = buffer.size - offset) fun finish(buffer: ByteArray, offset: Int = 0, size: Int = buffer.size - offset)
fun finish(buffer: ByteBuffer) { fun finish(buffer: ByteBuffer) {
finish(buffer.array, buffer.readOffset, buffer.readAllSize()) finish(buffer.array, buffer.readOffset, buffer.readAllSize())
@ -189,11 +191,11 @@ interface HttpContent {
finish(302) finish(302)
} }
fun noCache() { fun noCache() = cacheControl(CacheControl.NoCache)
setResponseHeader("Cache-Control", "no-cache") fun noStore() = cacheControl(CacheControl.NoStore)
}
fun noStore() { fun finish(msg: String) {
setResponseHeader("Cache-Control", "no-store") write(msg)
finish()
} }
} }

2
web/src/main/kotlin/cn/tursom/web/utils/EmptyHttpContent.kt
View File

@ -15,7 +15,7 @@ class EmptyHttpContent(
override val body: ByteBuffer? = null, override val body: ByteBuffer? = null,
override val clientIp: SocketAddress = InetSocketAddress(0), override val clientIp: SocketAddress = InetSocketAddress(0),
override val method: String = "GET", override val method: String = "GET",
override val responseBody: ByteArrayOutputStream = ByteArrayOutputStream(0), //override val responseBody: ByteArrayOutputStream = ByteArrayOutputStream(0),
override val cookieMap: Map<String, Cookie> = mapOf() override val cookieMap: Map<String, Cookie> = mapOf()
) : HttpContent { ) : HttpContent {
override fun getHeader(header: String): String? = null override fun getHeader(header: String): String? = null