tursom/TursomServer
0
0
Fork 0
mirror of https://github.com/tursom/TursomServer.git synced 2025-03-03 22:30:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

优化 Web 写入性能

Browse Source
This commit is contained in:
tursom 2019-11-21 16:26:46 +08:00
parent 367b3067c0
commit 3993a8af1f
7 changed files with 309 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -4,4 +4,5 @@ gradle
gradlew
gradlew.bat
build
*/build/
*/build/
gradle.properties

22
build.gradle
View File

@ -1,5 +1,5 @@
buildscript {
ext.kotlinVersion = '1.3.50'
ext.kotlinVersion = '1.3.60'
repositories {
mavenLocal()
@ -11,6 +11,7 @@ buildscript {
}
allprojects {
apply plugin: "maven-publish"
apply plugin: 'java'
apply plugin: 'kotlin'
@ -46,4 +47,23 @@ allprojects {
artifacts {
archives sourcesJar
}
publishing {
repositories {
maven {
name = "GitHubPackages"
url = uri("https://maven.pkg.github.com/tursom/TursomServer")
credentials {
println project.findProperty("gpr.user")
//username = project.findProperty("gpr.user") ?: System.getenv("USERNAME")
username = "tursom"
password = project.findProperty("gpr.key") ?: System.getenv("PASSWORD")
}
}
}
publications {
gpr(MavenPublication) {
from(components.java)
}
}
}
}

470
src/main/kotlin/cn/tursom/core/datastruct/StringRadixTree.kt
View File

@ -6,257 +6,257 @@ package cn.tursom.core.datastruct
* 删除节点的时候可能会导致冗余，需要额外的检查，以保持基数树的特性
*/
class StringRadixTree<T> {
private val root = Node<T>()
private val root = Node<T>()
/**
* 为一个位置设置一个值
* @param value 需要设置的值，为 null 时会自动删除这个节点
* @return 节点的旧值
*/
operator fun set(route: String, value: T?): T? {
val context = Context(route)
var node: Node<T>? = root
var prev: Node<T> = root
var oldValue: T? = null
while (node != null) {
var nodeLocation = 0 // 节点匹配位置的指针，本应该封装起来，但没必要
// 查找直到节点或路径被搜索完毕
while (nodeLocation < node.length && !context.end) {
if (node[nodeLocation] != context.peek) {
// 这里是一个适合插入的节点，进行插入操作
insert(node, nodeLocation, context, value)
return oldValue
}
context.add
nodeLocation++
}
if (context.end) {
// 如果路径被搜索完毕
if (nodeLocation == node.length) {
// 如果 node 与路径正好匹配
oldValue = node.value
node.value = value
testNode(node)
} else {
// 不是正好匹配的，只能打断这个节点以插入数据
branchNode(node, nodeLocation, context, value)
}
return oldValue
}
prev = node
node = node.subNodes[context.peek]
}
// 没找到合适的节点，直接对查找的最后节点进行插入操作
insert(prev, prev.length, context, value)
return oldValue
}
/**
* 为一个位置设置一个值
* @param value 需要设置的值，为 null 时会自动删除这个节点
* @return 节点的旧值
*/
operator fun set(route: String, value: T?): T? {
val context = Context(route)
var node: Node<T>? = root
var prev: Node<T> = root
var oldValue: T? = null
while (node != null) {
var nodeLocation = 0 // 节点匹配位置的指针，本应该封装起来，但没必要
// 查找直到节点或路径被搜索完毕
while (nodeLocation < node.length && !context.end) {
if (node[nodeLocation] != context.peek) {
// 这里是一个适合插入的节点，进行插入操作
insert(node, nodeLocation, context, value)
return oldValue
}
context.add
nodeLocation++
}
if (context.end) {
// 如果路径被搜索完毕
if (nodeLocation == node.length) {
// 如果 node 与路径正好匹配
oldValue = node.value
node.value = value
testNode(node)
} else {
// 不是正好匹配的，只能打断这个节点以插入数据
branchNode(node, nodeLocation, context, value)
}
return oldValue
}
prev = node
node = node.subNodes[context.peek]
}
// 没找到合适的节点，直接对查找的最后节点进行插入操作
insert(prev, prev.length, context, value)
return oldValue
}
/**
* @return 节点是否有改变
*/
private fun testNode(node: Node<T>): Boolean {
if (node.value == null) {
val nodeChanged = when (node.subNodes.size) {
0 -> removeNode(node)
1 -> mergeNode(node)
else -> false
}
var parent = node.parent ?: return nodeChanged
while (testNode(parent)) {
parent = parent.parent ?: return nodeChanged
}
}
return false
}
/**
* @return 节点是否有改变
*/
private fun testNode(node: Node<T>): Boolean {
if (node.value == null) {
val nodeChanged = when (node.subNodes.size) {
0 -> removeNode(node)
1 -> mergeNode(node)
else -> false
}
var parent = node.parent ?: return nodeChanged
while (testNode(parent)) {
parent = parent.parent ?: return nodeChanged
}
}
return false
}
/**
* 移除节点，不会进行后续的检查工作
* @return 节点是否有改变
*/
private fun removeNode(node: Node<T>): Boolean {
val parent = node.parent
return if (parent != null) {
parent.remove(node[0])
true
} else {
node.str = ""
false
}
}
/**
* 移除节点，不会进行后续的检查工作
* @return 节点是否有改变
*/
private fun removeNode(node: Node<T>): Boolean {
val parent = node.parent
return if (parent != null) {
parent.remove(node[0])
true
} else {
node.str = ""
false
}
}
/**
* 合并节点与其子节点，不会进行后续的检查工作
* @return 节点是否有改变
*/
private fun mergeNode(node: Node<T>): Boolean {
// 只有节点值为0且只有一个字节点的时候才会进行合并
if (node.value != null || node.subNodes.size != 1) return false
// 魔法操作（笑）
val subNode = node.subNodes.first()!!
node.str += subNode.str
node.subNodes = subNode.subNodes
node.subNodes.forEach { (_, u) -> u.parent = subNode }
node.value = subNode.value
return true
}
/**
* 合并节点与其子节点，不会进行后续的检查工作
* @return 节点是否有改变
*/
private fun mergeNode(node: Node<T>): Boolean {
// 只有节点值为0且只有一个字节点的时候才会进行合并
if (node.value != null || node.subNodes.size != 1) return false
// 魔法操作（笑）
val subNode = node.subNodes.first()!!
node.str += subNode.str
node.subNodes = subNode.subNodes
node.subNodes.forEach { (_, u) -> u.parent = subNode }
node.value = subNode.value
return true
}
/**
* 将一个节点打断成一对父子节点，并插入一个新子节点
*/
private fun branchNode(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return
Node(node, node.str.substring(nodeLocation, node.str.length), node.value)
node.str = node.str.substring(0, nodeLocation)
if (context.end) {
node.value = value
} else {
node.value = null
//node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value)
}
}
/**
* 将一个节点打断成一对父子节点，并插入一个新子节点
*/
private fun branchNode(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return
Node(node, node.str.substring(nodeLocation, node.str.length), node.value)
node.str = node.str.substring(0, nodeLocation)
if (context.end) {
node.value = value
} else {
node.value = null
//node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value)
}
}
/**
* 针对一个节点插入数据
*/
private fun insert(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return
if (node.value == null) {
if (node.subNodes.isEmpty()) {
node.value = value
node.str = context.remains
return
} else {
if (node.str.isEmpty()) {
//node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value)
return
} else if (node.str == context.remains) {
node.value = value
return
}
}
}
if (nodeLocation != node.length) {
branchNode(node, nodeLocation, context, value)
} else {
//val subNode = Node(context.remains, value, node)
//node.subNodes[subNode[0]] = subNode
node.addSubNode(context.remains, value)
}
}
/**
* 针对一个节点插入数据
*/
private fun insert(node: Node<T>, nodeLocation: Int, context: Context, value: T?) {
value ?: return
if (node.value == null) {
if (node.subNodes.isEmpty()) {
node.value = value
node.str = context.remains
return
} else {
if (node.str.isEmpty()) {
//node.subNodes[context.peek] = Node(context.remains, value, node)
node.addSubNode(context.remains, value)
return
} else if (node.str == context.remains) {
node.value = value
return
}
}
}
if (nodeLocation != node.length) {
branchNode(node, nodeLocation, context, value)
} else {
//val subNode = Node(context.remains, value, node)
//node.subNodes[subNode[0]] = subNode
node.addSubNode(context.remains, value)
}
}
operator fun get(route: String): T? {
val context = Context(route)
var node: Node<T>? = root
while (node != null) {
var nodeLocation = 0
while (nodeLocation < node.length) {
if (node[nodeLocation] != context.get) return null
nodeLocation++
}
if (context.end) return node.value
node = node.subNodes[context.peek]
}
return null
}
operator fun get(route: String): T? {
val context = Context(route)
var node: Node<T>? = root
while (node != null) {
var nodeLocation = 0
while (nodeLocation < node.length) {
if (context.end || node[nodeLocation] != context.get) return null
nodeLocation++
}
if (context.end) return node.value
node = node.subNodes[context.peek]
}
return null
}
fun listGet(route: String): List<Pair<T?, Int>>? {
val context = Context(route)
var node: Node<T>? = root
val result = ArrayList<Pair<T?, Int>>()
var recodedSize = 0
while (node != null) {
var nodeLocation = 0
while (nodeLocation < node.length) {
if (node[nodeLocation] != context.get) {
result.add(node.value to recodedSize)
result.add(null to route.length)
return result
}
nodeLocation++
recodedSize++
}
result.add(node.value to recodedSize)
if (context.end) {
return result
}
node = node.subNodes[context.peek]
}
return result
}
fun listGet(route: String): List<Pair<T?, Int>>? {
val context = Context(route)
var node: Node<T>? = root
val result = ArrayList<Pair<T?, Int>>()
var recodedSize = 0
while (node != null) {
var nodeLocation = 0
while (nodeLocation < node.length) {
if (node[nodeLocation] != context.get) {
result.add(node.value to recodedSize)
result.add(null to route.length)
return result
}
nodeLocation++
recodedSize++
}
result.add(node.value to recodedSize)
if (context.end) {
return result
}
node = node.subNodes[context.peek]
}
return result
}
private fun toString(node: Node<T>, stringBuilder: StringBuilder, indentation: String) {
if (indentation.isEmpty()) {
stringBuilder.append("\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, " ")
}
} else {
stringBuilder.append("$indentation|--\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, "$indentation| ")
}
}
}
private fun toString(node: Node<T>, stringBuilder: StringBuilder, indentation: String) {
if (indentation.isEmpty()) {
stringBuilder.append("\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, " ")
}
} else {
stringBuilder.append("$indentation|--\"${node.str.replace("\"", "\"\"")}\": ${node.value}\n")
node.subNodes.forEach { subNode ->
toString(subNode.value, stringBuilder, "$indentation| ")
}
}
}
override fun toString(): String {
val stringBuilder = StringBuilder()
toString(root, stringBuilder, "")
if (stringBuilder.isNotEmpty()) stringBuilder.deleteCharAt(stringBuilder.length - 1)
return stringBuilder.toString()
}
override fun toString(): String {
val stringBuilder = StringBuilder()
toString(root, stringBuilder, "")
if (stringBuilder.isNotEmpty()) stringBuilder.deleteCharAt(stringBuilder.length - 1)
return stringBuilder.toString()
}
/**
* 基数树的节点，用来储存数据和子节点
*/
private data class Node<T>(var str: String = "", var value: T? = null, var parent: Node<T>? = null, var subNodes: SimpMap<Char, Node<T>> = ArrayMap(0)) {
constructor(parent: Node<T>, str: String = "", value: T? = null) : this(str, value, parent, parent.subNodes) {
parent.subNodes = ArrayMap(1)
parent.subNodes.setAndGet(this[0], this)
subNodes.forEach { (_, u) -> u.parent = this }
}
/**
* 基数树的节点，用来储存数据和子节点
*/
private data class Node<T>(var str: String = "", var value: T? = null, var parent: Node<T>? = null, var subNodes: SimpMap<Char, Node<T>> = ArrayMap(0)) {
constructor(parent: Node<T>, str: String = "", value: T? = null) : this(str, value, parent, parent.subNodes) {
parent.subNodes = ArrayMap(1)
parent.subNodes.setAndGet(this[0], this)
subNodes.forEach { (_, u) -> u.parent = this }
}
val length get() = str.length
operator fun get(index: Int) = str[index]
fun addSubNode(key: String, value: T?) {
if (subNodes is ArrayMap && subNodes.size > 16) {
val oldNodes = subNodes
subNodes = SimpHashMap()
subNodes.putAll(oldNodes)
}
subNodes.setAndGet(key[0], Node(key, value, this))
}
val length get() = str.length
operator fun get(index: Int) = str[index]
fun addSubNode(key: String, value: T?) {
if (subNodes is ArrayMap && subNodes.size > 16) {
val oldNodes = subNodes
subNodes = SimpHashMap()
subNodes.putAll(oldNodes)
}
subNodes.setAndGet(key[0], Node(key, value, this))
}
fun remove(key: Char) {
if (subNodes is SimpHashMap && subNodes.size < 8) {
val oldNodes = subNodes
subNodes = ArrayMap()
subNodes.putAll(oldNodes)
}
subNodes.delete(key)
}
fun remove(key: Char) {
if (subNodes is SimpHashMap && subNodes.size < 8) {
val oldNodes = subNodes
subNodes = ArrayMap()
subNodes.putAll(oldNodes)
}
subNodes.delete(key)
}
override fun toString(): String {
val sb = StringBuilder("Node(str=\"$str\", value=\"$value\", parent=\"${parent?.str}\", subNodes=")
subNodes.forEach { (t, _) ->
sb.append(t)
}
return sb.toString()
}
}
override fun toString(): String {
val sb = StringBuilder("Node(str=\"$str\", value=\"$value\", parent=\"${parent?.str}\", subNodes=")
subNodes.forEach { (t, _) ->
sb.append(t)
}
return sb.toString()
}
}
/**
* 路径匹配时需要的环境
*/
private data class Context(private val route: String, private var location: Int = 0) {
val peek get() = route[location]
val get get() = route[location++]
val add get() = location++
val end get() = location == route.length
val remains get() = if (location <= 0) route else route.substring(location, route.length)
}
/**
* 路径匹配时需要的环境
*/
private data class Context(private val route: String, private var location: Int = 0) {
val peek get() = route[location]
val get get() = route[location++]
val add get() = location++
val end get() = location == route.length
val remains get() = if (location <= 0) route else route.substring(location, route.length)
}
}
//fun main() {

37
web/netty-web/src/main/kotlin/cn/tursom/web/netty/NettyHttpContent.kt
View File

@ -5,6 +5,7 @@ import cn.tursom.core.buffer.ByteBuffer
import cn.tursom.web.AdvanceHttpContent
import cn.tursom.web.utils.Chunked
import io.netty.buffer.ByteBuf
import io.netty.buffer.CompositeByteBuf
import io.netty.buffer.Unpooled
import io.netty.channel.ChannelHandlerContext
import io.netty.handler.codec.http.*
@ -41,11 +42,12 @@ open class NettyHttpContent(
val responseMap = HashMap<String, Any>()
val responseListMap = HashMap<String, ArrayList<Any>>()
override val responseBody = ByteArrayOutputStream()
//override val responseBody = ByteArrayOutputStream()
override var responseCode: Int = 200
override var responseMessage: String? = null
override val method: String get() = httpMethod.name()
val chunkedList = ArrayList<ByteBuffer>()
val responseBodyBuf: CompositeByteBuf = ctx.alloc().compositeBuffer()!!
override fun getHeader(header: String): String? {
return headers[header]
@ -88,27 +90,36 @@ open class NettyHttpContent(
}
override fun write(message: String) {
responseBody.write(message.toByteArray())
responseBodyBuf.addComponent(Unpooled.wrappedBuffer(message.toByteArray()))
//responseBody.write(message.toByteArray())
}
override fun write(byte: Byte) {
responseBody.write(byte.toInt())
val buffer = ctx.alloc().buffer(1).writeByte(byte.toInt())
responseBodyBuf.addComponent(buffer)
//responseBody.write(byte.toInt())
}
override fun write(bytes: ByteArray, offset: Int, size: Int) {
responseBody.write(bytes, offset, size)
responseBodyBuf.addComponent(Unpooled.wrappedBuffer(bytes, offset, size))
//responseBody.write(bytes, offset, size)
}
override fun write(buffer: ByteBuffer) {
buffer.writeTo(responseBody)
//buffer.writeTo(responseBody)
responseBodyBuf.addComponent(if (buffer is NettyByteBuffer) {
buffer.byteBuf
} else {
Unpooled.wrappedBuffer(buffer.readBuffer())
})
}
override fun reset() {
responseBody.reset()
responseBodyBuf.clear()
}
override fun finish() {
finish(responseBody.buf, 0, responseBody.size())
finish(responseBodyBuf)
}
override fun finish(buffer: ByteArray, offset: Int, size: Int) {
@ -132,13 +143,13 @@ open class NettyHttpContent(
fun finish(response: FullHttpResponse) {
val heads = response.headers()
addHeaders(
heads, mapOf(
HttpHeaderNames.CONTENT_TYPE to "${HttpHeaderValues.TEXT_PLAIN}; charset=UTF-8",
HttpHeaderNames.CONTENT_LENGTH to response.content().readableBytes(),
HttpHeaderNames.CONNECTION to HttpHeaderValues.KEEP_ALIVE
heads,
mapOf(
HttpHeaderNames.CONTENT_TYPE to "${HttpHeaderValues.TEXT_PLAIN}; charset=UTF-8",
HttpHeaderNames.CONTENT_LENGTH to response.content().readableBytes(),
HttpHeaderNames.CONNECTION to HttpHeaderValues.KEEP_ALIVE
)
)
)
ctx.writeAndFlush(response)
}

34
web/netty-web/src/main/kotlin/cn/tursom/web/netty/NettyHttpHandler.kt
View File

@ -8,25 +8,25 @@ import io.netty.handler.codec.http.FullHttpRequest
@ChannelHandler.Sharable
class NettyHttpHandler(
private val handler: HttpHandler<NettyHttpContent, NettyExceptionContent>
private val handler: HttpHandler<NettyHttpContent, NettyExceptionContent>
) : SimpleChannelInboundHandler<FullHttpRequest>() {
override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) {
val handlerContext = NettyHttpContent(ctx, msg)
try {
handler.handle(handlerContext)
} catch (e: Throwable) {
handlerContext.write("${e.javaClass}: ${e.message}")
}
}
override fun channelRead0(ctx: ChannelHandlerContext, msg: FullHttpRequest) {
val handlerContext = NettyHttpContent(ctx, msg)
try {
handler.handle(handlerContext)
} catch (e: Throwable) {
handlerContext.finish("${e.javaClass}: ${e.message}")
}
}
override fun channelReadComplete(ctx: ChannelHandlerContext) {
super.channelReadComplete(ctx)
ctx.flush()
}
override fun channelReadComplete(ctx: ChannelHandlerContext) {
super.channelReadComplete(ctx)
ctx.flush()
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable?) {
if (cause != null) handler.exception(NettyExceptionContent(ctx, cause))
ctx.close()
}
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable?) {
if (cause != null) handler.exception(NettyExceptionContent(ctx, cause))
ctx.close()
}
}

16
web/src/main/kotlin/cn/tursom/web/HttpContent.kt
View File

@ -20,7 +20,7 @@ interface HttpContent {
val body: ByteBuffer?
val clientIp: SocketAddress
val method: String
val responseBody: ByteArrayOutputStream
//val responseBody: ByteArrayOutputStream
val cookieMap: Map<String, Cookie>
val realIp
get() = getHeader("X-Forwarded-For") ?: clientIp.toString().let { str ->
@ -46,7 +46,9 @@ interface HttpContent {
fun write(buffer: ByteBuffer)
fun reset()
fun finish() = finish(responseBody.buf, 0, responseBody.count)
//fun finish() = finish(responseBody.buf, 0, responseBody.count)
fun finish()
fun finish(buffer: ByteArray, offset: Int = 0, size: Int = buffer.size - offset)
fun finish(buffer: ByteBuffer) {
finish(buffer.array, buffer.readOffset, buffer.readAllSize())
@ -189,11 +191,11 @@ interface HttpContent {
finish(302)
}
fun noCache() {
setResponseHeader("Cache-Control", "no-cache")
}
fun noCache() = cacheControl(CacheControl.NoCache)
fun noStore() = cacheControl(CacheControl.NoStore)
fun noStore() {
setResponseHeader("Cache-Control", "no-store")
fun finish(msg: String) {
write(msg)
finish()
}
}

2
web/src/main/kotlin/cn/tursom/web/utils/EmptyHttpContent.kt
View File

@ -15,7 +15,7 @@ class EmptyHttpContent(
override val body: ByteBuffer? = null,
override val clientIp: SocketAddress = InetSocketAddress(0),
override val method: String = "GET",
override val responseBody: ByteArrayOutputStream = ByteArrayOutputStream(0),
//override val responseBody: ByteArrayOutputStream = ByteArrayOutputStream(0),
override val cookieMap: Map<String, Cookie> = mapOf()
) : HttpContent {
override fun getHeader(header: String): String? = null